From cfc3763358f9d4006b30820e6818941f259e2f0c Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Thu, 2 Jul 2026 17:11:37 +0800 Subject: [PATCH 01/16] feat(build): add kodo cache backend --- internal/build/cache/kodo.go | 217 +++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 internal/build/cache/kodo.go diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go new file mode 100644 index 00000000..55ef379e --- /dev/null +++ b/internal/build/cache/kodo.go @@ -0,0 +1,217 @@ +package cache + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "path" + "strings" + "time" + + qiniuclient "github.com/qiniu/go-sdk/v7/client" + "github.com/qiniu/go-sdk/v7/storagev2/credentials" + httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/go-sdk/v7/storagev2/objects" + "github.com/qiniu/go-sdk/v7/storagev2/uploader" + "github.com/qiniu/go-sdk/v7/storagev2/uptoken" +) + +const kodoEntryMetadataKey = "llar-entry" + +type KodoConfig struct { + AccessKey string + SecretKey string + Bucket string + Prefix string +} + +type kodoCache struct { + bucket string + prefix string + credentials *credentials.Credentials + objects *objects.ObjectsManager + uploader *uploader.UploadManager +} + +func NewKodo(cfg KodoConfig) Cache { + cred := credentials.NewCredentials(cfg.AccessKey, cfg.SecretKey) + options := httpclient.Options{Credentials: cred} + return &kodoCache{ + bucket: cfg.Bucket, + prefix: strings.Trim(cfg.Prefix, "/"), + credentials: cred, + objects: objects.NewObjectsManager(&objects.ObjectsManagerOptions{ + Options: options, + }), + uploader: uploader.NewUploadManager(&uploader.UploadManagerOptions{ + Options: options, + }), + } +} + +func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { + objectName := c.objectName(key) + object, err := c.objects.Bucket(c.bucket).Object(objectName).Stat().Call(ctx) + if err != nil { + if isKodoObjectNotFound(err) { + return Entry{}, false, nil + } + return Entry{}, false, err + } + entry, ok := kodoEntryFromMetadata(object.Metadata) + return entry, ok, nil +} + +func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) (Entry, error) { + entryMetadata, err := encodeKodoEntry(entry) + if err != nil { + return Entry{}, err + } + + objectName := c.objectName(key) + putPolicy, err := uptoken.NewPutPolicyWithKey(c.bucket, objectName, time.Now().Add(time.Hour)) + if err != nil { + return Entry{}, err + } + + reader, writer := io.Pipe() + errc := make(chan error, 1) + go func() { + err := writeTarGzip(writer, output) + _ = writer.CloseWithError(err) + errc <- err + }() + + err = c.uploader.UploadReader(ctx, reader, &uploader.ObjectOptions{ + BucketName: c.bucket, + ObjectName: &objectName, + FileName: path.Base(objectName), + ContentType: "application/gzip", + UpToken: uptoken.NewSigner(putPolicy, c.credentials), + Metadata: map[string]string{ + kodoEntryMetadataKey: entryMetadata, + }, + }, nil) + if err != nil { + _ = reader.CloseWithError(err) + <-errc + return Entry{}, err + } + if err := <-errc; err != nil { + return Entry{}, err + } + return entry, nil +} + +func (c *kodoCache) objectName(key Key) string { + parts := make([]string, 0, 3) + if c.prefix != "" { + parts = append(parts, c.prefix) + } + parts = append(parts, strings.Trim(key.Module.Path, "/"), key.Matrix+".tar.gz") + return strings.Join(parts, "/") +} + +func encodeKodoEntry(entry Entry) (string, error) { + data, err := json.Marshal(entry) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(data), nil +} + +func kodoEntryFromMetadata(metadata map[string]string) (Entry, bool) { + raw := metadata[kodoEntryMetadataKey] + if raw == "" { + raw = metadata["x-qn-meta-"+kodoEntryMetadataKey] + } + if raw == "" { + return Entry{}, false + } + data, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil { + return Entry{}, false + } + var entry Entry + if err := json.Unmarshal(data, &entry); err != nil { + return Entry{}, false + } + return entry, true +} + +func isKodoObjectNotFound(err error) bool { + var info *qiniuclient.ErrorInfo + return errors.As(err, &info) && info.Code == 612 +} + +func writeTarGzip(w io.Writer, src fs.FS) error { + gzw := gzip.NewWriter(w) + tw := tar.NewWriter(gzw) + + if err := fs.WalkDir(src, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if name == "." { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + if !info.IsDir() && !info.Mode().IsRegular() { + return fmt.Errorf("archive %s: unsupported file mode %s", name, info.Mode()) + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = name + if info.IsDir() && !strings.HasSuffix(header.Name, "/") { + header.Name += "/" + } + header.ModTime = time.Unix(0, 0) + header.AccessTime = time.Unix(0, 0) + header.ChangeTime = time.Unix(0, 0) + header.Uid = 0 + header.Gid = 0 + header.Uname = "" + header.Gname = "" + + if err := tw.WriteHeader(header); err != nil { + return err + } + if info.IsDir() { + return nil + } + + file, err := src.Open(name) + if err != nil { + return err + } + _, copyErr := io.Copy(tw, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }); err != nil { + _ = tw.Close() + _ = gzw.Close() + return err + } + + if err := tw.Close(); err != nil { + _ = gzw.Close() + return err + } + return gzw.Close() +} From 40fdfc97525d7b091be0bd949734369d5b3cd500 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Thu, 2 Jul 2026 17:44:25 +0800 Subject: [PATCH 02/16] fix(build): include version in kodo cache key --- internal/build/cache/kodo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 55ef379e..0ddd3c8c 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -110,11 +110,11 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) } func (c *kodoCache) objectName(key Key) string { - parts := make([]string, 0, 3) + parts := make([]string, 0, 4) if c.prefix != "" { parts = append(parts, c.prefix) } - parts = append(parts, strings.Trim(key.Module.Path, "/"), key.Matrix+".tar.gz") + parts = append(parts, strings.Trim(key.Module.Path, "/"), strings.Trim(key.Module.Version, "/"), key.Matrix+".tar.gz") return strings.Join(parts, "/") } From 50c4b859c2534af1b3522726861e1d6bb7cce9b9 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Thu, 2 Jul 2026 19:10:51 +0800 Subject: [PATCH 03/16] feat(build): record kodo cache artifacts --- go.mod | 8 +- go.sum | 21 +- internal/build/cache/kodo.go | 267 +++++++++++++++++++++++--- internal/build/cache/kodo_e2e_test.go | 225 ++++++++++++++++++++++ 4 files changed, 491 insertions(+), 30 deletions(-) create mode 100644 internal/build/cache/kodo_e2e_test.go diff --git a/go.mod b/go.mod index 87ed41a4..65f799d9 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/spf13/cobra v1.10.2 golang.org/x/mod v0.32.0 golang.org/x/sys v0.40.0 + gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 ) @@ -31,6 +32,10 @@ require ( github.com/goplus/gogen v1.20.6 // indirect github.com/goplus/reflectx v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/klauspost/compress v1.18.1 // indirect @@ -47,8 +52,9 @@ require ( github.com/visualfc/gid v0.3.0 // indirect github.com/visualfc/goembed v0.3.2 // indirect github.com/visualfc/xtype v0.2.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.40.0 // indirect modernc.org/fileutil v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index b1e1febd..3d0b574e 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,14 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -83,6 +91,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -102,6 +111,8 @@ github.com/visualfc/goembed v0.3.2/go.mod h1:jCVCz/yTJGyslo6Hta+pYxWWBuq9ADCcIVZ github.com/visualfc/xtype v0.2.0 h1:0ESNXyWHtK01kaOzOyqHsR1ZjEPdNu/IWPZkf0VOHl8= github.com/visualfc/xtype v0.2.0/go.mod h1:183MDtzLqyDkCm5zCH42vJGq/aQE5W25k3Z6UOZxLF0= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= @@ -109,16 +120,18 @@ golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 0ddd3c8c..93170edd 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -4,18 +4,26 @@ import ( "archive/tar" "compress/gzip" "context" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/fs" + "net/url" + "os" "path" + "path/filepath" "strings" "time" + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/mod/module" qiniuclient "github.com/qiniu/go-sdk/v7/client" "github.com/qiniu/go-sdk/v7/storagev2/credentials" + qiniudownloader "github.com/qiniu/go-sdk/v7/storagev2/downloader" httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client" "github.com/qiniu/go-sdk/v7/storagev2/objects" "github.com/qiniu/go-sdk/v7/storagev2/uploader" @@ -25,38 +33,71 @@ import ( const kodoEntryMetadataKey = "llar-entry" type KodoConfig struct { - AccessKey string - SecretKey string - Bucket string - Prefix string + AccessKey string + SecretKey string + Bucket string + Prefix string + WorkspaceDir string + Artifacts artifact.Store } type kodoCache struct { - bucket string - prefix string - credentials *credentials.Credentials - objects *objects.ObjectsManager - uploader *uploader.UploadManager + bucket string + prefix string + workspaceDir string + artifacts artifact.Store + credentials *credentials.Credentials + objects *objects.ObjectsManager + uploader *uploader.UploadManager + downloader *qiniudownloader.DownloadManager } func NewKodo(cfg KodoConfig) Cache { cred := credentials.NewCredentials(cfg.AccessKey, cfg.SecretKey) options := httpclient.Options{Credentials: cred} return &kodoCache{ - bucket: cfg.Bucket, - prefix: strings.Trim(cfg.Prefix, "/"), - credentials: cred, + bucket: cfg.Bucket, + prefix: strings.Trim(cfg.Prefix, "/"), + workspaceDir: cfg.WorkspaceDir, + artifacts: cfg.Artifacts, + credentials: cred, objects: objects.NewObjectsManager(&objects.ObjectsManagerOptions{ Options: options, }), uploader: uploader.NewUploadManager(&uploader.UploadManagerOptions{ Options: options, }), + downloader: qiniudownloader.NewDownloadManager(&qiniudownloader.DownloadManagerOptions{ + Options: options, + }), } } func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { objectName := c.objectName(key) + var checksum string + if c.artifacts != nil { + artifact, ok, err := c.artifacts.Get(ctx, artifactKey(key)) + if err != nil { + return Entry{}, false, err + } + if !ok { + return Entry{}, false, nil + } + if artifact.Source.Type != "kodo" { + return Entry{}, false, fmt.Errorf("artifact source type = %q, want kodo", artifact.Source.Type) + } + bucket, name, err := parseKodoSourceURL(artifact.Source.URL) + if err != nil { + return Entry{}, false, err + } + if bucket != c.bucket { + return Entry{}, false, fmt.Errorf("artifact bucket = %q, want %q", bucket, c.bucket) + } + objectName = name + checksum = artifact.Checksum + } + object, err := c.objects.Bucket(c.bucket).Object(objectName).Stat().Call(ctx) if err != nil { if isKodoObjectNotFound(err) { @@ -65,7 +106,15 @@ func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { return Entry{}, false, err } entry, ok := kodoEntryFromMetadata(object.Metadata) - return entry, ok, nil + if !ok { + return Entry{}, false, fmt.Errorf("read kodo entry metadata for %s", objectName) + } + if c.workspaceDir != "" { + if err := c.restore(ctx, key, objectName, checksum); err != nil { + return Entry{}, false, err + } + } + return entry, true, nil } func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) (Entry, error) { @@ -80,15 +129,23 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) return Entry{}, err } - reader, writer := io.Pipe() - errc := make(chan error, 1) - go func() { - err := writeTarGzip(writer, output) - _ = writer.CloseWithError(err) - errc <- err - }() + file, err := os.CreateTemp("", "llar-kodo-*.tar.gz") + if err != nil { + return Entry{}, err + } + defer os.Remove(file.Name()) + + hash := sha256.New() + if err := writeTarGzip(io.MultiWriter(file, hash), output); err != nil { + _ = file.Close() + return Entry{}, err + } + if err := file.Close(); err != nil { + return Entry{}, err + } + checksum := hex.EncodeToString(hash.Sum(nil)) - err = c.uploader.UploadReader(ctx, reader, &uploader.ObjectOptions{ + err = c.uploader.UploadFile(ctx, file.Name(), &uploader.ObjectOptions{ BucketName: c.bucket, ObjectName: &objectName, FileName: path.Base(objectName), @@ -99,12 +156,20 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) }, }, nil) if err != nil { - _ = reader.CloseWithError(err) - <-errc return Entry{}, err } - if err := <-errc; err != nil { - return Entry{}, err + if c.artifacts != nil { + if _, err := c.artifacts.Put(ctx, artifactKey(key), artifact.Artifact{ + Source: artifact.Source{ + Type: "kodo", + URL: kodoSourceURL(c.bucket, objectName), + }, + Type: "tar.gz", + Metadata: entry.Metadata, + Checksum: checksum, + }); err != nil { + return Entry{}, err + } } return entry, nil } @@ -118,6 +183,88 @@ func (c *kodoCache) objectName(key Key) string { return strings.Join(parts, "/") } +func (c *kodoCache) installDir(key Key) (string, error) { + escaped, err := module.EscapePath(key.Module.Path) + if err != nil { + return "", err + } + return filepath.Join(c.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, key.Module.Version, key.Matrix)), nil +} + +func (c *kodoCache) restore(ctx context.Context, key Key, objectName, checksum string) error { + installDir, err := c.installDir(key) + if err != nil { + return err + } + file, err := os.CreateTemp("", "llar-kodo-restore-*.tar.gz") + if err != nil { + return err + } + fileName := file.Name() + if err := file.Close(); err != nil { + _ = os.Remove(fileName) + return err + } + defer os.Remove(fileName) + + _, err = c.downloader.DownloadToFile(ctx, objectName, fileName, &qiniudownloader.ObjectOptions{ + GenerateOptions: qiniudownloader.GenerateOptions{ + BucketName: c.bucket, + }, + }) + if err != nil { + return err + } + if checksum != "" { + got, err := fileSHA256(fileName) + if err != nil { + return err + } + if got != checksum { + return fmt.Errorf("kodo artifact checksum = %s, want %s", got, checksum) + } + } + if err := os.RemoveAll(installDir); err != nil { + return err + } + if err := os.MkdirAll(installDir, 0o755); err != nil { + return err + } + file, err = os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + return extractTarGzip(file, installDir) +} + +func artifactKey(key Key) artifact.Key { + return artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + } +} + +func kodoSourceURL(bucket, objectName string) string { + return (&url.URL{Scheme: "kodo", Host: bucket, Path: "/" + objectName}).String() +} + +func parseKodoSourceURL(raw string) (string, string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", "", err + } + if u.Scheme != "kodo" || u.Host == "" { + return "", "", fmt.Errorf("invalid kodo source url %q", raw) + } + objectName := strings.TrimPrefix(u.Path, "/") + if objectName == "" { + return "", "", fmt.Errorf("invalid kodo source url %q", raw) + } + return u.Host, objectName, nil +} + func encodeKodoEntry(entry Entry) (string, error) { data, err := json.Marshal(entry) if err != nil { @@ -150,6 +297,19 @@ func isKodoObjectNotFound(err error) bool { return errors.As(err, &info) && info.Code == 612 } +func fileSHA256(name string) (string, error) { + file, err := os.Open(name) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + func writeTarGzip(w io.Writer, src fs.FS) error { gzw := gzip.NewWriter(w) tw := tar.NewWriter(gzw) @@ -215,3 +375,60 @@ func writeTarGzip(w io.Writer, src fs.FS) error { } return gzw.Close() } + +func extractTarGzip(r io.Reader, dst string) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + name, err := cleanTarName(header.Name) + if err != nil { + return err + } + target := filepath.Join(dst, name) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, header.FileInfo().Mode().Perm()); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, header.FileInfo().Mode().Perm()) + if err != nil { + return err + } + _, copyErr := io.Copy(file, tr) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + default: + return fmt.Errorf("extract %s: unsupported tar type %d", header.Name, header.Typeflag) + } + } +} + +func cleanTarName(name string) (string, error) { + name = filepath.Clean(filepath.FromSlash(name)) + if name == "." || filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("unsafe tar path %q", name) + } + return name, nil +} diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go new file mode 100644 index 00000000..eeef3aab --- /dev/null +++ b/internal/build/cache/kodo_e2e_test.go @@ -0,0 +1,225 @@ +package cache + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/mod/module" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func TestKodoObjectName(t *testing.T) { + c := NewKodo(KodoConfig{Prefix: "/cache/"}).(*kodoCache) + key := Key{ + Module: module.Version{Path: "madler/zlib", Version: "v1.3.2"}, + Matrix: "amd64-linux", + } + if got, want := c.objectName(key), "cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { + t.Fatalf("object name = %q, want %q", got, want) + } + if got, want := kodoSourceURL("llar-test", c.objectName(key)), "kodo://llar-test/cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { + t.Fatalf("source url = %q, want %q", got, want) + } +} + +func TestKodoE2E_PutGet(t *testing.T) { + accessKey := os.Getenv("QINIU_ACCESS_KEY") + secretKey := os.Getenv("QINIU_SECRET_KEY") + bucket := os.Getenv("QINIU_BUCKET") + if accessKey == "" || secretKey == "" || bucket == "" { + t.Skip("QINIU_ACCESS_KEY, QINIU_SECRET_KEY, and QINIU_BUCKET are required") + } + + prefix := strings.Trim(os.Getenv("QINIU_PREFIX"), "/") + if prefix != "" { + prefix += "/" + } + prefix += fmt.Sprintf("llar-kodo-e2e/%d", time.Now().UnixNano()) + store := newKodoE2EArtifactStore(t) + + const zlibVersion = "v1.3.1" + matrix := hostMatrix() + workspaceDir, installDir, metadata := buildZlibWithLLAR(t, zlibVersion, matrix) + c := NewKodo(KodoConfig{ + AccessKey: accessKey, + SecretKey: secretKey, + Bucket: bucket, + Prefix: prefix, + WorkspaceDir: workspaceDir, + Artifacts: store, + }).(*kodoCache) + + ctx := context.Background() + key := Key{ + Module: module.Version{Path: "madler/zlib", Version: zlibVersion}, + Matrix: matrix, + } + objectName := c.objectName(key) + if want := prefix + "/madler/zlib/" + zlibVersion + "/" + matrix + ".tar.gz"; objectName != want { + t.Fatalf("object name = %q, want %q", objectName, want) + } + defer func() { + if err := c.objects.Bucket(c.bucket).Object(objectName).Delete().Call(ctx); err != nil && !isKodoObjectNotFound(err) { + t.Errorf("delete %s: %v", objectName, err) + } + }() + + if _, ok, err := c.Get(ctx, key); err != nil { + t.Fatalf("Get before Put failed: %v", err) + } else if ok { + t.Fatalf("Get before Put hit %s", objectName) + } + + want := Entry{ + Metadata: metadata, + Deps: []module.Version{{Path: "example/dep", Version: "v1.0.0"}}, + } + got, err := c.Put(ctx, key, os.DirFS(installDir), want) + if err != nil { + t.Fatalf("Put failed: %v", err) + } + if got.Metadata != want.Metadata || !slices.Equal(got.Deps, want.Deps) { + t.Fatalf("Put entry = %+v, want %+v", got, want) + } + + stored, ok, err := store.Get(ctx, artifactKey(key)) + if err != nil { + t.Fatalf("artifact Get after Put failed: %v", err) + } + if !ok { + t.Fatal("artifact Get after Put missed") + } + if stored.Source.Type != "kodo" { + t.Fatalf("artifact source type = %q, want kodo", stored.Source.Type) + } + if stored.Source.URL != kodoSourceURL(bucket, objectName) { + t.Fatalf("artifact source url = %q, want %q", stored.Source.URL, kodoSourceURL(bucket, objectName)) + } + if stored.Type != "tar.gz" || stored.Metadata != metadata || len(stored.Checksum) != 64 { + t.Fatalf("artifact = %+v, want tar.gz metadata %q and sha256 checksum", stored, metadata) + } + + if err := os.RemoveAll(installDir); err != nil { + t.Fatalf("remove install dir before Get: %v", err) + } + got, ok, err = c.Get(ctx, key) + if err != nil { + t.Fatalf("Get after Put failed: %v", err) + } + if !ok { + t.Fatal("Get after Put missed") + } + if got.Metadata != want.Metadata || !slices.Equal(got.Deps, want.Deps) { + t.Fatalf("Get entry = %+v, want %+v", got, want) + } + if _, err := os.Stat(filepath.Join(installDir, "include", "zlib.h")); err != nil { + t.Fatalf("restored zlib include not found in %s: %v", installDir, err) + } + if _, err := os.Stat(filepath.Join(installDir, "lib")); err != nil { + t.Fatalf("restored zlib lib dir not found in %s: %v", installDir, err) + } +} + +func newKodoE2EArtifactStore(t *testing.T) artifact.Store { + t.Helper() + + var dial gorm.Dialector + if dsn := os.Getenv("POSTGRES_DSN"); dsn != "" { + dial = postgres.Open(dsn) + } else { + dial = sqlite.Open(filepath.Join(t.TempDir(), "artifacts.db")) + } + db, err := gorm.Open(dial, &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + t.Fatalf("gorm.Open: %v", err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("db.DB: %v", err) + } + t.Cleanup(func() { + if err := sqlDB.Close(); err != nil { + t.Fatalf("db.Close: %v", err) + } + }) + if err := db.Exec("DROP TABLE IF EXISTS artifacts").Error; err != nil { + t.Fatalf("drop artifacts: %v", err) + } + + store, err := artifact.NewGormStore(db) + if err != nil { + t.Fatalf("NewGormStore: %v", err) + } + return store +} + +func buildZlibWithLLAR(t *testing.T, version, matrix string) (string, string, string) { + t.Helper() + + llar, err := exec.LookPath("llar") + if err != nil { + t.Skip("llar not found in PATH") + } + for _, tool := range []string{"cmake", "git"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s not found in PATH", tool) + } + } + + dir := t.TempDir() + home := filepath.Join(dir, "home") + cacheHome := filepath.Join(dir, "cache") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create home: %v", err) + } + if err := os.MkdirAll(cacheHome, 0o755); err != nil { + t.Fatalf("create cache home: %v", err) + } + t.Setenv("HOME", home) + t.Setenv("XDG_CACHE_HOME", cacheHome) + + userCacheDir, err := os.UserCacheDir() + if err != nil { + t.Fatalf("UserCacheDir: %v", err) + } + + cmd := exec.Command(llar, "make", "madler/zlib@"+version) + cmd.Env = os.Environ() + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("llar make madler/zlib@%s failed: %v\n%s", version, err, out) + } + + metadata := strings.TrimSpace(string(out)) + if metadata != "-lz" { + t.Fatalf("llar make metadata = %q, want -lz", metadata) + } + + workspaceDir := filepath.Join(userCacheDir, ".llar", "workspaces") + installDir := filepath.Join(workspaceDir, fmt.Sprintf("madler/zlib@%s-%s", version, matrix)) + if _, err := os.Stat(filepath.Join(installDir, "include", "zlib.h")); err != nil { + t.Fatalf("zlib include not found in %s: %v", installDir, err) + } + if _, err := os.Stat(filepath.Join(installDir, "lib")); err != nil { + t.Fatalf("zlib lib dir not found in %s: %v", installDir, err) + } + return workspaceDir, installDir, metadata +} + +func hostMatrix() string { + return runtime.GOARCH + "-" + runtime.GOOS +} From d7b35adad259dd1ab4b8891b19ac2a8c6a3bd6c0 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Thu, 2 Jul 2026 19:21:12 +0800 Subject: [PATCH 04/16] test(build): use local kodo e2e formulas --- internal/build/cache/kodo_e2e_test.go | 20 ++++++++++-- .../DaveGamble/cJSON/v1.7.18/CJSON_llar.gox | 7 +++++ .../formulas/DaveGamble/cJSON/versions.json | 8 +++++ .../formulas/madler/zlib/v1.3.1/Zlib_llar.gox | 31 +++++++++++++++++++ .../formulas/madler/zlib/versions.json | 4 +++ .../pnggroup/libpng/v1.6.47/Libpng_llar.gox | 7 +++++ .../formulas/pnggroup/libpng/versions.json | 8 +++++ 7 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 testdata/kodo-e2e/formulas/DaveGamble/cJSON/v1.7.18/CJSON_llar.gox create mode 100644 testdata/kodo-e2e/formulas/DaveGamble/cJSON/versions.json create mode 100644 testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox create mode 100644 testdata/kodo-e2e/formulas/madler/zlib/versions.json create mode 100644 testdata/kodo-e2e/formulas/pnggroup/libpng/v1.6.47/Libpng_llar.gox create mode 100644 testdata/kodo-e2e/formulas/pnggroup/libpng/versions.json diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go index eeef3aab..28b00dcd 100644 --- a/internal/build/cache/kodo_e2e_test.go +++ b/internal/build/cache/kodo_e2e_test.go @@ -197,11 +197,13 @@ func buildZlibWithLLAR(t *testing.T, version, matrix string) (string, string, st t.Fatalf("UserCacheDir: %v", err) } - cmd := exec.Command(llar, "make", "madler/zlib@"+version) + formulaRoot := kodoE2EFormulaRoot(t) + cmd := exec.Command(llar, "make", "./madler/zlib@"+version) + cmd.Dir = formulaRoot cmd.Env = os.Environ() out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("llar make madler/zlib@%s failed: %v\n%s", version, err, out) + t.Fatalf("llar make ./madler/zlib@%s failed: %v\n%s", version, err, out) } metadata := strings.TrimSpace(string(out)) @@ -220,6 +222,20 @@ func buildZlibWithLLAR(t *testing.T, version, matrix string) (string, string, st return workspaceDir, installDir, metadata } +func kodoE2EFormulaRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + root := filepath.Clean(filepath.Join(filepath.Dir(file), "../../..")) + dir := filepath.Join(root, "testdata", "kodo-e2e", "formulas") + if _, err := os.Stat(filepath.Join(dir, "madler", "zlib", "versions.json")); err != nil { + t.Fatalf("kodo e2e formula root: %v", err) + } + return dir +} + func hostMatrix() string { return runtime.GOARCH + "-" + runtime.GOOS } diff --git a/testdata/kodo-e2e/formulas/DaveGamble/cJSON/v1.7.18/CJSON_llar.gox b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/v1.7.18/CJSON_llar.gox new file mode 100644 index 00000000..7878c335 --- /dev/null +++ b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/v1.7.18/CJSON_llar.gox @@ -0,0 +1,7 @@ +id "DaveGamble/cJSON" + +fromVer "v1.7.18" + +onBuild (ctx, proj, out) => { + out.setMetadata "-lcjson" +} diff --git a/testdata/kodo-e2e/formulas/DaveGamble/cJSON/versions.json b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/versions.json new file mode 100644 index 00000000..7d569631 --- /dev/null +++ b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/versions.json @@ -0,0 +1,8 @@ +{ + "path": "DaveGamble/cJSON", + "deps": { + "v1.7.18": [ + {"path": "madler/zlib", "version": "v1.3.1"} + ] + } +} diff --git a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox new file mode 100644 index 00000000..c7566c91 --- /dev/null +++ b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox @@ -0,0 +1,31 @@ +import "os" + +id "madler/zlib" + +fromVer "v1.3.1" + +onBuild (ctx, proj, out) => { + installDir, err := ctx.outputDir() + if err != nil { + out.addErr err + return + } + + exec "mkdir", "-p", installDir + "/include", installDir + "/lib" + if lastErr != nil { + out.addErr lastErr + return + } + err = os.writeFile(installDir+"/include/zlib.h", []byte("/* zlib e2e fixture */\n"), 0o644) + if err != nil { + out.addErr err + return + } + err = os.writeFile(installDir+"/lib/libz.a", []byte("zlib fixture archive\n"), 0o644) + if err != nil { + out.addErr err + return + } + + out.setMetadata "-lz" +} diff --git a/testdata/kodo-e2e/formulas/madler/zlib/versions.json b/testdata/kodo-e2e/formulas/madler/zlib/versions.json new file mode 100644 index 00000000..ca477e53 --- /dev/null +++ b/testdata/kodo-e2e/formulas/madler/zlib/versions.json @@ -0,0 +1,4 @@ +{ + "path": "madler/zlib", + "deps": {} +} diff --git a/testdata/kodo-e2e/formulas/pnggroup/libpng/v1.6.47/Libpng_llar.gox b/testdata/kodo-e2e/formulas/pnggroup/libpng/v1.6.47/Libpng_llar.gox new file mode 100644 index 00000000..0c47939d --- /dev/null +++ b/testdata/kodo-e2e/formulas/pnggroup/libpng/v1.6.47/Libpng_llar.gox @@ -0,0 +1,7 @@ +id "pnggroup/libpng" + +fromVer "v1.6.47" + +onBuild (ctx, proj, out) => { + out.setMetadata "-lpng" +} diff --git a/testdata/kodo-e2e/formulas/pnggroup/libpng/versions.json b/testdata/kodo-e2e/formulas/pnggroup/libpng/versions.json new file mode 100644 index 00000000..2dd8dddc --- /dev/null +++ b/testdata/kodo-e2e/formulas/pnggroup/libpng/versions.json @@ -0,0 +1,8 @@ +{ + "path": "pnggroup/libpng", + "deps": { + "v1.6.47": [ + {"path": "madler/zlib", "version": "v1.3.1"} + ] + } +} From 01b39afb3fb97e2606ede668af9a99302a8625ae Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Thu, 2 Jul 2026 19:39:49 +0800 Subject: [PATCH 05/16] test(build): add kodo build e2e cases --- testdata/kodo-e2e/main.go | 896 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 896 insertions(+) create mode 100644 testdata/kodo-e2e/main.go diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go new file mode 100644 index 00000000..1af6c696 --- /dev/null +++ b/testdata/kodo-e2e/main.go @@ -0,0 +1,896 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "log" + "net/url" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "sync" + "time" + + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/internal/build" + buildcache "github.com/goplus/llar/internal/build/cache" + "github.com/goplus/llar/internal/modules" + "github.com/goplus/llar/mod/module" + qiniuclient "github.com/qiniu/go-sdk/v7/client" + "github.com/qiniu/go-sdk/v7/storagev2/credentials" + qiniudownloader "github.com/qiniu/go-sdk/v7/storagev2/downloader" + httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/go-sdk/v7/storagev2/objects" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +const ( + defaultPostgresDSN = "host=localhost port=5432 user=llar password=llar dbname=llar_e2e sslmode=disable" + defaultTarget = "madler/zlib@v1.3.1" + defaultSharedTargets = "DaveGamble/cJSON@v1.7.18,pnggroup/libpng@v1.6.47" + defaultFormulaRoot = "testdata/kodo-e2e/formulas" + kodoEntryMetadataKey = "llar-entry" +) + +func main() { + var cfg config + flag.StringVar(&cfg.postgresDSN, "postgres-dsn", envOrDefault("POSTGRES_DSN", defaultPostgresDSN), "Postgres DSN; empty uses a temporary SQLite database") + flag.StringVar(&cfg.accessKey, "qiniu-access-key", os.Getenv("QINIU_ACCESS_KEY"), "Qiniu access key") + flag.StringVar(&cfg.secretKey, "qiniu-secret-key", os.Getenv("QINIU_SECRET_KEY"), "Qiniu secret key") + flag.StringVar(&cfg.bucket, "qiniu-bucket", os.Getenv("QINIU_BUCKET"), "Qiniu Kodo bucket") + flag.StringVar(&cfg.prefix, "qiniu-prefix", os.Getenv("QINIU_PREFIX"), "Qiniu Kodo object prefix") + flag.StringVar(&cfg.formulaRoot, "formula-root", defaultFormulaRoot, "local formula root") + flag.StringVar(&cfg.target, "target", defaultTarget, "target module@version") + flag.StringVar(&cfg.sharedTargets, "shared-targets", defaultSharedTargets, "comma-separated module@version targets sharing a dependency") + flag.StringVar(&cfg.matrix, "matrix", hostMatrix(), "matrix string") + flag.DurationVar(&cfg.timeout, "timeout", 15*time.Minute, "E2E timeout") + flag.Parse() + + if err := cfg.validate(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + if err := run(cfg); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +type config struct { + postgresDSN string + accessKey string + secretKey string + bucket string + prefix string + formulaRoot string + target string + sharedTargets string + matrix string + timeout time.Duration +} + +func (c *config) validate() error { + var err error + c.accessKey = strings.TrimSpace(c.accessKey) + c.secretKey = strings.TrimSpace(c.secretKey) + c.bucket = strings.TrimSpace(c.bucket) + c.prefix = strings.Trim(strings.TrimSpace(c.prefix), "/") + c.matrix = strings.TrimSpace(c.matrix) + c.formulaRoot, err = filepath.Abs(c.formulaRoot) + if err != nil { + return fmt.Errorf("formula root: %w", err) + } + if c.accessKey == "" { + return fmt.Errorf("missing required QINIU_ACCESS_KEY or -qiniu-access-key") + } + if c.secretKey == "" { + return fmt.Errorf("missing required QINIU_SECRET_KEY or -qiniu-secret-key") + } + if c.bucket == "" { + return fmt.Errorf("missing required QINIU_BUCKET or -qiniu-bucket") + } + if c.matrix == "" { + return fmt.Errorf("missing required -matrix") + } + if _, err := parseTarget(c.target); err != nil { + return fmt.Errorf("-target: %w", err) + } + targets, err := parseTargets(c.sharedTargets) + if err != nil { + return fmt.Errorf("-shared-targets: %w", err) + } + if len(targets) < 2 { + return fmt.Errorf("-shared-targets must contain at least two targets") + } + if c.timeout <= 0 { + return fmt.Errorf("-timeout must be positive") + } + if _, err := os.Stat(filepath.Join(c.formulaRoot, "madler", "zlib", "versions.json")); err != nil { + return fmt.Errorf("formula root %s: %w", c.formulaRoot, err) + } + return nil +} + +func run(cfg config) error { + ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + defer cancel() + + target, err := parseTarget(cfg.target) + if err != nil { + return err + } + sharedTargets, err := parseTargets(cfg.sharedTargets) + if err != nil { + return err + } + + runPrefix := cfg.prefix + if runPrefix != "" { + runPrefix += "/" + } + runPrefix += fmt.Sprintf("llar-kodo-e2e/%d", time.Now().UnixNano()) + + db, cleanupDB, err := openDatabase(cfg.postgresDSN) + if err != nil { + return err + } + defer cleanupDB() + if err := resetDatabase(ctx, db); err != nil { + return err + } + artifacts, err := artifact.NewGormStore(db) + if err != nil { + return fmt.Errorf("NewGormStore: %w", err) + } + if count := artifactCount(ctx, db); count != 0 { + return fmt.Errorf("artifact count after reset = %d, want 0", count) + } + + kodo := newKodoClient(cfg) + if err := kodo.deletePrefix(ctx, runPrefix); err != nil { + return fmt.Errorf("cleanup kodo prefix before run: %w", err) + } + defer func() { + if err := kodo.deletePrefix(context.Background(), runPrefix); err != nil { + log.Printf("cleanup kodo prefix after run: %v", err) + } + }() + + s := suite{ + cfg: configData{ + accessKey: cfg.accessKey, + secretKey: cfg.secretKey, + bucket: cfg.bucket, + prefix: runPrefix, + formulaRoot: cfg.formulaRoot, + target: target, + sharedTargets: sharedTargets, + matrix: cfg.matrix, + }, + formulas: newLocalFormulaStore(cfg.formulaRoot), + artifacts: artifacts, + db: db, + kodo: kodo, + } + + for _, target := range append([]module.Version{target}, sharedTargets...) { + if err := validateLocalFormula(cfg.formulaRoot, target); err != nil { + return err + } + } + + for _, step := range []struct { + name string + run func(context.Context) error + }{ + {"cold build uploads and stores artifact", s.coldBuild}, + {"repeated build uses stored artifact", s.repeatedBuild}, + {"new builder instance uses persisted artifact cache", s.persistedCache}, + {"different matrix stores independent artifact", s.differentMatrix}, + {"concurrent duplicate build stores one artifact", s.concurrentDuplicate}, + {"concurrent targets sharing dependency both complete", s.concurrentSharedDependency}, + } { + start := time.Now() + log.Printf("RUN %s", step.name) + if err := step.run(ctx); err != nil { + return fmt.Errorf("%s: %w", step.name, err) + } + log.Printf("PASS %s (%s)", step.name, time.Since(start).Round(time.Millisecond)) + } + + wantArtifacts := int64(6) + if count := artifactCount(ctx, db); count != wantArtifacts { + return fmt.Errorf("artifact count after E2E cases = %d, want %d", count, wantArtifacts) + } + if count, err := kodo.objectCount(ctx, runPrefix); err != nil { + return err + } else if count != wantArtifacts { + return fmt.Errorf("kodo object count under %s = %d, want %d", runPrefix, count, wantArtifacts) + } + log.Printf("PASS Kodo build E2E (%d artifacts)", wantArtifacts) + return nil +} + +type configData struct { + accessKey string + secretKey string + bucket string + prefix string + formulaRoot string + target module.Version + sharedTargets []module.Version + matrix string +} + +type suite struct { + cfg configData + formulas *localFormulaStore + artifacts artifact.Store + db *gorm.DB + kodo *kodoClient + + baseCache *countingCache + baseWorkspace string + baseResult build.Result + baseArtifact artifact.Artifact +} + +func (s *suite) coldBuild(ctx context.Context) error { + s.baseWorkspace = mustTempDir("llar-kodo-e2e-workspace-") + s.baseCache = s.newCache(s.baseWorkspace) + + got, err := s.build(ctx, s.cfg.target, s.cfg.matrix, s.baseWorkspace, s.baseCache) + if err != nil { + return err + } + if s.baseCache.totalPuts() != 1 { + return fmt.Errorf("cache Put calls = %d, want 1", s.baseCache.totalPuts()) + } + if got.Metadata != "-lz" { + return fmt.Errorf("metadata = %q, want -lz", got.Metadata) + } + if err := assertZlibOutput(got.OutputDir); err != nil { + return err + } + key := cacheKey(s.cfg.target, s.cfg.matrix) + art, err := s.assertStoredArtifact(ctx, key, "-lz", nil) + if err != nil { + return err + } + s.baseResult = got + s.baseArtifact = art + return nil +} + +func (s *suite) repeatedBuild(ctx context.Context) error { + got, err := s.build(ctx, s.cfg.target, s.cfg.matrix, s.baseWorkspace, s.baseCache) + if err != nil { + return err + } + if got != s.baseResult { + return fmt.Errorf("result = %+v, want %+v", got, s.baseResult) + } + if s.baseCache.totalPuts() != 1 { + return fmt.Errorf("cache Put calls after cache hit = %d, want 1", s.baseCache.totalPuts()) + } + art, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, s.cfg.matrix), "-lz", nil) + if err != nil { + return err + } + if art != s.baseArtifact { + return fmt.Errorf("stored artifact changed = %+v, want %+v", art, s.baseArtifact) + } + return nil +} + +func (s *suite) persistedCache(ctx context.Context) error { + workspace := mustTempDir("llar-kodo-e2e-persisted-") + c := s.newCache(workspace) + got, err := s.build(ctx, s.cfg.target, s.cfg.matrix, workspace, c) + if err != nil { + return err + } + if got.Metadata != s.baseResult.Metadata { + return fmt.Errorf("metadata = %q, want %q", got.Metadata, s.baseResult.Metadata) + } + if c.totalPuts() != 0 { + return fmt.Errorf("cache Put calls = %d, want 0", c.totalPuts()) + } + if err := assertZlibOutput(got.OutputDir); err != nil { + return err + } + return nil +} + +func (s *suite) differentMatrix(ctx context.Context) error { + matrix := s.cfg.matrix + "-variant" + workspace := mustTempDir("llar-kodo-e2e-matrix-") + c := s.newCache(workspace) + got, err := s.build(ctx, s.cfg.target, matrix, workspace, c) + if err != nil { + return err + } + if got.Metadata != "-lz" { + return fmt.Errorf("metadata = %q, want -lz", got.Metadata) + } + if c.totalPuts() != 1 { + return fmt.Errorf("cache Put calls = %d, want 1", c.totalPuts()) + } + if _, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, matrix), "-lz", nil); err != nil { + return err + } + return nil +} + +func (s *suite) concurrentDuplicate(ctx context.Context) error { + matrix := s.cfg.matrix + "-concurrent" + workspace := mustTempDir("llar-kodo-e2e-concurrent-") + c := s.newCache(workspace) + + results := make(chan buildResult, 2) + start := make(chan struct{}) + for range 2 { + go func() { + <-start + got, err := s.build(ctx, s.cfg.target, matrix, workspace, c) + results <- buildResult{result: got, err: err} + }() + } + close(start) + + first, err := waitBuildResult(ctx, results) + if err != nil { + return err + } + second, err := waitBuildResult(ctx, results) + if err != nil { + return err + } + if first.err != nil { + return fmt.Errorf("first build: %w", first.err) + } + if second.err != nil { + return fmt.Errorf("second build: %w", second.err) + } + if first.result != second.result { + return fmt.Errorf("concurrent result = %+v, want %+v", second.result, first.result) + } + if c.totalPuts() != 1 { + return fmt.Errorf("cache Put calls = %d, want 1", c.totalPuts()) + } + if _, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, matrix), "-lz", nil); err != nil { + return err + } + return nil +} + +func (s *suite) concurrentSharedDependency(ctx context.Context) error { + matrix := s.cfg.matrix + "-shareddep" + workspace := mustTempDir("llar-kodo-e2e-shared-") + c := s.newCache(workspace) + + results := make(chan namedBuildResult, len(s.cfg.sharedTargets)) + start := make(chan struct{}) + for _, target := range s.cfg.sharedTargets { + go func(target module.Version) { + <-start + got, err := s.build(ctx, target, matrix, workspace, c) + results <- namedBuildResult{target: target, result: got, err: err} + }(target) + } + close(start) + + gotByTarget := make(map[string]build.Result, len(s.cfg.sharedTargets)) + for range s.cfg.sharedTargets { + result, err := waitNamedBuildResult(ctx, results) + if err != nil { + return err + } + if result.err != nil { + return fmt.Errorf("build %s@%s: %w", result.target.Path, result.target.Version, result.err) + } + gotByTarget[targetKey(result.target)] = result.result + } + if c.totalPuts() != 3 { + return fmt.Errorf("cache Put calls = %d, want 3", c.totalPuts()) + } + + zlib := module.Version{Path: "madler/zlib", Version: "v1.3.1"} + if c.putCount(cacheKey(zlib, matrix)) != 1 { + return fmt.Errorf("shared dependency Put calls = %d, want 1", c.putCount(cacheKey(zlib, matrix))) + } + if _, err := s.assertStoredArtifact(ctx, cacheKey(zlib, matrix), "-lz", nil); err != nil { + return err + } + + for _, target := range s.cfg.sharedTargets { + got, ok := gotByTarget[targetKey(target)] + if !ok { + return fmt.Errorf("missing result for %s", targetKey(target)) + } + wantMetadata := map[string]string{ + "DaveGamble/cJSON": "-lcjson", + "pnggroup/libpng": "-lpng", + }[target.Path] + if got.Metadata != wantMetadata { + return fmt.Errorf("%s metadata = %q, want %q", targetKey(target), got.Metadata, wantMetadata) + } + if _, err := s.assertStoredArtifact(ctx, cacheKey(target, matrix), wantMetadata, []module.Version{zlib}); err != nil { + return err + } + } + return nil +} + +func (s *suite) build(ctx context.Context, target module.Version, matrix, workspaceDir string, c buildcache.Cache) (build.Result, error) { + mods, err := modules.Load(ctx, target, modules.Options{FormulaStore: s.formulas}) + if err != nil { + return build.Result{}, fmt.Errorf("modules.Load %s: %w", targetKey(target), err) + } + builder, err := build.NewBuilder(build.Options{ + Store: s.formulas, + MatrixStr: matrix, + WorkspaceDir: workspaceDir, + Cache: c, + }) + if err != nil { + return build.Result{}, fmt.Errorf("NewBuilder: %w", err) + } + results, err := builder.Build(ctx, mods) + if err != nil { + return build.Result{}, fmt.Errorf("Build %s: %w", targetKey(target), err) + } + if len(results) == 0 { + return build.Result{}, fmt.Errorf("Build %s returned no results", targetKey(target)) + } + return results[len(results)-1], nil +} + +func (s *suite) newCache(workspaceDir string) *countingCache { + return &countingCache{ + inner: buildcache.NewKodo(buildcache.KodoConfig{ + AccessKey: s.cfg.accessKey, + SecretKey: s.cfg.secretKey, + Bucket: s.cfg.bucket, + Prefix: s.cfg.prefix, + WorkspaceDir: workspaceDir, + Artifacts: s.artifacts, + }), + } +} + +func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, metadata string, deps []module.Version) (artifact.Artifact, error) { + got, ok, err := s.artifacts.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) + if err != nil { + return artifact.Artifact{}, fmt.Errorf("Get stored artifact %s: %w", keyString(key), err) + } + if !ok { + return artifact.Artifact{}, fmt.Errorf("stored artifact missing for %s", keyString(key)) + } + if got.Source.Type != "kodo" { + return artifact.Artifact{}, fmt.Errorf("source type = %q, want kodo", got.Source.Type) + } + if got.Type != "tar.gz" { + return artifact.Artifact{}, fmt.Errorf("artifact type = %q, want tar.gz", got.Type) + } + if got.Metadata != metadata { + return artifact.Artifact{}, fmt.Errorf("artifact metadata = %q, want %q", got.Metadata, metadata) + } + if len(got.Checksum) != 64 { + return artifact.Artifact{}, fmt.Errorf("artifact checksum = %q, want sha256 hex", got.Checksum) + } + + bucket, objectName, err := parseKodoSourceURL(got.Source.URL) + if err != nil { + return artifact.Artifact{}, err + } + if bucket != s.cfg.bucket { + return artifact.Artifact{}, fmt.Errorf("artifact bucket = %q, want %q", bucket, s.cfg.bucket) + } + wantObject := objectNameFor(s.cfg.prefix, key) + if objectName != wantObject { + return artifact.Artifact{}, fmt.Errorf("artifact object = %q, want %q", objectName, wantObject) + } + + entry, err := s.kodo.entry(ctx, objectName) + if err != nil { + return artifact.Artifact{}, err + } + if entry.Metadata != metadata { + return artifact.Artifact{}, fmt.Errorf("kodo entry metadata = %q, want %q", entry.Metadata, metadata) + } + if !slices.Equal(entry.Deps, deps) { + return artifact.Artifact{}, fmt.Errorf("kodo entry deps = %+v, want %+v", entry.Deps, deps) + } + if err := s.kodo.assertChecksum(ctx, objectName, got.Checksum); err != nil { + return artifact.Artifact{}, err + } + return got, nil +} + +type localFormulaStore struct { + root string + mu sync.Mutex + locks map[string]*sync.Mutex +} + +func newLocalFormulaStore(root string) *localFormulaStore { + return &localFormulaStore{ + root: root, + locks: make(map[string]*sync.Mutex), + } +} + +func (s *localFormulaStore) ModuleFS(ctx context.Context, modPath string) (fs.FS, error) { + dir := filepath.Join(s.root, filepath.FromSlash(modPath)) + if _, err := os.Stat(filepath.Join(dir, "versions.json")); err != nil { + return nil, fmt.Errorf("local formula %s: %w", modPath, err) + } + return os.DirFS(dir), nil +} + +func (s *localFormulaStore) LockModule(modPath string) (func(), error) { + if modPath == "" { + return nil, fmt.Errorf("empty module path") + } + s.mu.Lock() + lock := s.locks[modPath] + if lock == nil { + lock = &sync.Mutex{} + s.locks[modPath] = lock + } + s.mu.Unlock() + + lock.Lock() + return lock.Unlock, nil +} + +type countingCache struct { + inner buildcache.Cache + mu sync.Mutex + puts map[buildcache.Key]int +} + +func (c *countingCache) Get(ctx context.Context, key buildcache.Key) (buildcache.Entry, bool, error) { + return c.inner.Get(ctx, key) +} + +func (c *countingCache) Put(ctx context.Context, key buildcache.Key, output fs.FS, entry buildcache.Entry) (buildcache.Entry, error) { + got, err := c.inner.Put(ctx, key, output, entry) + if err != nil { + return buildcache.Entry{}, err + } + c.mu.Lock() + if c.puts == nil { + c.puts = make(map[buildcache.Key]int) + } + c.puts[key]++ + c.mu.Unlock() + return got, nil +} + +func (c *countingCache) putCount(key buildcache.Key) int { + c.mu.Lock() + defer c.mu.Unlock() + return c.puts[key] +} + +func (c *countingCache) totalPuts() int { + c.mu.Lock() + defer c.mu.Unlock() + var n int + for _, count := range c.puts { + n += count + } + return n +} + +type kodoClient struct { + bucket string + objects *objects.ObjectsManager + downloader *qiniudownloader.DownloadManager +} + +func newKodoClient(cfg config) *kodoClient { + cred := credentials.NewCredentials(cfg.accessKey, cfg.secretKey) + options := httpclient.Options{Credentials: cred} + return &kodoClient{ + bucket: cfg.bucket, + objects: objects.NewObjectsManager(&objects.ObjectsManagerOptions{ + Options: options, + }), + downloader: qiniudownloader.NewDownloadManager(&qiniudownloader.DownloadManagerOptions{ + Options: options, + }), + } +} + +func (c *kodoClient) entry(ctx context.Context, objectName string) (buildcache.Entry, error) { + object, err := c.objects.Bucket(c.bucket).Object(objectName).Stat().Call(ctx) + if err != nil { + return buildcache.Entry{}, fmt.Errorf("stat kodo object %s: %w", objectName, err) + } + raw := object.Metadata[kodoEntryMetadataKey] + if raw == "" { + raw = object.Metadata["x-qn-meta-"+kodoEntryMetadataKey] + } + if raw == "" { + return buildcache.Entry{}, fmt.Errorf("kodo object %s missing %s metadata", objectName, kodoEntryMetadataKey) + } + data, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil { + return buildcache.Entry{}, fmt.Errorf("decode kodo entry metadata for %s: %w", objectName, err) + } + var entry buildcache.Entry + if err := json.Unmarshal(data, &entry); err != nil { + return buildcache.Entry{}, fmt.Errorf("unmarshal kodo entry metadata for %s: %w", objectName, err) + } + return entry, nil +} + +func (c *kodoClient) assertChecksum(ctx context.Context, objectName, checksum string) error { + file, err := os.CreateTemp("", "llar-kodo-e2e-download-*.tar.gz") + if err != nil { + return err + } + name := file.Name() + if err := file.Close(); err != nil { + _ = os.Remove(name) + return err + } + defer os.Remove(name) + + if _, err := c.downloader.DownloadToFile(ctx, objectName, name, &qiniudownloader.ObjectOptions{ + GenerateOptions: qiniudownloader.GenerateOptions{ + BucketName: c.bucket, + }, + }); err != nil { + return fmt.Errorf("download kodo object %s: %w", objectName, err) + } + got, err := fileSHA256(name) + if err != nil { + return err + } + if got != checksum { + return fmt.Errorf("kodo object %s checksum = %s, want %s", objectName, got, checksum) + } + return nil +} + +func (c *kodoClient) objectCount(ctx context.Context, prefix string) (int64, error) { + lister := c.objects.Bucket(c.bucket).List(ctx, &objects.ListObjectsOptions{Prefix: prefix}) + defer lister.Close() + var count int64 + var details objects.ObjectDetails + for lister.Next(&details) { + count++ + } + if err := lister.Error(); err != nil { + return 0, fmt.Errorf("list kodo prefix %s: %w", prefix, err) + } + return count, nil +} + +func (c *kodoClient) deletePrefix(ctx context.Context, prefix string) error { + lister := c.objects.Bucket(c.bucket).List(ctx, &objects.ListObjectsOptions{Prefix: prefix}) + var names []string + var details objects.ObjectDetails + for lister.Next(&details) { + names = append(names, details.Name) + } + if err := lister.Close(); err != nil { + return fmt.Errorf("list kodo prefix %s: %w", prefix, err) + } + for _, name := range names { + if err := c.objects.Bucket(c.bucket).Object(name).Delete().Call(ctx); err != nil && !isKodoObjectNotFound(err) { + return fmt.Errorf("delete kodo object %s: %w", name, err) + } + } + return nil +} + +func openDatabase(dsn string) (*gorm.DB, func(), error) { + var dial gorm.Dialector + var cleanup func() + if strings.TrimSpace(dsn) == "" { + name := filepath.Join(mustTempDir("llar-kodo-e2e-db-"), "artifacts.db") + dial = sqlite.Open(name) + cleanup = func() {} + } else { + dial = postgres.Open(dsn) + cleanup = func() {} + } + db, err := gorm.Open(dial, &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + return nil, cleanup, fmt.Errorf("open database: %w", err) + } + sqlDB, err := db.DB() + if err != nil { + return nil, cleanup, fmt.Errorf("database handle: %w", err) + } + return db, func() { + _ = sqlDB.Close() + cleanup() + }, nil +} + +func resetDatabase(ctx context.Context, db *gorm.DB) error { + if err := db.WithContext(ctx).Exec("DROP TABLE IF EXISTS artifacts").Error; err != nil { + return fmt.Errorf("drop artifacts table: %w", err) + } + return nil +} + +func artifactCount(ctx context.Context, db *gorm.DB) int64 { + var count int64 + if err := db.WithContext(ctx).Table("artifacts").Count(&count).Error; err != nil { + return -1 + } + return count +} + +func validateLocalFormula(root string, target module.Version) error { + dir := filepath.Join(root, filepath.FromSlash(target.Path)) + if _, err := os.Stat(filepath.Join(dir, "versions.json")); err != nil { + return fmt.Errorf("local formula for %s: %w", target.Path, err) + } + return nil +} + +func parseTarget(value string) (module.Version, error) { + path, version, ok := strings.Cut(strings.TrimSpace(value), "@") + if !ok || path == "" || version == "" { + return module.Version{}, fmt.Errorf("target must be module@version, got %q", value) + } + return module.Version{Path: path, Version: version}, nil +} + +func parseTargets(value string) ([]module.Version, error) { + parts := strings.Split(value, ",") + targets := make([]module.Version, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + target, err := parseTarget(part) + if err != nil { + return nil, err + } + targets = append(targets, target) + } + return targets, nil +} + +func cacheKey(target module.Version, matrix string) buildcache.Key { + return buildcache.Key{Module: target, Matrix: matrix} +} + +func keyString(key buildcache.Key) string { + return fmt.Sprintf("%s@%s matrix=%s", key.Module.Path, key.Module.Version, key.Matrix) +} + +func targetKey(target module.Version) string { + return target.Path + "@" + target.Version +} + +func objectNameFor(prefix string, key buildcache.Key) string { + parts := make([]string, 0, 4) + if prefix != "" { + parts = append(parts, strings.Trim(prefix, "/")) + } + parts = append(parts, strings.Trim(key.Module.Path, "/"), strings.Trim(key.Module.Version, "/"), key.Matrix+".tar.gz") + return strings.Join(parts, "/") +} + +func parseKodoSourceURL(raw string) (string, string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", "", err + } + if u.Scheme != "kodo" || u.Host == "" { + return "", "", fmt.Errorf("invalid kodo source url %q", raw) + } + objectName := strings.TrimPrefix(u.Path, "/") + if objectName == "" { + return "", "", fmt.Errorf("invalid kodo source url %q", raw) + } + return u.Host, objectName, nil +} + +func assertZlibOutput(dir string) error { + for _, name := range []string{ + filepath.Join("include", "zlib.h"), + filepath.Join("lib", "libz.a"), + } { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + return fmt.Errorf("zlib output %s missing in %s: %w", name, dir, err) + } + } + return nil +} + +func isKodoObjectNotFound(err error) bool { + var info *qiniuclient.ErrorInfo + return errors.As(err, &info) && info.Code == 612 +} + +func fileSHA256(name string) (string, error) { + file, err := os.Open(name) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func mustTempDir(pattern string) string { + dir, err := os.MkdirTemp("", pattern) + if err != nil { + panic(err) + } + return dir +} + +func envOrDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func hostMatrix() string { + return runtime.GOARCH + "-" + runtime.GOOS +} + +type buildResult struct { + result build.Result + err error +} + +type namedBuildResult struct { + target module.Version + result build.Result + err error +} + +func waitBuildResult(ctx context.Context, ch <-chan buildResult) (buildResult, error) { + select { + case result := <-ch: + return result, nil + case <-ctx.Done(): + return buildResult{}, ctx.Err() + } +} + +func waitNamedBuildResult(ctx context.Context, ch <-chan namedBuildResult) (namedBuildResult, error) { + select { + case result := <-ch: + return result, nil + case <-ctx.Done(): + return namedBuildResult{}, ctx.Err() + } +} From 97b8bb5b7b0560c26ef3a05c3cc8d058490a3738 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Thu, 2 Jul 2026 20:19:05 +0800 Subject: [PATCH 06/16] fix(build): store kodo public artifact urls --- .github/workflows/kodo.yml | 3 + internal/build/cache/kodo.go | 58 ++++++++++++----- internal/build/cache/kodo_e2e_test.go | 60 ++++++++++++++++-- testdata/kodo-e2e/main.go | 89 +++++++++++++++++++++++---- 4 files changed, 181 insertions(+), 29 deletions(-) diff --git a/.github/workflows/kodo.yml b/.github/workflows/kodo.yml index fa0f3df5..ee705b04 100644 --- a/.github/workflows/kodo.yml +++ b/.github/workflows/kodo.yml @@ -40,15 +40,18 @@ jobs: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} QINIU_BUCKET: ${{ secrets.QINIU_BUCKET }} + QINIU_PUBLIC_DOMAIN: llar.liuxi.ng run: | test -n "$QINIU_ACCESS_KEY" test -n "$QINIU_SECRET_KEY" test -n "$QINIU_BUCKET" + test -n "$QINIU_PUBLIC_DOMAIN" - name: Run Kodo artifact E2E env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} QINIU_BUCKET: ${{ secrets.QINIU_BUCKET }} + QINIU_PUBLIC_DOMAIN: llar.liuxi.ng QINIU_PREFIX: ${{ secrets.QINIU_PREFIX }} run: go test -v -ldflags="-checklinkname=0" ./internal/artifact -run '^TestKodoArtifactE2E$' -count=1 diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 93170edd..158da2a6 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -36,6 +36,7 @@ type KodoConfig struct { AccessKey string SecretKey string Bucket string + PublicDomain string Prefix string WorkspaceDir string Artifacts artifact.Store @@ -43,6 +44,7 @@ type KodoConfig struct { type kodoCache struct { bucket string + publicDomain string prefix string workspaceDir string artifacts artifact.Store @@ -57,6 +59,7 @@ func NewKodo(cfg KodoConfig) Cache { options := httpclient.Options{Credentials: cred} return &kodoCache{ bucket: cfg.Bucket, + publicDomain: normalizePublicDomain(cfg.PublicDomain), prefix: strings.Trim(cfg.Prefix, "/"), workspaceDir: cfg.WorkspaceDir, artifacts: cfg.Artifacts, @@ -87,13 +90,10 @@ func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { if artifact.Source.Type != "kodo" { return Entry{}, false, fmt.Errorf("artifact source type = %q, want kodo", artifact.Source.Type) } - bucket, name, err := parseKodoSourceURL(artifact.Source.URL) + name, err := parseKodoSourceURL(artifact.Source.URL) if err != nil { return Entry{}, false, err } - if bucket != c.bucket { - return Entry{}, false, fmt.Errorf("artifact bucket = %q, want %q", bucket, c.bucket) - } objectName = name checksum = artifact.Checksum } @@ -144,6 +144,14 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) return Entry{}, err } checksum := hex.EncodeToString(hash.Sum(nil)) + var sourceURL string + if c.artifacts != nil { + var err error + sourceURL, err = kodoSourceURL(c.publicDomain, objectName) + if err != nil { + return Entry{}, err + } + } err = c.uploader.UploadFile(ctx, file.Name(), &uploader.ObjectOptions{ BucketName: c.bucket, @@ -162,7 +170,7 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) if _, err := c.artifacts.Put(ctx, artifactKey(key), artifact.Artifact{ Source: artifact.Source{ Type: "kodo", - URL: kodoSourceURL(c.bucket, objectName), + URL: sourceURL, }, Type: "tar.gz", Metadata: entry.Metadata, @@ -246,23 +254,45 @@ func artifactKey(key Key) artifact.Key { } } -func kodoSourceURL(bucket, objectName string) string { - return (&url.URL{Scheme: "kodo", Host: bucket, Path: "/" + objectName}).String() +func normalizePublicDomain(domain string) string { + domain = strings.TrimRight(strings.TrimSpace(domain), "/") + if domain == "" || strings.Contains(domain, "://") { + return domain + } + return "http://" + domain } -func parseKodoSourceURL(raw string) (string, string, error) { +func kodoSourceURL(domain, objectName string) (string, error) { + u, err := url.Parse(normalizePublicDomain(domain)) + if err != nil { + return "", err + } + if u.Scheme != "http" && u.Scheme != "https" || u.Host == "" { + return "", fmt.Errorf("kodo public domain must be http(s), got %q", domain) + } + u.Path = "/" + objectName + u.RawPath = "" + u.RawQuery = "" + u.Fragment = "" + return u.String(), nil +} + +func parseKodoSourceURL(raw string) (string, error) { u, err := url.Parse(raw) if err != nil { - return "", "", err + return "", err + } + if u.Scheme != "http" && u.Scheme != "https" || u.Host == "" { + return "", fmt.Errorf("invalid kodo source url %q", raw) } - if u.Scheme != "kodo" || u.Host == "" { - return "", "", fmt.Errorf("invalid kodo source url %q", raw) + objectName, err := url.PathUnescape(strings.TrimPrefix(u.EscapedPath(), "/")) + if err != nil { + return "", err } - objectName := strings.TrimPrefix(u.Path, "/") if objectName == "" { - return "", "", fmt.Errorf("invalid kodo source url %q", raw) + return "", fmt.Errorf("invalid kodo source url %q", raw) } - return u.Host, objectName, nil + return objectName, nil } func encodeKodoEntry(entry Entry) (string, error) { diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go index 28b00dcd..c24ad766 100644 --- a/internal/build/cache/kodo_e2e_test.go +++ b/internal/build/cache/kodo_e2e_test.go @@ -2,7 +2,11 @@ package cache import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" + "io" + "net/http" "os" "os/exec" "path/filepath" @@ -29,15 +33,23 @@ func TestKodoObjectName(t *testing.T) { if got, want := c.objectName(key), "cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { t.Fatalf("object name = %q, want %q", got, want) } - if got, want := kodoSourceURL("llar-test", c.objectName(key)), "kodo://llar-test/cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { + got, err := kodoSourceURL("llar.liuxi.ng", c.objectName(key)) + if err != nil { + t.Fatal(err) + } + if want := "http://llar.liuxi.ng/cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { t.Fatalf("source url = %q, want %q", got, want) } + if _, err := parseKodoSourceURL("file:///cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"); err == nil { + t.Fatal("non-http source url should be rejected") + } } func TestKodoE2E_PutGet(t *testing.T) { accessKey := os.Getenv("QINIU_ACCESS_KEY") secretKey := os.Getenv("QINIU_SECRET_KEY") bucket := os.Getenv("QINIU_BUCKET") + publicDomain := envOrDefault("QINIU_PUBLIC_DOMAIN", "llar.liuxi.ng") if accessKey == "" || secretKey == "" || bucket == "" { t.Skip("QINIU_ACCESS_KEY, QINIU_SECRET_KEY, and QINIU_BUCKET are required") } @@ -56,12 +68,14 @@ func TestKodoE2E_PutGet(t *testing.T) { AccessKey: accessKey, SecretKey: secretKey, Bucket: bucket, + PublicDomain: publicDomain, Prefix: prefix, WorkspaceDir: workspaceDir, Artifacts: store, }).(*kodoCache) - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() key := Key{ Module: module.Version{Path: "madler/zlib", Version: zlibVersion}, Matrix: matrix, @@ -104,12 +118,19 @@ func TestKodoE2E_PutGet(t *testing.T) { if stored.Source.Type != "kodo" { t.Fatalf("artifact source type = %q, want kodo", stored.Source.Type) } - if stored.Source.URL != kodoSourceURL(bucket, objectName) { - t.Fatalf("artifact source url = %q, want %q", stored.Source.URL, kodoSourceURL(bucket, objectName)) + wantURL, err := kodoSourceURL(publicDomain, objectName) + if err != nil { + t.Fatal(err) + } + if stored.Source.URL != wantURL { + t.Fatalf("artifact source url = %q, want %q", stored.Source.URL, wantURL) } if stored.Type != "tar.gz" || stored.Metadata != metadata || len(stored.Checksum) != 64 { t.Fatalf("artifact = %+v, want tar.gz metadata %q and sha256 checksum", stored, metadata) } + if err := assertPublicURLChecksum(ctx, stored.Source.URL, stored.Checksum); err != nil { + t.Fatal(err) + } if err := os.RemoveAll(installDir); err != nil { t.Fatalf("remove install dir before Get: %v", err) @@ -239,3 +260,34 @@ func kodoE2EFormulaRoot(t *testing.T) string { func hostMatrix() string { return runtime.GOARCH + "-" + runtime.GOOS } + +func envOrDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func assertPublicURLChecksum(ctx context.Context, rawURL, checksum string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET %s: %s", rawURL, resp.Status) + } + hash := sha256.New() + if _, err := io.Copy(hash, resp.Body); err != nil { + return err + } + got := hex.EncodeToString(hash.Sum(nil)) + if got != checksum { + return fmt.Errorf("GET %s checksum = %s, want %s", rawURL, got, checksum) + } + return nil +} diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go index 1af6c696..e14a52a6 100644 --- a/testdata/kodo-e2e/main.go +++ b/testdata/kodo-e2e/main.go @@ -12,6 +12,7 @@ import ( "io" "io/fs" "log" + "net/http" "net/url" "os" "path/filepath" @@ -42,6 +43,7 @@ const ( defaultTarget = "madler/zlib@v1.3.1" defaultSharedTargets = "DaveGamble/cJSON@v1.7.18,pnggroup/libpng@v1.6.47" defaultFormulaRoot = "testdata/kodo-e2e/formulas" + defaultPublicDomain = "llar.liuxi.ng" kodoEntryMetadataKey = "llar-entry" ) @@ -51,6 +53,7 @@ func main() { flag.StringVar(&cfg.accessKey, "qiniu-access-key", os.Getenv("QINIU_ACCESS_KEY"), "Qiniu access key") flag.StringVar(&cfg.secretKey, "qiniu-secret-key", os.Getenv("QINIU_SECRET_KEY"), "Qiniu secret key") flag.StringVar(&cfg.bucket, "qiniu-bucket", os.Getenv("QINIU_BUCKET"), "Qiniu Kodo bucket") + flag.StringVar(&cfg.publicDomain, "qiniu-public-domain", envOrDefault("QINIU_PUBLIC_DOMAIN", defaultPublicDomain), "Qiniu Kodo public download domain") flag.StringVar(&cfg.prefix, "qiniu-prefix", os.Getenv("QINIU_PREFIX"), "Qiniu Kodo object prefix") flag.StringVar(&cfg.formulaRoot, "formula-root", defaultFormulaRoot, "local formula root") flag.StringVar(&cfg.target, "target", defaultTarget, "target module@version") @@ -74,6 +77,7 @@ type config struct { accessKey string secretKey string bucket string + publicDomain string prefix string formulaRoot string target string @@ -87,6 +91,7 @@ func (c *config) validate() error { c.accessKey = strings.TrimSpace(c.accessKey) c.secretKey = strings.TrimSpace(c.secretKey) c.bucket = strings.TrimSpace(c.bucket) + c.publicDomain = normalizePublicDomain(c.publicDomain) c.prefix = strings.Trim(strings.TrimSpace(c.prefix), "/") c.matrix = strings.TrimSpace(c.matrix) c.formulaRoot, err = filepath.Abs(c.formulaRoot) @@ -102,6 +107,9 @@ func (c *config) validate() error { if c.bucket == "" { return fmt.Errorf("missing required QINIU_BUCKET or -qiniu-bucket") } + if _, err := parseHTTPURL(c.publicDomain); err != nil { + return fmt.Errorf("-qiniu-public-domain: %w", err) + } if c.matrix == "" { return fmt.Errorf("missing required -matrix") } @@ -174,6 +182,7 @@ func run(cfg config) error { accessKey: cfg.accessKey, secretKey: cfg.secretKey, bucket: cfg.bucket, + publicDomain: cfg.publicDomain, prefix: runPrefix, formulaRoot: cfg.formulaRoot, target: target, @@ -228,6 +237,7 @@ type configData struct { accessKey string secretKey string bucket string + publicDomain string prefix string formulaRoot string target module.Version @@ -465,6 +475,7 @@ func (s *suite) newCache(workspaceDir string) *countingCache { AccessKey: s.cfg.accessKey, SecretKey: s.cfg.secretKey, Bucket: s.cfg.bucket, + PublicDomain: s.cfg.publicDomain, Prefix: s.cfg.prefix, WorkspaceDir: workspaceDir, Artifacts: s.artifacts, @@ -497,17 +508,18 @@ func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, me return artifact.Artifact{}, fmt.Errorf("artifact checksum = %q, want sha256 hex", got.Checksum) } - bucket, objectName, err := parseKodoSourceURL(got.Source.URL) + objectName, err := parseKodoSourceURL(got.Source.URL) if err != nil { return artifact.Artifact{}, err } - if bucket != s.cfg.bucket { - return artifact.Artifact{}, fmt.Errorf("artifact bucket = %q, want %q", bucket, s.cfg.bucket) - } wantObject := objectNameFor(s.cfg.prefix, key) if objectName != wantObject { return artifact.Artifact{}, fmt.Errorf("artifact object = %q, want %q", objectName, wantObject) } + wantURL := publicURL(s.cfg.publicDomain, wantObject) + if got.Source.URL != wantURL { + return artifact.Artifact{}, fmt.Errorf("artifact source url = %q, want %q", got.Source.URL, wantURL) + } entry, err := s.kodo.entry(ctx, objectName) if err != nil { @@ -522,6 +534,9 @@ func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, me if err := s.kodo.assertChecksum(ctx, objectName, got.Checksum); err != nil { return artifact.Artifact{}, err } + if err := assertPublicURLChecksum(ctx, got.Source.URL, got.Checksum); err != nil { + return artifact.Artifact{}, err + } return got, nil } @@ -802,19 +817,47 @@ func objectNameFor(prefix string, key buildcache.Key) string { return strings.Join(parts, "/") } -func parseKodoSourceURL(raw string) (string, string, error) { +func normalizePublicDomain(domain string) string { + domain = strings.TrimRight(strings.TrimSpace(domain), "/") + if domain == "" || strings.Contains(domain, "://") { + return domain + } + return "http://" + domain +} + +func parseHTTPURL(raw string) (*url.URL, error) { u, err := url.Parse(raw) if err != nil { - return "", "", err + return nil, err } - if u.Scheme != "kodo" || u.Host == "" { - return "", "", fmt.Errorf("invalid kodo source url %q", raw) + if u.Scheme != "http" && u.Scheme != "https" || u.Host == "" { + return nil, fmt.Errorf("must be http(s), got %q", raw) + } + return u, nil +} + +func parseKodoSourceURL(raw string) (string, error) { + u, err := parseHTTPURL(raw) + if err != nil { + return "", err + } + objectName, err := url.PathUnescape(strings.TrimPrefix(u.EscapedPath(), "/")) + if err != nil { + return "", err } - objectName := strings.TrimPrefix(u.Path, "/") if objectName == "" { - return "", "", fmt.Errorf("invalid kodo source url %q", raw) + return "", fmt.Errorf("invalid kodo source url %q", raw) } - return u.Host, objectName, nil + return objectName, nil +} + +func publicURL(domain, objectName string) string { + u, _ := parseHTTPURL(domain) + u.Path = "/" + objectName + u.RawPath = "" + u.RawQuery = "" + u.Fragment = "" + return u.String() } func assertZlibOutput(dir string) error { @@ -847,6 +890,30 @@ func fileSHA256(name string) (string, error) { return hex.EncodeToString(hash.Sum(nil)), nil } +func assertPublicURLChecksum(ctx context.Context, rawURL, checksum string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET %s: %s", rawURL, resp.Status) + } + hash := sha256.New() + if _, err := io.Copy(hash, resp.Body); err != nil { + return err + } + got := hex.EncodeToString(hash.Sum(nil)) + if got != checksum { + return fmt.Errorf("GET %s checksum = %s, want %s", rawURL, got, checksum) + } + return nil +} + func mustTempDir(pattern string) string { dir, err := os.MkdirTemp("", pattern) if err != nil { From d040224fb07f9675851cef088efbf7467c015d8e Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 10:41:07 +0800 Subject: [PATCH 07/16] fix(build): use insert-only kodo uploads --- internal/build/cache/kodo.go | 13 +++++++++++++ internal/build/cache/kodo_e2e_test.go | 28 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 158da2a6..66cf0dd7 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -128,6 +128,7 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) if err != nil { return Entry{}, err } + putPolicy.SetInsertOnly(1) file, err := os.CreateTemp("", "llar-kodo-*.tar.gz") if err != nil { @@ -164,6 +165,13 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) }, }, nil) if err != nil { + if isKodoObjectExists(err) { + if got, ok, getErr := c.Get(ctx, key); getErr != nil { + return Entry{}, getErr + } else if ok { + return got, nil + } + } return Entry{}, err } if c.artifacts != nil { @@ -327,6 +335,11 @@ func isKodoObjectNotFound(err error) bool { return errors.As(err, &info) && info.Code == 612 } +func isKodoObjectExists(err error) bool { + var info *qiniuclient.ErrorInfo + return errors.As(err, &info) && info.Code == 614 +} + func fileSHA256(name string) (string, error) { file, err := os.Open(name) if err != nil { diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go index c24ad766..ffa8c3bc 100644 --- a/internal/build/cache/kodo_e2e_test.go +++ b/internal/build/cache/kodo_e2e_test.go @@ -132,6 +132,34 @@ func TestKodoE2E_PutGet(t *testing.T) { t.Fatal(err) } + if err := os.WriteFile(filepath.Join(installDir, "lib", "libz.a"), []byte("conflicting zlib archive\n"), 0o644); err != nil { + t.Fatalf("rewrite zlib archive before conflicting Put: %v", err) + } + conflict := Entry{ + Metadata: "-lz-conflict", + Deps: []module.Version{{Path: "example/other", Version: "v2.0.0"}}, + } + got, err = c.Put(ctx, key, os.DirFS(installDir), conflict) + if err != nil { + t.Fatalf("conflicting Put failed: %v", err) + } + if got.Metadata != want.Metadata || !slices.Equal(got.Deps, want.Deps) { + t.Fatalf("conflicting Put entry = %+v, want existing %+v", got, want) + } + afterConflict, ok, err := store.Get(ctx, artifactKey(key)) + if err != nil { + t.Fatalf("artifact Get after conflicting Put failed: %v", err) + } + if !ok { + t.Fatal("artifact Get after conflicting Put missed") + } + if afterConflict != stored { + t.Fatalf("artifact after conflicting Put = %+v, want existing %+v", afterConflict, stored) + } + if err := assertPublicURLChecksum(ctx, stored.Source.URL, stored.Checksum); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(installDir); err != nil { t.Fatalf("remove install dir before Get: %v", err) } From d4bcae11d49486da0298500447fafaf851b65f1c Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 14:29:30 +0800 Subject: [PATCH 08/16] test(build): exercise concurrent kodo uploads --- testdata/kodo-e2e/main.go | 68 +++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go index e14a52a6..eb1d682f 100644 --- a/testdata/kodo-e2e/main.go +++ b/testdata/kodo-e2e/main.go @@ -347,18 +347,24 @@ func (s *suite) differentMatrix(ctx context.Context) error { func (s *suite) concurrentDuplicate(ctx context.Context) error { matrix := s.cfg.matrix + "-concurrent" - workspace := mustTempDir("llar-kodo-e2e-concurrent-") - c := s.newCache(workspace) + key := cacheKey(s.cfg.target, matrix) + workspace1 := mustTempDir("llar-kodo-e2e-concurrent-") + workspace2 := mustTempDir("llar-kodo-e2e-concurrent-") + c1 := s.newCache(workspace1) + c2 := s.newCache(workspace2) results := make(chan buildResult, 2) start := make(chan struct{}) - for range 2 { - go func() { - <-start - got, err := s.build(ctx, s.cfg.target, matrix, workspace, c) - results <- buildResult{result: got, err: err} - }() - } + go func() { + <-start + got, err := s.build(ctx, s.cfg.target, matrix, workspace1, c1) + results <- buildResult{result: got, err: err} + }() + go func() { + <-start + got, err := s.build(ctx, s.cfg.target, matrix, workspace2, c2) + results <- buildResult{result: got, err: err} + }() close(start) first, err := waitBuildResult(ctx, results) @@ -375,13 +381,19 @@ func (s *suite) concurrentDuplicate(ctx context.Context) error { if second.err != nil { return fmt.Errorf("second build: %w", second.err) } - if first.result != second.result { - return fmt.Errorf("concurrent result = %+v, want %+v", second.result, first.result) + if first.result.Metadata != second.result.Metadata { + return fmt.Errorf("concurrent metadata = %q, want %q", second.result.Metadata, first.result.Metadata) } - if c.totalPuts() != 1 { - return fmt.Errorf("cache Put calls = %d, want 1", c.totalPuts()) + if err := assertZlibOutput(first.result.OutputDir); err != nil { + return err } - if _, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, matrix), "-lz", nil); err != nil { + if err := assertZlibOutput(second.result.OutputDir); err != nil { + return err + } + if total := c1.totalPuts() + c2.totalPuts(); total != 2 { + return fmt.Errorf("cache Put calls = %d, want 2", total) + } + if _, err := s.assertStoredArtifact(ctx, key, "-lz", nil); err != nil { return err } return nil @@ -414,13 +426,13 @@ func (s *suite) concurrentSharedDependency(ctx context.Context) error { } gotByTarget[targetKey(result.target)] = result.result } - if c.totalPuts() != 3 { - return fmt.Errorf("cache Put calls = %d, want 3", c.totalPuts()) + if c.totalPuts() != 4 { + return fmt.Errorf("cache Put calls = %d, want 4", c.totalPuts()) } zlib := module.Version{Path: "madler/zlib", Version: "v1.3.1"} - if c.putCount(cacheKey(zlib, matrix)) != 1 { - return fmt.Errorf("shared dependency Put calls = %d, want 1", c.putCount(cacheKey(zlib, matrix))) + if c.putCount(cacheKey(zlib, matrix)) != 2 { + return fmt.Errorf("shared dependency Put calls = %d, want 2", c.putCount(cacheKey(zlib, matrix))) } if _, err := s.assertStoredArtifact(ctx, cacheKey(zlib, matrix), "-lz", nil); err != nil { return err @@ -541,16 +553,11 @@ func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, me } type localFormulaStore struct { - root string - mu sync.Mutex - locks map[string]*sync.Mutex + root string } func newLocalFormulaStore(root string) *localFormulaStore { - return &localFormulaStore{ - root: root, - locks: make(map[string]*sync.Mutex), - } + return &localFormulaStore{root: root} } func (s *localFormulaStore) ModuleFS(ctx context.Context, modPath string) (fs.FS, error) { @@ -565,16 +572,7 @@ func (s *localFormulaStore) LockModule(modPath string) (func(), error) { if modPath == "" { return nil, fmt.Errorf("empty module path") } - s.mu.Lock() - lock := s.locks[modPath] - if lock == nil { - lock = &sync.Mutex{} - s.locks[modPath] = lock - } - s.mu.Unlock() - - lock.Lock() - return lock.Unlock, nil + return func() {}, nil } type countingCache struct { From 27ad575fb8d33ba8b34530144951ad223526da91 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 16:57:13 +0800 Subject: [PATCH 09/16] chore(build): drop kodo cache workflow changes --- .github/workflows/kodo.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/kodo.yml b/.github/workflows/kodo.yml index ee705b04..fa0f3df5 100644 --- a/.github/workflows/kodo.yml +++ b/.github/workflows/kodo.yml @@ -40,18 +40,15 @@ jobs: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} QINIU_BUCKET: ${{ secrets.QINIU_BUCKET }} - QINIU_PUBLIC_DOMAIN: llar.liuxi.ng run: | test -n "$QINIU_ACCESS_KEY" test -n "$QINIU_SECRET_KEY" test -n "$QINIU_BUCKET" - test -n "$QINIU_PUBLIC_DOMAIN" - name: Run Kodo artifact E2E env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} QINIU_BUCKET: ${{ secrets.QINIU_BUCKET }} - QINIU_PUBLIC_DOMAIN: llar.liuxi.ng QINIU_PREFIX: ${{ secrets.QINIU_PREFIX }} run: go test -v -ldflags="-checklinkname=0" ./internal/artifact -run '^TestKodoArtifactE2E$' -count=1 From c9377b8cbe593bd8513c523b69f9589a78180f4b Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 17:05:43 +0800 Subject: [PATCH 10/16] fix(build): rely on artifact metadata for kodo cache --- internal/build/cache/kodo.go | 80 +++++---------------------- internal/build/cache/kodo_e2e_test.go | 9 +-- testdata/kodo-e2e/main.go | 51 +++-------------- 3 files changed, 23 insertions(+), 117 deletions(-) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 66cf0dd7..446a88be 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -5,9 +5,7 @@ import ( "compress/gzip" "context" "crypto/sha256" - "encoding/base64" "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -30,8 +28,6 @@ import ( "github.com/qiniu/go-sdk/v7/storagev2/uptoken" ) -const kodoEntryMetadataKey = "llar-entry" - type KodoConfig struct { AccessKey string SecretKey string @@ -77,52 +73,32 @@ func NewKodo(cfg KodoConfig) Cache { } func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { - objectName := c.objectName(key) - var checksum string - if c.artifacts != nil { - artifact, ok, err := c.artifacts.Get(ctx, artifactKey(key)) - if err != nil { - return Entry{}, false, err - } - if !ok { - return Entry{}, false, nil - } - if artifact.Source.Type != "kodo" { - return Entry{}, false, fmt.Errorf("artifact source type = %q, want kodo", artifact.Source.Type) - } - name, err := parseKodoSourceURL(artifact.Source.URL) - if err != nil { - return Entry{}, false, err - } - objectName = name - checksum = artifact.Checksum + if c.artifacts == nil { + return Entry{}, false, nil } - - object, err := c.objects.Bucket(c.bucket).Object(objectName).Stat().Call(ctx) + art, ok, err := c.artifacts.Get(ctx, artifactKey(key)) if err != nil { - if isKodoObjectNotFound(err) { - return Entry{}, false, nil - } return Entry{}, false, err } - entry, ok := kodoEntryFromMetadata(object.Metadata) if !ok { - return Entry{}, false, fmt.Errorf("read kodo entry metadata for %s", objectName) + return Entry{}, false, nil + } + if art.Source.Type != "kodo" { + return Entry{}, false, fmt.Errorf("artifact source type = %q, want kodo", art.Source.Type) + } + objectName, err := parseKodoSourceURL(art.Source.URL) + if err != nil { + return Entry{}, false, err } if c.workspaceDir != "" { - if err := c.restore(ctx, key, objectName, checksum); err != nil { + if err := c.restore(ctx, key, objectName, art.Checksum); err != nil { return Entry{}, false, err } } - return entry, true, nil + return Entry{Metadata: art.Metadata}, true, nil } func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) (Entry, error) { - entryMetadata, err := encodeKodoEntry(entry) - if err != nil { - return Entry{}, err - } - objectName := c.objectName(key) putPolicy, err := uptoken.NewPutPolicyWithKey(c.bucket, objectName, time.Now().Add(time.Hour)) if err != nil { @@ -160,9 +136,6 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) FileName: path.Base(objectName), ContentType: "application/gzip", UpToken: uptoken.NewSigner(putPolicy, c.credentials), - Metadata: map[string]string{ - kodoEntryMetadataKey: entryMetadata, - }, }, nil) if err != nil { if isKodoObjectExists(err) { @@ -303,33 +276,6 @@ func parseKodoSourceURL(raw string) (string, error) { return objectName, nil } -func encodeKodoEntry(entry Entry) (string, error) { - data, err := json.Marshal(entry) - if err != nil { - return "", err - } - return base64.RawURLEncoding.EncodeToString(data), nil -} - -func kodoEntryFromMetadata(metadata map[string]string) (Entry, bool) { - raw := metadata[kodoEntryMetadataKey] - if raw == "" { - raw = metadata["x-qn-meta-"+kodoEntryMetadataKey] - } - if raw == "" { - return Entry{}, false - } - data, err := base64.RawURLEncoding.DecodeString(raw) - if err != nil { - return Entry{}, false - } - var entry Entry - if err := json.Unmarshal(data, &entry); err != nil { - return Entry{}, false - } - return entry, true -} - func isKodoObjectNotFound(err error) bool { var info *qiniuclient.ErrorInfo return errors.As(err, &info) && info.Code == 612 diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go index ffa8c3bc..d70fc0a0 100644 --- a/internal/build/cache/kodo_e2e_test.go +++ b/internal/build/cache/kodo_e2e_test.go @@ -11,7 +11,6 @@ import ( "os/exec" "path/filepath" "runtime" - "slices" "strings" "testing" "time" @@ -98,13 +97,12 @@ func TestKodoE2E_PutGet(t *testing.T) { want := Entry{ Metadata: metadata, - Deps: []module.Version{{Path: "example/dep", Version: "v1.0.0"}}, } got, err := c.Put(ctx, key, os.DirFS(installDir), want) if err != nil { t.Fatalf("Put failed: %v", err) } - if got.Metadata != want.Metadata || !slices.Equal(got.Deps, want.Deps) { + if got.Metadata != want.Metadata { t.Fatalf("Put entry = %+v, want %+v", got, want) } @@ -137,13 +135,12 @@ func TestKodoE2E_PutGet(t *testing.T) { } conflict := Entry{ Metadata: "-lz-conflict", - Deps: []module.Version{{Path: "example/other", Version: "v2.0.0"}}, } got, err = c.Put(ctx, key, os.DirFS(installDir), conflict) if err != nil { t.Fatalf("conflicting Put failed: %v", err) } - if got.Metadata != want.Metadata || !slices.Equal(got.Deps, want.Deps) { + if got.Metadata != want.Metadata { t.Fatalf("conflicting Put entry = %+v, want existing %+v", got, want) } afterConflict, ok, err := store.Get(ctx, artifactKey(key)) @@ -170,7 +167,7 @@ func TestKodoE2E_PutGet(t *testing.T) { if !ok { t.Fatal("Get after Put missed") } - if got.Metadata != want.Metadata || !slices.Equal(got.Deps, want.Deps) { + if got.Metadata != want.Metadata { t.Fatalf("Get entry = %+v, want %+v", got, want) } if _, err := os.Stat(filepath.Join(installDir, "include", "zlib.h")); err != nil { diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go index eb1d682f..4dd1de06 100644 --- a/testdata/kodo-e2e/main.go +++ b/testdata/kodo-e2e/main.go @@ -3,9 +3,7 @@ package main import ( "context" "crypto/sha256" - "encoding/base64" "encoding/hex" - "encoding/json" "errors" "flag" "fmt" @@ -17,7 +15,6 @@ import ( "os" "path/filepath" "runtime" - "slices" "strings" "sync" "time" @@ -44,7 +41,6 @@ const ( defaultSharedTargets = "DaveGamble/cJSON@v1.7.18,pnggroup/libpng@v1.6.47" defaultFormulaRoot = "testdata/kodo-e2e/formulas" defaultPublicDomain = "llar.liuxi.ng" - kodoEntryMetadataKey = "llar-entry" ) func main() { @@ -276,7 +272,7 @@ func (s *suite) coldBuild(ctx context.Context) error { return err } key := cacheKey(s.cfg.target, s.cfg.matrix) - art, err := s.assertStoredArtifact(ctx, key, "-lz", nil) + art, err := s.assertStoredArtifact(ctx, key, "-lz") if err != nil { return err } @@ -296,7 +292,7 @@ func (s *suite) repeatedBuild(ctx context.Context) error { if s.baseCache.totalPuts() != 1 { return fmt.Errorf("cache Put calls after cache hit = %d, want 1", s.baseCache.totalPuts()) } - art, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, s.cfg.matrix), "-lz", nil) + art, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, s.cfg.matrix), "-lz") if err != nil { return err } @@ -339,7 +335,7 @@ func (s *suite) differentMatrix(ctx context.Context) error { if c.totalPuts() != 1 { return fmt.Errorf("cache Put calls = %d, want 1", c.totalPuts()) } - if _, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, matrix), "-lz", nil); err != nil { + if _, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, matrix), "-lz"); err != nil { return err } return nil @@ -393,7 +389,7 @@ func (s *suite) concurrentDuplicate(ctx context.Context) error { if total := c1.totalPuts() + c2.totalPuts(); total != 2 { return fmt.Errorf("cache Put calls = %d, want 2", total) } - if _, err := s.assertStoredArtifact(ctx, key, "-lz", nil); err != nil { + if _, err := s.assertStoredArtifact(ctx, key, "-lz"); err != nil { return err } return nil @@ -434,7 +430,7 @@ func (s *suite) concurrentSharedDependency(ctx context.Context) error { if c.putCount(cacheKey(zlib, matrix)) != 2 { return fmt.Errorf("shared dependency Put calls = %d, want 2", c.putCount(cacheKey(zlib, matrix))) } - if _, err := s.assertStoredArtifact(ctx, cacheKey(zlib, matrix), "-lz", nil); err != nil { + if _, err := s.assertStoredArtifact(ctx, cacheKey(zlib, matrix), "-lz"); err != nil { return err } @@ -450,7 +446,7 @@ func (s *suite) concurrentSharedDependency(ctx context.Context) error { if got.Metadata != wantMetadata { return fmt.Errorf("%s metadata = %q, want %q", targetKey(target), got.Metadata, wantMetadata) } - if _, err := s.assertStoredArtifact(ctx, cacheKey(target, matrix), wantMetadata, []module.Version{zlib}); err != nil { + if _, err := s.assertStoredArtifact(ctx, cacheKey(target, matrix), wantMetadata); err != nil { return err } } @@ -495,7 +491,7 @@ func (s *suite) newCache(workspaceDir string) *countingCache { } } -func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, metadata string, deps []module.Version) (artifact.Artifact, error) { +func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, metadata string) (artifact.Artifact, error) { got, ok, err := s.artifacts.Get(ctx, artifact.Key{ Module: key.Module.Path, Version: key.Module.Version, @@ -533,16 +529,6 @@ func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, me return artifact.Artifact{}, fmt.Errorf("artifact source url = %q, want %q", got.Source.URL, wantURL) } - entry, err := s.kodo.entry(ctx, objectName) - if err != nil { - return artifact.Artifact{}, err - } - if entry.Metadata != metadata { - return artifact.Artifact{}, fmt.Errorf("kodo entry metadata = %q, want %q", entry.Metadata, metadata) - } - if !slices.Equal(entry.Deps, deps) { - return artifact.Artifact{}, fmt.Errorf("kodo entry deps = %+v, want %+v", entry.Deps, deps) - } if err := s.kodo.assertChecksum(ctx, objectName, got.Checksum); err != nil { return artifact.Artifact{}, err } @@ -635,29 +621,6 @@ func newKodoClient(cfg config) *kodoClient { } } -func (c *kodoClient) entry(ctx context.Context, objectName string) (buildcache.Entry, error) { - object, err := c.objects.Bucket(c.bucket).Object(objectName).Stat().Call(ctx) - if err != nil { - return buildcache.Entry{}, fmt.Errorf("stat kodo object %s: %w", objectName, err) - } - raw := object.Metadata[kodoEntryMetadataKey] - if raw == "" { - raw = object.Metadata["x-qn-meta-"+kodoEntryMetadataKey] - } - if raw == "" { - return buildcache.Entry{}, fmt.Errorf("kodo object %s missing %s metadata", objectName, kodoEntryMetadataKey) - } - data, err := base64.RawURLEncoding.DecodeString(raw) - if err != nil { - return buildcache.Entry{}, fmt.Errorf("decode kodo entry metadata for %s: %w", objectName, err) - } - var entry buildcache.Entry - if err := json.Unmarshal(data, &entry); err != nil { - return buildcache.Entry{}, fmt.Errorf("unmarshal kodo entry metadata for %s: %w", objectName, err) - } - return entry, nil -} - func (c *kodoClient) assertChecksum(ctx context.Context, objectName, checksum string) error { file, err := os.CreateTemp("", "llar-kodo-e2e-download-*.tar.gz") if err != nil { From 1f6b053a82c518883103b95048bc4f35be81d1f2 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 17:09:22 +0800 Subject: [PATCH 11/16] chore(build): inline kodo artifact keys --- internal/build/cache/kodo.go | 20 ++++++++++---------- internal/build/cache/kodo_e2e_test.go | 12 ++++++++++-- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 446a88be..8132e692 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -76,7 +76,11 @@ func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { if c.artifacts == nil { return Entry{}, false, nil } - art, ok, err := c.artifacts.Get(ctx, artifactKey(key)) + art, ok, err := c.artifacts.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) if err != nil { return Entry{}, false, err } @@ -148,7 +152,11 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) return Entry{}, err } if c.artifacts != nil { - if _, err := c.artifacts.Put(ctx, artifactKey(key), artifact.Artifact{ + if _, err := c.artifacts.Put(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }, artifact.Artifact{ Source: artifact.Source{ Type: "kodo", URL: sourceURL, @@ -227,14 +235,6 @@ func (c *kodoCache) restore(ctx context.Context, key Key, objectName, checksum s return extractTarGzip(file, installDir) } -func artifactKey(key Key) artifact.Key { - return artifact.Key{ - Module: key.Module.Path, - Version: key.Module.Version, - MatrixStr: key.Matrix, - } -} - func normalizePublicDomain(domain string) string { domain = strings.TrimRight(strings.TrimSpace(domain), "/") if domain == "" || strings.Contains(domain, "://") { diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go index d70fc0a0..8c496eff 100644 --- a/internal/build/cache/kodo_e2e_test.go +++ b/internal/build/cache/kodo_e2e_test.go @@ -106,7 +106,11 @@ func TestKodoE2E_PutGet(t *testing.T) { t.Fatalf("Put entry = %+v, want %+v", got, want) } - stored, ok, err := store.Get(ctx, artifactKey(key)) + stored, ok, err := store.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) if err != nil { t.Fatalf("artifact Get after Put failed: %v", err) } @@ -143,7 +147,11 @@ func TestKodoE2E_PutGet(t *testing.T) { if got.Metadata != want.Metadata { t.Fatalf("conflicting Put entry = %+v, want existing %+v", got, want) } - afterConflict, ok, err := store.Get(ctx, artifactKey(key)) + afterConflict, ok, err := store.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) if err != nil { t.Fatalf("artifact Get after conflicting Put failed: %v", err) } From 13f8eaa88d25b96577162f211b31929d6b57fdf6 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 17:41:56 +0800 Subject: [PATCH 12/16] fix(build): adapt kodo cache artifact get errors --- internal/build/cache/kodo.go | 8 ++++---- internal/build/cache/kodo_e2e_test.go | 12 +++--------- testdata/kodo-e2e/main.go | 5 +---- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 8132e692..50187e0c 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -76,17 +76,17 @@ func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { if c.artifacts == nil { return Entry{}, false, nil } - art, ok, err := c.artifacts.Get(ctx, artifact.Key{ + art, err := c.artifacts.Get(ctx, artifact.Key{ Module: key.Module.Path, Version: key.Module.Version, MatrixStr: key.Matrix, }) + if errors.Is(err, artifact.ErrNotFound) { + return Entry{}, false, nil + } if err != nil { return Entry{}, false, err } - if !ok { - return Entry{}, false, nil - } if art.Source.Type != "kodo" { return Entry{}, false, fmt.Errorf("artifact source type = %q, want kodo", art.Source.Type) } diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go index 8c496eff..05f718b6 100644 --- a/internal/build/cache/kodo_e2e_test.go +++ b/internal/build/cache/kodo_e2e_test.go @@ -106,7 +106,7 @@ func TestKodoE2E_PutGet(t *testing.T) { t.Fatalf("Put entry = %+v, want %+v", got, want) } - stored, ok, err := store.Get(ctx, artifact.Key{ + stored, err := store.Get(ctx, artifact.Key{ Module: key.Module.Path, Version: key.Module.Version, MatrixStr: key.Matrix, @@ -114,9 +114,6 @@ func TestKodoE2E_PutGet(t *testing.T) { if err != nil { t.Fatalf("artifact Get after Put failed: %v", err) } - if !ok { - t.Fatal("artifact Get after Put missed") - } if stored.Source.Type != "kodo" { t.Fatalf("artifact source type = %q, want kodo", stored.Source.Type) } @@ -147,7 +144,7 @@ func TestKodoE2E_PutGet(t *testing.T) { if got.Metadata != want.Metadata { t.Fatalf("conflicting Put entry = %+v, want existing %+v", got, want) } - afterConflict, ok, err := store.Get(ctx, artifact.Key{ + afterConflict, err := store.Get(ctx, artifact.Key{ Module: key.Module.Path, Version: key.Module.Version, MatrixStr: key.Matrix, @@ -155,9 +152,6 @@ func TestKodoE2E_PutGet(t *testing.T) { if err != nil { t.Fatalf("artifact Get after conflicting Put failed: %v", err) } - if !ok { - t.Fatal("artifact Get after conflicting Put missed") - } if afterConflict != stored { t.Fatalf("artifact after conflicting Put = %+v, want existing %+v", afterConflict, stored) } @@ -168,7 +162,7 @@ func TestKodoE2E_PutGet(t *testing.T) { if err := os.RemoveAll(installDir); err != nil { t.Fatalf("remove install dir before Get: %v", err) } - got, ok, err = c.Get(ctx, key) + got, ok, err := c.Get(ctx, key) if err != nil { t.Fatalf("Get after Put failed: %v", err) } diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go index 4dd1de06..2b7065de 100644 --- a/testdata/kodo-e2e/main.go +++ b/testdata/kodo-e2e/main.go @@ -492,7 +492,7 @@ func (s *suite) newCache(workspaceDir string) *countingCache { } func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, metadata string) (artifact.Artifact, error) { - got, ok, err := s.artifacts.Get(ctx, artifact.Key{ + got, err := s.artifacts.Get(ctx, artifact.Key{ Module: key.Module.Path, Version: key.Module.Version, MatrixStr: key.Matrix, @@ -500,9 +500,6 @@ func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, me if err != nil { return artifact.Artifact{}, fmt.Errorf("Get stored artifact %s: %w", keyString(key), err) } - if !ok { - return artifact.Artifact{}, fmt.Errorf("stored artifact missing for %s", keyString(key)) - } if got.Source.Type != "kodo" { return artifact.Artifact{}, fmt.Errorf("source type = %q, want kodo", got.Source.Type) } From d663df805b66da940b3969fbff8253dd208cf61d Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 18:00:39 +0800 Subject: [PATCH 13/16] fix(build): restore kodo artifacts by type --- internal/build/cache/kodo.go | 87 ++++++++++++++++++++--- internal/build/cache/kodo_restore_test.go | 84 ++++++++++++++++++++++ 2 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 internal/build/cache/kodo_restore_test.go diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 50187e0c..3025b50c 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -2,6 +2,7 @@ package cache import ( "archive/tar" + "archive/zip" "compress/gzip" "context" "crypto/sha256" @@ -95,7 +96,7 @@ func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { return Entry{}, false, err } if c.workspaceDir != "" { - if err := c.restore(ctx, key, objectName, art.Checksum); err != nil { + if err := c.restore(ctx, key, objectName, art.Type, art.Checksum); err != nil { return Entry{}, false, err } } @@ -188,7 +189,7 @@ func (c *kodoCache) installDir(key Key) (string, error) { return filepath.Join(c.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, key.Module.Version, key.Matrix)), nil } -func (c *kodoCache) restore(ctx context.Context, key Key, objectName, checksum string) error { +func (c *kodoCache) restore(ctx context.Context, key Key, objectName, artifactType, checksum string) error { installDir, err := c.installDir(key) if err != nil { return err @@ -227,12 +228,7 @@ func (c *kodoCache) restore(ctx context.Context, key Key, objectName, checksum s if err := os.MkdirAll(installDir, 0o755); err != nil { return err } - file, err = os.Open(fileName) - if err != nil { - return err - } - defer file.Close() - return extractTarGzip(file, installDir) + return extractArtifact(fileName, artifactType, installDir) } func normalizePublicDomain(domain string) string { @@ -365,6 +361,22 @@ func writeTarGzip(w io.Writer, src fs.FS) error { return gzw.Close() } +func extractArtifact(fileName, artifactType, dst string) error { + switch artifactType { + case "tar.gz": + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + return extractTarGzip(file, dst) + case "zip": + return extractZip(fileName, dst) + default: + return fmt.Errorf("unsupported artifact type %q", artifactType) + } +} + func extractTarGzip(r io.Reader, dst string) error { gzr, err := gzip.NewReader(r) if err != nil { @@ -381,7 +393,7 @@ func extractTarGzip(r io.Reader, dst string) error { if err != nil { return err } - name, err := cleanTarName(header.Name) + name, err := cleanArchiveName(header.Name) if err != nil { return err } @@ -414,10 +426,63 @@ func extractTarGzip(r io.Reader, dst string) error { } } -func cleanTarName(name string) (string, error) { +func extractZip(fileName, dst string) error { + zr, err := zip.OpenReader(fileName) + if err != nil { + return err + } + defer zr.Close() + + for _, entry := range zr.File { + name, err := cleanArchiveName(entry.Name) + if err != nil { + return err + } + target := filepath.Join(dst, name) + info := entry.FileInfo() + mode := info.Mode() + + if info.IsDir() { + if err := os.MkdirAll(target, mode.Perm()); err != nil { + return err + } + continue + } + if !mode.IsRegular() { + return fmt.Errorf("extract %s: unsupported zip mode %s", entry.Name, mode) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + src, err := entry.Open() + if err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode.Perm()) + if err != nil { + _ = src.Close() + return err + } + _, copyErr := io.Copy(file, src) + closeErr := file.Close() + srcCloseErr := src.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if srcCloseErr != nil { + return srcCloseErr + } + } + return nil +} + +func cleanArchiveName(name string) (string, error) { name = filepath.Clean(filepath.FromSlash(name)) if name == "." || filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("unsafe tar path %q", name) + return "", fmt.Errorf("unsafe archive path %q", name) } return name, nil } diff --git a/internal/build/cache/kodo_restore_test.go b/internal/build/cache/kodo_restore_test.go new file mode 100644 index 00000000..1bf49da6 --- /dev/null +++ b/internal/build/cache/kodo_restore_test.go @@ -0,0 +1,84 @@ +package cache + +import ( + "archive/zip" + "os" + "path/filepath" + "testing" +) + +func TestExtractArtifactUsesArtifactType(t *testing.T) { + src := t.TempDir() + if err := os.MkdirAll(filepath.Join(src, "include"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "include", "zlib.h"), []byte("zlib header\n"), 0o644); err != nil { + t.Fatal(err) + } + + t.Run("tar.gz", func(t *testing.T) { + artifactFile := filepath.Join(t.TempDir(), "artifact.tar.gz") + file, err := os.Create(artifactFile) + if err != nil { + t.Fatal(err) + } + if err := writeTarGzip(file, os.DirFS(src)); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + dst := t.TempDir() + if err := extractArtifact(artifactFile, "tar.gz", dst); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(dst, "include", "zlib.h"), "zlib header\n") + }) + + t.Run("zip", func(t *testing.T) { + artifactFile := filepath.Join(t.TempDir(), "artifact.zip") + file, err := os.Create(artifactFile) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(file) + w, err := zw.Create("include/zlib.h") + if err != nil { + _ = zw.Close() + _ = file.Close() + t.Fatal(err) + } + if _, err := w.Write([]byte("zlib header\n")); err != nil { + _ = zw.Close() + _ = file.Close() + t.Fatal(err) + } + if err := zw.Close(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + dst := t.TempDir() + if err := extractArtifact(artifactFile, "zip", dst); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(dst, "include", "zlib.h"), "zlib header\n") + }) +} + +func assertFileContent(t *testing.T, name, want string) { + t.Helper() + + got, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + if string(got) != want { + t.Fatalf("%s = %q, want %q", name, got, want) + } +} From 78e3a48e1d32eee926d637b9cf40b77913520699 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 20:00:40 +0800 Subject: [PATCH 14/16] fix(build): derive kodo restore object from cache key --- internal/build/cache/kodo.go | 23 +---------- internal/build/cache/kodo_e2e_test.go | 3 -- internal/build/cache/kodo_restore_test.go | 47 +++++++++++++++++++++++ 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 3025b50c..5afbeb90 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -91,11 +91,8 @@ func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { if art.Source.Type != "kodo" { return Entry{}, false, fmt.Errorf("artifact source type = %q, want kodo", art.Source.Type) } - objectName, err := parseKodoSourceURL(art.Source.URL) - if err != nil { - return Entry{}, false, err - } if c.workspaceDir != "" { + objectName := c.objectName(key) if err := c.restore(ctx, key, objectName, art.Type, art.Checksum); err != nil { return Entry{}, false, err } @@ -254,24 +251,6 @@ func kodoSourceURL(domain, objectName string) (string, error) { return u.String(), nil } -func parseKodoSourceURL(raw string) (string, error) { - u, err := url.Parse(raw) - if err != nil { - return "", err - } - if u.Scheme != "http" && u.Scheme != "https" || u.Host == "" { - return "", fmt.Errorf("invalid kodo source url %q", raw) - } - objectName, err := url.PathUnescape(strings.TrimPrefix(u.EscapedPath(), "/")) - if err != nil { - return "", err - } - if objectName == "" { - return "", fmt.Errorf("invalid kodo source url %q", raw) - } - return objectName, nil -} - func isKodoObjectNotFound(err error) bool { var info *qiniuclient.ErrorInfo return errors.As(err, &info) && info.Code == 612 diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go index 05f718b6..26be6651 100644 --- a/internal/build/cache/kodo_e2e_test.go +++ b/internal/build/cache/kodo_e2e_test.go @@ -39,9 +39,6 @@ func TestKodoObjectName(t *testing.T) { if want := "http://llar.liuxi.ng/cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { t.Fatalf("source url = %q, want %q", got, want) } - if _, err := parseKodoSourceURL("file:///cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"); err == nil { - t.Fatal("non-http source url should be rejected") - } } func TestKodoE2E_PutGet(t *testing.T) { diff --git a/internal/build/cache/kodo_restore_test.go b/internal/build/cache/kodo_restore_test.go index 1bf49da6..ae742285 100644 --- a/internal/build/cache/kodo_restore_test.go +++ b/internal/build/cache/kodo_restore_test.go @@ -2,11 +2,42 @@ package cache import ( "archive/zip" + "context" "os" "path/filepath" "testing" + + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/mod/module" ) +func TestKodoGetDoesNotParseSourceURL(t *testing.T) { + c := NewKodo(KodoConfig{ + Artifacts: staticArtifactStore{ + art: artifact.Artifact{ + Source: artifact.Source{Type: "kodo", URL: "not a kodo object name"}, + Type: "zip", + Metadata: "-lz", + }, + }, + }).(*kodoCache) + key := Key{ + Module: module.Version{Path: "madler/zlib", Version: "v1.3.2"}, + Matrix: "amd64-linux", + } + + got, ok, err := c.Get(context.Background(), key) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("Get missed") + } + if got.Metadata != "-lz" { + t.Fatalf("metadata = %q, want -lz", got.Metadata) + } +} + func TestExtractArtifactUsesArtifactType(t *testing.T) { src := t.TempDir() if err := os.MkdirAll(filepath.Join(src, "include"), 0o755); err != nil { @@ -71,6 +102,22 @@ func TestExtractArtifactUsesArtifactType(t *testing.T) { }) } +type staticArtifactStore struct { + art artifact.Artifact +} + +func (s staticArtifactStore) Get(context.Context, artifact.Key) (artifact.Artifact, error) { + return s.art, nil +} + +func (s staticArtifactStore) Put(_ context.Context, _ artifact.Key, art artifact.Artifact) (artifact.Artifact, error) { + return art, nil +} + +func (s staticArtifactStore) Delete(context.Context, artifact.Key) error { + return nil +} + func assertFileContent(t *testing.T, name, want string) { t.Helper() From e7f3a274dba29cd9877facf2960986153fe3f121 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 20:02:53 +0800 Subject: [PATCH 15/16] chore(build): remove redundant kodo get guards --- internal/build/cache/kodo.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index 5afbeb90..fb480061 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -74,9 +74,6 @@ func NewKodo(cfg KodoConfig) Cache { } func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { - if c.artifacts == nil { - return Entry{}, false, nil - } art, err := c.artifacts.Get(ctx, artifact.Key{ Module: key.Module.Path, Version: key.Module.Version, @@ -88,9 +85,6 @@ func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { if err != nil { return Entry{}, false, err } - if art.Source.Type != "kodo" { - return Entry{}, false, fmt.Errorf("artifact source type = %q, want kodo", art.Source.Type) - } if c.workspaceDir != "" { objectName := c.objectName(key) if err := c.restore(ctx, key, objectName, art.Type, art.Checksum); err != nil { From ead30d0aac385003113a5ca5aa1b46af603dbb93 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 20:05:16 +0800 Subject: [PATCH 16/16] chore(build): require kodo artifact store in put --- internal/build/cache/kodo.go | 40 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go index fb480061..4a455d96 100644 --- a/internal/build/cache/kodo.go +++ b/internal/build/cache/kodo.go @@ -117,13 +117,9 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) return Entry{}, err } checksum := hex.EncodeToString(hash.Sum(nil)) - var sourceURL string - if c.artifacts != nil { - var err error - sourceURL, err = kodoSourceURL(c.publicDomain, objectName) - if err != nil { - return Entry{}, err - } + sourceURL, err := kodoSourceURL(c.publicDomain, objectName) + if err != nil { + return Entry{}, err } err = c.uploader.UploadFile(ctx, file.Name(), &uploader.ObjectOptions{ @@ -143,22 +139,20 @@ func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) } return Entry{}, err } - if c.artifacts != nil { - if _, err := c.artifacts.Put(ctx, artifact.Key{ - Module: key.Module.Path, - Version: key.Module.Version, - MatrixStr: key.Matrix, - }, artifact.Artifact{ - Source: artifact.Source{ - Type: "kodo", - URL: sourceURL, - }, - Type: "tar.gz", - Metadata: entry.Metadata, - Checksum: checksum, - }); err != nil { - return Entry{}, err - } + if _, err := c.artifacts.Put(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }, artifact.Artifact{ + Source: artifact.Source{ + Type: "kodo", + URL: sourceURL, + }, + Type: "tar.gz", + Metadata: entry.Metadata, + Checksum: checksum, + }); err != nil { + return Entry{}, err } return entry, nil }