diff --git a/go.mod b/go.mod index 87ed41a4..65f799d9 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/spf13/cobra v1.10.2 golang.org/x/mod v0.32.0 golang.org/x/sys v0.40.0 + gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 ) @@ -31,6 +32,10 @@ require ( github.com/goplus/gogen v1.20.6 // indirect github.com/goplus/reflectx v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/klauspost/compress v1.18.1 // indirect @@ -47,8 +52,9 @@ require ( github.com/visualfc/gid v0.3.0 // indirect github.com/visualfc/goembed v0.3.2 // indirect github.com/visualfc/xtype v0.2.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.40.0 // indirect modernc.org/fileutil v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index b1e1febd..3d0b574e 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,14 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -83,6 +91,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -102,6 +111,8 @@ github.com/visualfc/goembed v0.3.2/go.mod h1:jCVCz/yTJGyslo6Hta+pYxWWBuq9ADCcIVZ github.com/visualfc/xtype v0.2.0 h1:0ESNXyWHtK01kaOzOyqHsR1ZjEPdNu/IWPZkf0VOHl8= github.com/visualfc/xtype v0.2.0/go.mod h1:183MDtzLqyDkCm5zCH42vJGq/aQE5W25k3Z6UOZxLF0= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= @@ -109,16 +120,18 @@ golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= diff --git a/internal/build/cache/kodo.go b/internal/build/cache/kodo.go new file mode 100644 index 00000000..4a455d96 --- /dev/null +++ b/internal/build/cache/kodo.go @@ -0,0 +1,455 @@ +package cache + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/mod/module" + qiniuclient "github.com/qiniu/go-sdk/v7/client" + "github.com/qiniu/go-sdk/v7/storagev2/credentials" + qiniudownloader "github.com/qiniu/go-sdk/v7/storagev2/downloader" + httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/go-sdk/v7/storagev2/objects" + "github.com/qiniu/go-sdk/v7/storagev2/uploader" + "github.com/qiniu/go-sdk/v7/storagev2/uptoken" +) + +type KodoConfig struct { + AccessKey string + SecretKey string + Bucket string + PublicDomain string + Prefix string + WorkspaceDir string + Artifacts artifact.Store +} + +type kodoCache struct { + bucket string + publicDomain string + prefix string + workspaceDir string + artifacts artifact.Store + credentials *credentials.Credentials + objects *objects.ObjectsManager + uploader *uploader.UploadManager + downloader *qiniudownloader.DownloadManager +} + +func NewKodo(cfg KodoConfig) Cache { + cred := credentials.NewCredentials(cfg.AccessKey, cfg.SecretKey) + options := httpclient.Options{Credentials: cred} + return &kodoCache{ + bucket: cfg.Bucket, + publicDomain: normalizePublicDomain(cfg.PublicDomain), + prefix: strings.Trim(cfg.Prefix, "/"), + workspaceDir: cfg.WorkspaceDir, + artifacts: cfg.Artifacts, + credentials: cred, + objects: objects.NewObjectsManager(&objects.ObjectsManagerOptions{ + Options: options, + }), + uploader: uploader.NewUploadManager(&uploader.UploadManagerOptions{ + Options: options, + }), + downloader: qiniudownloader.NewDownloadManager(&qiniudownloader.DownloadManagerOptions{ + Options: options, + }), + } +} + +func (c *kodoCache) Get(ctx context.Context, key Key) (Entry, bool, error) { + art, err := c.artifacts.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) + if errors.Is(err, artifact.ErrNotFound) { + return Entry{}, false, nil + } + if err != nil { + return Entry{}, false, err + } + if c.workspaceDir != "" { + objectName := c.objectName(key) + if err := c.restore(ctx, key, objectName, art.Type, art.Checksum); err != nil { + return Entry{}, false, err + } + } + return Entry{Metadata: art.Metadata}, true, nil +} + +func (c *kodoCache) Put(ctx context.Context, key Key, output fs.FS, entry Entry) (Entry, error) { + objectName := c.objectName(key) + putPolicy, err := uptoken.NewPutPolicyWithKey(c.bucket, objectName, time.Now().Add(time.Hour)) + if err != nil { + return Entry{}, err + } + putPolicy.SetInsertOnly(1) + + file, err := os.CreateTemp("", "llar-kodo-*.tar.gz") + if err != nil { + return Entry{}, err + } + defer os.Remove(file.Name()) + + hash := sha256.New() + if err := writeTarGzip(io.MultiWriter(file, hash), output); err != nil { + _ = file.Close() + return Entry{}, err + } + if err := file.Close(); err != nil { + return Entry{}, err + } + checksum := hex.EncodeToString(hash.Sum(nil)) + sourceURL, err := kodoSourceURL(c.publicDomain, objectName) + if err != nil { + return Entry{}, err + } + + err = c.uploader.UploadFile(ctx, file.Name(), &uploader.ObjectOptions{ + BucketName: c.bucket, + ObjectName: &objectName, + FileName: path.Base(objectName), + ContentType: "application/gzip", + UpToken: uptoken.NewSigner(putPolicy, c.credentials), + }, nil) + if err != nil { + if isKodoObjectExists(err) { + if got, ok, getErr := c.Get(ctx, key); getErr != nil { + return Entry{}, getErr + } else if ok { + return got, nil + } + } + return Entry{}, err + } + if _, err := c.artifacts.Put(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }, artifact.Artifact{ + Source: artifact.Source{ + Type: "kodo", + URL: sourceURL, + }, + Type: "tar.gz", + Metadata: entry.Metadata, + Checksum: checksum, + }); err != nil { + return Entry{}, err + } + return entry, nil +} + +func (c *kodoCache) objectName(key Key) string { + parts := make([]string, 0, 4) + if c.prefix != "" { + parts = append(parts, c.prefix) + } + parts = append(parts, strings.Trim(key.Module.Path, "/"), strings.Trim(key.Module.Version, "/"), key.Matrix+".tar.gz") + return strings.Join(parts, "/") +} + +func (c *kodoCache) installDir(key Key) (string, error) { + escaped, err := module.EscapePath(key.Module.Path) + if err != nil { + return "", err + } + return filepath.Join(c.workspaceDir, fmt.Sprintf("%s@%s-%s", escaped, key.Module.Version, key.Matrix)), nil +} + +func (c *kodoCache) restore(ctx context.Context, key Key, objectName, artifactType, checksum string) error { + installDir, err := c.installDir(key) + if err != nil { + return err + } + file, err := os.CreateTemp("", "llar-kodo-restore-*.tar.gz") + if err != nil { + return err + } + fileName := file.Name() + if err := file.Close(); err != nil { + _ = os.Remove(fileName) + return err + } + defer os.Remove(fileName) + + _, err = c.downloader.DownloadToFile(ctx, objectName, fileName, &qiniudownloader.ObjectOptions{ + GenerateOptions: qiniudownloader.GenerateOptions{ + BucketName: c.bucket, + }, + }) + if err != nil { + return err + } + if checksum != "" { + got, err := fileSHA256(fileName) + if err != nil { + return err + } + if got != checksum { + return fmt.Errorf("kodo artifact checksum = %s, want %s", got, checksum) + } + } + if err := os.RemoveAll(installDir); err != nil { + return err + } + if err := os.MkdirAll(installDir, 0o755); err != nil { + return err + } + return extractArtifact(fileName, artifactType, installDir) +} + +func normalizePublicDomain(domain string) string { + domain = strings.TrimRight(strings.TrimSpace(domain), "/") + if domain == "" || strings.Contains(domain, "://") { + return domain + } + return "http://" + domain +} + +func kodoSourceURL(domain, objectName string) (string, error) { + u, err := url.Parse(normalizePublicDomain(domain)) + if err != nil { + return "", err + } + if u.Scheme != "http" && u.Scheme != "https" || u.Host == "" { + return "", fmt.Errorf("kodo public domain must be http(s), got %q", domain) + } + u.Path = "/" + objectName + u.RawPath = "" + u.RawQuery = "" + u.Fragment = "" + return u.String(), nil +} + +func isKodoObjectNotFound(err error) bool { + var info *qiniuclient.ErrorInfo + return errors.As(err, &info) && info.Code == 612 +} + +func isKodoObjectExists(err error) bool { + var info *qiniuclient.ErrorInfo + return errors.As(err, &info) && info.Code == 614 +} + +func fileSHA256(name string) (string, error) { + file, err := os.Open(name) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func writeTarGzip(w io.Writer, src fs.FS) error { + gzw := gzip.NewWriter(w) + tw := tar.NewWriter(gzw) + + if err := fs.WalkDir(src, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if name == "." { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + if !info.IsDir() && !info.Mode().IsRegular() { + return fmt.Errorf("archive %s: unsupported file mode %s", name, info.Mode()) + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = name + if info.IsDir() && !strings.HasSuffix(header.Name, "/") { + header.Name += "/" + } + header.ModTime = time.Unix(0, 0) + header.AccessTime = time.Unix(0, 0) + header.ChangeTime = time.Unix(0, 0) + header.Uid = 0 + header.Gid = 0 + header.Uname = "" + header.Gname = "" + + if err := tw.WriteHeader(header); err != nil { + return err + } + if info.IsDir() { + return nil + } + + file, err := src.Open(name) + if err != nil { + return err + } + _, copyErr := io.Copy(tw, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }); err != nil { + _ = tw.Close() + _ = gzw.Close() + return err + } + + if err := tw.Close(); err != nil { + _ = gzw.Close() + return err + } + return gzw.Close() +} + +func extractArtifact(fileName, artifactType, dst string) error { + switch artifactType { + case "tar.gz": + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + return extractTarGzip(file, dst) + case "zip": + return extractZip(fileName, dst) + default: + return fmt.Errorf("unsupported artifact type %q", artifactType) + } +} + +func extractTarGzip(r io.Reader, dst string) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + name, err := cleanArchiveName(header.Name) + if err != nil { + return err + } + target := filepath.Join(dst, name) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, header.FileInfo().Mode().Perm()); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, header.FileInfo().Mode().Perm()) + if err != nil { + return err + } + _, copyErr := io.Copy(file, tr) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + default: + return fmt.Errorf("extract %s: unsupported tar type %d", header.Name, header.Typeflag) + } + } +} + +func extractZip(fileName, dst string) error { + zr, err := zip.OpenReader(fileName) + if err != nil { + return err + } + defer zr.Close() + + for _, entry := range zr.File { + name, err := cleanArchiveName(entry.Name) + if err != nil { + return err + } + target := filepath.Join(dst, name) + info := entry.FileInfo() + mode := info.Mode() + + if info.IsDir() { + if err := os.MkdirAll(target, mode.Perm()); err != nil { + return err + } + continue + } + if !mode.IsRegular() { + return fmt.Errorf("extract %s: unsupported zip mode %s", entry.Name, mode) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + src, err := entry.Open() + if err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode.Perm()) + if err != nil { + _ = src.Close() + return err + } + _, copyErr := io.Copy(file, src) + closeErr := file.Close() + srcCloseErr := src.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if srcCloseErr != nil { + return srcCloseErr + } + } + return nil +} + +func cleanArchiveName(name string) (string, error) { + name = filepath.Clean(filepath.FromSlash(name)) + if name == "." || filepath.IsAbs(name) || name == ".." || strings.HasPrefix(name, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("unsafe archive path %q", name) + } + return name, nil +} diff --git a/internal/build/cache/kodo_e2e_test.go b/internal/build/cache/kodo_e2e_test.go new file mode 100644 index 00000000..26be6651 --- /dev/null +++ b/internal/build/cache/kodo_e2e_test.go @@ -0,0 +1,317 @@ +package cache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/mod/module" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func TestKodoObjectName(t *testing.T) { + c := NewKodo(KodoConfig{Prefix: "/cache/"}).(*kodoCache) + key := Key{ + Module: module.Version{Path: "madler/zlib", Version: "v1.3.2"}, + Matrix: "amd64-linux", + } + if got, want := c.objectName(key), "cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { + t.Fatalf("object name = %q, want %q", got, want) + } + got, err := kodoSourceURL("llar.liuxi.ng", c.objectName(key)) + if err != nil { + t.Fatal(err) + } + if want := "http://llar.liuxi.ng/cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { + t.Fatalf("source url = %q, want %q", got, want) + } +} + +func TestKodoE2E_PutGet(t *testing.T) { + accessKey := os.Getenv("QINIU_ACCESS_KEY") + secretKey := os.Getenv("QINIU_SECRET_KEY") + bucket := os.Getenv("QINIU_BUCKET") + publicDomain := envOrDefault("QINIU_PUBLIC_DOMAIN", "llar.liuxi.ng") + if accessKey == "" || secretKey == "" || bucket == "" { + t.Skip("QINIU_ACCESS_KEY, QINIU_SECRET_KEY, and QINIU_BUCKET are required") + } + + prefix := strings.Trim(os.Getenv("QINIU_PREFIX"), "/") + if prefix != "" { + prefix += "/" + } + prefix += fmt.Sprintf("llar-kodo-e2e/%d", time.Now().UnixNano()) + store := newKodoE2EArtifactStore(t) + + const zlibVersion = "v1.3.1" + matrix := hostMatrix() + workspaceDir, installDir, metadata := buildZlibWithLLAR(t, zlibVersion, matrix) + c := NewKodo(KodoConfig{ + AccessKey: accessKey, + SecretKey: secretKey, + Bucket: bucket, + PublicDomain: publicDomain, + Prefix: prefix, + WorkspaceDir: workspaceDir, + Artifacts: store, + }).(*kodoCache) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + key := Key{ + Module: module.Version{Path: "madler/zlib", Version: zlibVersion}, + Matrix: matrix, + } + objectName := c.objectName(key) + if want := prefix + "/madler/zlib/" + zlibVersion + "/" + matrix + ".tar.gz"; objectName != want { + t.Fatalf("object name = %q, want %q", objectName, want) + } + defer func() { + if err := c.objects.Bucket(c.bucket).Object(objectName).Delete().Call(ctx); err != nil && !isKodoObjectNotFound(err) { + t.Errorf("delete %s: %v", objectName, err) + } + }() + + if _, ok, err := c.Get(ctx, key); err != nil { + t.Fatalf("Get before Put failed: %v", err) + } else if ok { + t.Fatalf("Get before Put hit %s", objectName) + } + + want := Entry{ + Metadata: metadata, + } + got, err := c.Put(ctx, key, os.DirFS(installDir), want) + if err != nil { + t.Fatalf("Put failed: %v", err) + } + if got.Metadata != want.Metadata { + t.Fatalf("Put entry = %+v, want %+v", got, want) + } + + stored, err := store.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) + if err != nil { + t.Fatalf("artifact Get after Put failed: %v", err) + } + if stored.Source.Type != "kodo" { + t.Fatalf("artifact source type = %q, want kodo", stored.Source.Type) + } + wantURL, err := kodoSourceURL(publicDomain, objectName) + if err != nil { + t.Fatal(err) + } + if stored.Source.URL != wantURL { + t.Fatalf("artifact source url = %q, want %q", stored.Source.URL, wantURL) + } + if stored.Type != "tar.gz" || stored.Metadata != metadata || len(stored.Checksum) != 64 { + t.Fatalf("artifact = %+v, want tar.gz metadata %q and sha256 checksum", stored, metadata) + } + if err := assertPublicURLChecksum(ctx, stored.Source.URL, stored.Checksum); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(installDir, "lib", "libz.a"), []byte("conflicting zlib archive\n"), 0o644); err != nil { + t.Fatalf("rewrite zlib archive before conflicting Put: %v", err) + } + conflict := Entry{ + Metadata: "-lz-conflict", + } + got, err = c.Put(ctx, key, os.DirFS(installDir), conflict) + if err != nil { + t.Fatalf("conflicting Put failed: %v", err) + } + if got.Metadata != want.Metadata { + t.Fatalf("conflicting Put entry = %+v, want existing %+v", got, want) + } + afterConflict, err := store.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) + if err != nil { + t.Fatalf("artifact Get after conflicting Put failed: %v", err) + } + if afterConflict != stored { + t.Fatalf("artifact after conflicting Put = %+v, want existing %+v", afterConflict, stored) + } + if err := assertPublicURLChecksum(ctx, stored.Source.URL, stored.Checksum); err != nil { + t.Fatal(err) + } + + if err := os.RemoveAll(installDir); err != nil { + t.Fatalf("remove install dir before Get: %v", err) + } + got, ok, err := c.Get(ctx, key) + if err != nil { + t.Fatalf("Get after Put failed: %v", err) + } + if !ok { + t.Fatal("Get after Put missed") + } + if got.Metadata != want.Metadata { + t.Fatalf("Get entry = %+v, want %+v", got, want) + } + if _, err := os.Stat(filepath.Join(installDir, "include", "zlib.h")); err != nil { + t.Fatalf("restored zlib include not found in %s: %v", installDir, err) + } + if _, err := os.Stat(filepath.Join(installDir, "lib")); err != nil { + t.Fatalf("restored zlib lib dir not found in %s: %v", installDir, err) + } +} + +func newKodoE2EArtifactStore(t *testing.T) artifact.Store { + t.Helper() + + var dial gorm.Dialector + if dsn := os.Getenv("POSTGRES_DSN"); dsn != "" { + dial = postgres.Open(dsn) + } else { + dial = sqlite.Open(filepath.Join(t.TempDir(), "artifacts.db")) + } + db, err := gorm.Open(dial, &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + t.Fatalf("gorm.Open: %v", err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("db.DB: %v", err) + } + t.Cleanup(func() { + if err := sqlDB.Close(); err != nil { + t.Fatalf("db.Close: %v", err) + } + }) + if err := db.Exec("DROP TABLE IF EXISTS artifacts").Error; err != nil { + t.Fatalf("drop artifacts: %v", err) + } + + store, err := artifact.NewGormStore(db) + if err != nil { + t.Fatalf("NewGormStore: %v", err) + } + return store +} + +func buildZlibWithLLAR(t *testing.T, version, matrix string) (string, string, string) { + t.Helper() + + llar, err := exec.LookPath("llar") + if err != nil { + t.Skip("llar not found in PATH") + } + for _, tool := range []string{"cmake", "git"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s not found in PATH", tool) + } + } + + dir := t.TempDir() + home := filepath.Join(dir, "home") + cacheHome := filepath.Join(dir, "cache") + if err := os.MkdirAll(home, 0o755); err != nil { + t.Fatalf("create home: %v", err) + } + if err := os.MkdirAll(cacheHome, 0o755); err != nil { + t.Fatalf("create cache home: %v", err) + } + t.Setenv("HOME", home) + t.Setenv("XDG_CACHE_HOME", cacheHome) + + userCacheDir, err := os.UserCacheDir() + if err != nil { + t.Fatalf("UserCacheDir: %v", err) + } + + formulaRoot := kodoE2EFormulaRoot(t) + cmd := exec.Command(llar, "make", "./madler/zlib@"+version) + cmd.Dir = formulaRoot + cmd.Env = os.Environ() + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("llar make ./madler/zlib@%s failed: %v\n%s", version, err, out) + } + + metadata := strings.TrimSpace(string(out)) + if metadata != "-lz" { + t.Fatalf("llar make metadata = %q, want -lz", metadata) + } + + workspaceDir := filepath.Join(userCacheDir, ".llar", "workspaces") + installDir := filepath.Join(workspaceDir, fmt.Sprintf("madler/zlib@%s-%s", version, matrix)) + if _, err := os.Stat(filepath.Join(installDir, "include", "zlib.h")); err != nil { + t.Fatalf("zlib include not found in %s: %v", installDir, err) + } + if _, err := os.Stat(filepath.Join(installDir, "lib")); err != nil { + t.Fatalf("zlib lib dir not found in %s: %v", installDir, err) + } + return workspaceDir, installDir, metadata +} + +func kodoE2EFormulaRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + root := filepath.Clean(filepath.Join(filepath.Dir(file), "../../..")) + dir := filepath.Join(root, "testdata", "kodo-e2e", "formulas") + if _, err := os.Stat(filepath.Join(dir, "madler", "zlib", "versions.json")); err != nil { + t.Fatalf("kodo e2e formula root: %v", err) + } + return dir +} + +func hostMatrix() string { + return runtime.GOARCH + "-" + runtime.GOOS +} + +func envOrDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func assertPublicURLChecksum(ctx context.Context, rawURL, checksum string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET %s: %s", rawURL, resp.Status) + } + hash := sha256.New() + if _, err := io.Copy(hash, resp.Body); err != nil { + return err + } + got := hex.EncodeToString(hash.Sum(nil)) + if got != checksum { + return fmt.Errorf("GET %s checksum = %s, want %s", rawURL, got, checksum) + } + return nil +} diff --git a/internal/build/cache/kodo_restore_test.go b/internal/build/cache/kodo_restore_test.go new file mode 100644 index 00000000..ae742285 --- /dev/null +++ b/internal/build/cache/kodo_restore_test.go @@ -0,0 +1,131 @@ +package cache + +import ( + "archive/zip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/mod/module" +) + +func TestKodoGetDoesNotParseSourceURL(t *testing.T) { + c := NewKodo(KodoConfig{ + Artifacts: staticArtifactStore{ + art: artifact.Artifact{ + Source: artifact.Source{Type: "kodo", URL: "not a kodo object name"}, + Type: "zip", + Metadata: "-lz", + }, + }, + }).(*kodoCache) + key := Key{ + Module: module.Version{Path: "madler/zlib", Version: "v1.3.2"}, + Matrix: "amd64-linux", + } + + got, ok, err := c.Get(context.Background(), key) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("Get missed") + } + if got.Metadata != "-lz" { + t.Fatalf("metadata = %q, want -lz", got.Metadata) + } +} + +func TestExtractArtifactUsesArtifactType(t *testing.T) { + src := t.TempDir() + if err := os.MkdirAll(filepath.Join(src, "include"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "include", "zlib.h"), []byte("zlib header\n"), 0o644); err != nil { + t.Fatal(err) + } + + t.Run("tar.gz", func(t *testing.T) { + artifactFile := filepath.Join(t.TempDir(), "artifact.tar.gz") + file, err := os.Create(artifactFile) + if err != nil { + t.Fatal(err) + } + if err := writeTarGzip(file, os.DirFS(src)); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + dst := t.TempDir() + if err := extractArtifact(artifactFile, "tar.gz", dst); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(dst, "include", "zlib.h"), "zlib header\n") + }) + + t.Run("zip", func(t *testing.T) { + artifactFile := filepath.Join(t.TempDir(), "artifact.zip") + file, err := os.Create(artifactFile) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(file) + w, err := zw.Create("include/zlib.h") + if err != nil { + _ = zw.Close() + _ = file.Close() + t.Fatal(err) + } + if _, err := w.Write([]byte("zlib header\n")); err != nil { + _ = zw.Close() + _ = file.Close() + t.Fatal(err) + } + if err := zw.Close(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + dst := t.TempDir() + if err := extractArtifact(artifactFile, "zip", dst); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(dst, "include", "zlib.h"), "zlib header\n") + }) +} + +type staticArtifactStore struct { + art artifact.Artifact +} + +func (s staticArtifactStore) Get(context.Context, artifact.Key) (artifact.Artifact, error) { + return s.art, nil +} + +func (s staticArtifactStore) Put(_ context.Context, _ artifact.Key, art artifact.Artifact) (artifact.Artifact, error) { + return art, nil +} + +func (s staticArtifactStore) Delete(context.Context, artifact.Key) error { + return nil +} + +func assertFileContent(t *testing.T, name, want string) { + t.Helper() + + got, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + if string(got) != want { + t.Fatalf("%s = %q, want %q", name, got, want) + } +} diff --git a/testdata/kodo-e2e/formulas/DaveGamble/cJSON/v1.7.18/CJSON_llar.gox b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/v1.7.18/CJSON_llar.gox new file mode 100644 index 00000000..7878c335 --- /dev/null +++ b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/v1.7.18/CJSON_llar.gox @@ -0,0 +1,7 @@ +id "DaveGamble/cJSON" + +fromVer "v1.7.18" + +onBuild (ctx, proj, out) => { + out.setMetadata "-lcjson" +} diff --git a/testdata/kodo-e2e/formulas/DaveGamble/cJSON/versions.json b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/versions.json new file mode 100644 index 00000000..7d569631 --- /dev/null +++ b/testdata/kodo-e2e/formulas/DaveGamble/cJSON/versions.json @@ -0,0 +1,8 @@ +{ + "path": "DaveGamble/cJSON", + "deps": { + "v1.7.18": [ + {"path": "madler/zlib", "version": "v1.3.1"} + ] + } +} diff --git a/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox new file mode 100644 index 00000000..c7566c91 --- /dev/null +++ b/testdata/kodo-e2e/formulas/madler/zlib/v1.3.1/Zlib_llar.gox @@ -0,0 +1,31 @@ +import "os" + +id "madler/zlib" + +fromVer "v1.3.1" + +onBuild (ctx, proj, out) => { + installDir, err := ctx.outputDir() + if err != nil { + out.addErr err + return + } + + exec "mkdir", "-p", installDir + "/include", installDir + "/lib" + if lastErr != nil { + out.addErr lastErr + return + } + err = os.writeFile(installDir+"/include/zlib.h", []byte("/* zlib e2e fixture */\n"), 0o644) + if err != nil { + out.addErr err + return + } + err = os.writeFile(installDir+"/lib/libz.a", []byte("zlib fixture archive\n"), 0o644) + if err != nil { + out.addErr err + return + } + + out.setMetadata "-lz" +} diff --git a/testdata/kodo-e2e/formulas/madler/zlib/versions.json b/testdata/kodo-e2e/formulas/madler/zlib/versions.json new file mode 100644 index 00000000..ca477e53 --- /dev/null +++ b/testdata/kodo-e2e/formulas/madler/zlib/versions.json @@ -0,0 +1,4 @@ +{ + "path": "madler/zlib", + "deps": {} +} diff --git a/testdata/kodo-e2e/formulas/pnggroup/libpng/v1.6.47/Libpng_llar.gox b/testdata/kodo-e2e/formulas/pnggroup/libpng/v1.6.47/Libpng_llar.gox new file mode 100644 index 00000000..0c47939d --- /dev/null +++ b/testdata/kodo-e2e/formulas/pnggroup/libpng/v1.6.47/Libpng_llar.gox @@ -0,0 +1,7 @@ +id "pnggroup/libpng" + +fromVer "v1.6.47" + +onBuild (ctx, proj, out) => { + out.setMetadata "-lpng" +} diff --git a/testdata/kodo-e2e/formulas/pnggroup/libpng/versions.json b/testdata/kodo-e2e/formulas/pnggroup/libpng/versions.json new file mode 100644 index 00000000..2dd8dddc --- /dev/null +++ b/testdata/kodo-e2e/formulas/pnggroup/libpng/versions.json @@ -0,0 +1,8 @@ +{ + "path": "pnggroup/libpng", + "deps": { + "v1.6.47": [ + {"path": "madler/zlib", "version": "v1.3.1"} + ] + } +} diff --git a/testdata/kodo-e2e/main.go b/testdata/kodo-e2e/main.go new file mode 100644 index 00000000..2b7065de --- /dev/null +++ b/testdata/kodo-e2e/main.go @@ -0,0 +1,921 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "github.com/goplus/llar/internal/artifact" + "github.com/goplus/llar/internal/build" + buildcache "github.com/goplus/llar/internal/build/cache" + "github.com/goplus/llar/internal/modules" + "github.com/goplus/llar/mod/module" + qiniuclient "github.com/qiniu/go-sdk/v7/client" + "github.com/qiniu/go-sdk/v7/storagev2/credentials" + qiniudownloader "github.com/qiniu/go-sdk/v7/storagev2/downloader" + httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/go-sdk/v7/storagev2/objects" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +const ( + defaultPostgresDSN = "host=localhost port=5432 user=llar password=llar dbname=llar_e2e sslmode=disable" + defaultTarget = "madler/zlib@v1.3.1" + defaultSharedTargets = "DaveGamble/cJSON@v1.7.18,pnggroup/libpng@v1.6.47" + defaultFormulaRoot = "testdata/kodo-e2e/formulas" + defaultPublicDomain = "llar.liuxi.ng" +) + +func main() { + var cfg config + flag.StringVar(&cfg.postgresDSN, "postgres-dsn", envOrDefault("POSTGRES_DSN", defaultPostgresDSN), "Postgres DSN; empty uses a temporary SQLite database") + flag.StringVar(&cfg.accessKey, "qiniu-access-key", os.Getenv("QINIU_ACCESS_KEY"), "Qiniu access key") + flag.StringVar(&cfg.secretKey, "qiniu-secret-key", os.Getenv("QINIU_SECRET_KEY"), "Qiniu secret key") + flag.StringVar(&cfg.bucket, "qiniu-bucket", os.Getenv("QINIU_BUCKET"), "Qiniu Kodo bucket") + flag.StringVar(&cfg.publicDomain, "qiniu-public-domain", envOrDefault("QINIU_PUBLIC_DOMAIN", defaultPublicDomain), "Qiniu Kodo public download domain") + flag.StringVar(&cfg.prefix, "qiniu-prefix", os.Getenv("QINIU_PREFIX"), "Qiniu Kodo object prefix") + flag.StringVar(&cfg.formulaRoot, "formula-root", defaultFormulaRoot, "local formula root") + flag.StringVar(&cfg.target, "target", defaultTarget, "target module@version") + flag.StringVar(&cfg.sharedTargets, "shared-targets", defaultSharedTargets, "comma-separated module@version targets sharing a dependency") + flag.StringVar(&cfg.matrix, "matrix", hostMatrix(), "matrix string") + flag.DurationVar(&cfg.timeout, "timeout", 15*time.Minute, "E2E timeout") + flag.Parse() + + if err := cfg.validate(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + if err := run(cfg); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +type config struct { + postgresDSN string + accessKey string + secretKey string + bucket string + publicDomain string + prefix string + formulaRoot string + target string + sharedTargets string + matrix string + timeout time.Duration +} + +func (c *config) validate() error { + var err error + c.accessKey = strings.TrimSpace(c.accessKey) + c.secretKey = strings.TrimSpace(c.secretKey) + c.bucket = strings.TrimSpace(c.bucket) + c.publicDomain = normalizePublicDomain(c.publicDomain) + c.prefix = strings.Trim(strings.TrimSpace(c.prefix), "/") + c.matrix = strings.TrimSpace(c.matrix) + c.formulaRoot, err = filepath.Abs(c.formulaRoot) + if err != nil { + return fmt.Errorf("formula root: %w", err) + } + if c.accessKey == "" { + return fmt.Errorf("missing required QINIU_ACCESS_KEY or -qiniu-access-key") + } + if c.secretKey == "" { + return fmt.Errorf("missing required QINIU_SECRET_KEY or -qiniu-secret-key") + } + if c.bucket == "" { + return fmt.Errorf("missing required QINIU_BUCKET or -qiniu-bucket") + } + if _, err := parseHTTPURL(c.publicDomain); err != nil { + return fmt.Errorf("-qiniu-public-domain: %w", err) + } + if c.matrix == "" { + return fmt.Errorf("missing required -matrix") + } + if _, err := parseTarget(c.target); err != nil { + return fmt.Errorf("-target: %w", err) + } + targets, err := parseTargets(c.sharedTargets) + if err != nil { + return fmt.Errorf("-shared-targets: %w", err) + } + if len(targets) < 2 { + return fmt.Errorf("-shared-targets must contain at least two targets") + } + if c.timeout <= 0 { + return fmt.Errorf("-timeout must be positive") + } + if _, err := os.Stat(filepath.Join(c.formulaRoot, "madler", "zlib", "versions.json")); err != nil { + return fmt.Errorf("formula root %s: %w", c.formulaRoot, err) + } + return nil +} + +func run(cfg config) error { + ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + defer cancel() + + target, err := parseTarget(cfg.target) + if err != nil { + return err + } + sharedTargets, err := parseTargets(cfg.sharedTargets) + if err != nil { + return err + } + + runPrefix := cfg.prefix + if runPrefix != "" { + runPrefix += "/" + } + runPrefix += fmt.Sprintf("llar-kodo-e2e/%d", time.Now().UnixNano()) + + db, cleanupDB, err := openDatabase(cfg.postgresDSN) + if err != nil { + return err + } + defer cleanupDB() + if err := resetDatabase(ctx, db); err != nil { + return err + } + artifacts, err := artifact.NewGormStore(db) + if err != nil { + return fmt.Errorf("NewGormStore: %w", err) + } + if count := artifactCount(ctx, db); count != 0 { + return fmt.Errorf("artifact count after reset = %d, want 0", count) + } + + kodo := newKodoClient(cfg) + if err := kodo.deletePrefix(ctx, runPrefix); err != nil { + return fmt.Errorf("cleanup kodo prefix before run: %w", err) + } + defer func() { + if err := kodo.deletePrefix(context.Background(), runPrefix); err != nil { + log.Printf("cleanup kodo prefix after run: %v", err) + } + }() + + s := suite{ + cfg: configData{ + accessKey: cfg.accessKey, + secretKey: cfg.secretKey, + bucket: cfg.bucket, + publicDomain: cfg.publicDomain, + prefix: runPrefix, + formulaRoot: cfg.formulaRoot, + target: target, + sharedTargets: sharedTargets, + matrix: cfg.matrix, + }, + formulas: newLocalFormulaStore(cfg.formulaRoot), + artifacts: artifacts, + db: db, + kodo: kodo, + } + + for _, target := range append([]module.Version{target}, sharedTargets...) { + if err := validateLocalFormula(cfg.formulaRoot, target); err != nil { + return err + } + } + + for _, step := range []struct { + name string + run func(context.Context) error + }{ + {"cold build uploads and stores artifact", s.coldBuild}, + {"repeated build uses stored artifact", s.repeatedBuild}, + {"new builder instance uses persisted artifact cache", s.persistedCache}, + {"different matrix stores independent artifact", s.differentMatrix}, + {"concurrent duplicate build stores one artifact", s.concurrentDuplicate}, + {"concurrent targets sharing dependency both complete", s.concurrentSharedDependency}, + } { + start := time.Now() + log.Printf("RUN %s", step.name) + if err := step.run(ctx); err != nil { + return fmt.Errorf("%s: %w", step.name, err) + } + log.Printf("PASS %s (%s)", step.name, time.Since(start).Round(time.Millisecond)) + } + + wantArtifacts := int64(6) + if count := artifactCount(ctx, db); count != wantArtifacts { + return fmt.Errorf("artifact count after E2E cases = %d, want %d", count, wantArtifacts) + } + if count, err := kodo.objectCount(ctx, runPrefix); err != nil { + return err + } else if count != wantArtifacts { + return fmt.Errorf("kodo object count under %s = %d, want %d", runPrefix, count, wantArtifacts) + } + log.Printf("PASS Kodo build E2E (%d artifacts)", wantArtifacts) + return nil +} + +type configData struct { + accessKey string + secretKey string + bucket string + publicDomain string + prefix string + formulaRoot string + target module.Version + sharedTargets []module.Version + matrix string +} + +type suite struct { + cfg configData + formulas *localFormulaStore + artifacts artifact.Store + db *gorm.DB + kodo *kodoClient + + baseCache *countingCache + baseWorkspace string + baseResult build.Result + baseArtifact artifact.Artifact +} + +func (s *suite) coldBuild(ctx context.Context) error { + s.baseWorkspace = mustTempDir("llar-kodo-e2e-workspace-") + s.baseCache = s.newCache(s.baseWorkspace) + + got, err := s.build(ctx, s.cfg.target, s.cfg.matrix, s.baseWorkspace, s.baseCache) + if err != nil { + return err + } + if s.baseCache.totalPuts() != 1 { + return fmt.Errorf("cache Put calls = %d, want 1", s.baseCache.totalPuts()) + } + if got.Metadata != "-lz" { + return fmt.Errorf("metadata = %q, want -lz", got.Metadata) + } + if err := assertZlibOutput(got.OutputDir); err != nil { + return err + } + key := cacheKey(s.cfg.target, s.cfg.matrix) + art, err := s.assertStoredArtifact(ctx, key, "-lz") + if err != nil { + return err + } + s.baseResult = got + s.baseArtifact = art + return nil +} + +func (s *suite) repeatedBuild(ctx context.Context) error { + got, err := s.build(ctx, s.cfg.target, s.cfg.matrix, s.baseWorkspace, s.baseCache) + if err != nil { + return err + } + if got != s.baseResult { + return fmt.Errorf("result = %+v, want %+v", got, s.baseResult) + } + if s.baseCache.totalPuts() != 1 { + return fmt.Errorf("cache Put calls after cache hit = %d, want 1", s.baseCache.totalPuts()) + } + art, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, s.cfg.matrix), "-lz") + if err != nil { + return err + } + if art != s.baseArtifact { + return fmt.Errorf("stored artifact changed = %+v, want %+v", art, s.baseArtifact) + } + return nil +} + +func (s *suite) persistedCache(ctx context.Context) error { + workspace := mustTempDir("llar-kodo-e2e-persisted-") + c := s.newCache(workspace) + got, err := s.build(ctx, s.cfg.target, s.cfg.matrix, workspace, c) + if err != nil { + return err + } + if got.Metadata != s.baseResult.Metadata { + return fmt.Errorf("metadata = %q, want %q", got.Metadata, s.baseResult.Metadata) + } + if c.totalPuts() != 0 { + return fmt.Errorf("cache Put calls = %d, want 0", c.totalPuts()) + } + if err := assertZlibOutput(got.OutputDir); err != nil { + return err + } + return nil +} + +func (s *suite) differentMatrix(ctx context.Context) error { + matrix := s.cfg.matrix + "-variant" + workspace := mustTempDir("llar-kodo-e2e-matrix-") + c := s.newCache(workspace) + got, err := s.build(ctx, s.cfg.target, matrix, workspace, c) + if err != nil { + return err + } + if got.Metadata != "-lz" { + return fmt.Errorf("metadata = %q, want -lz", got.Metadata) + } + if c.totalPuts() != 1 { + return fmt.Errorf("cache Put calls = %d, want 1", c.totalPuts()) + } + if _, err := s.assertStoredArtifact(ctx, cacheKey(s.cfg.target, matrix), "-lz"); err != nil { + return err + } + return nil +} + +func (s *suite) concurrentDuplicate(ctx context.Context) error { + matrix := s.cfg.matrix + "-concurrent" + key := cacheKey(s.cfg.target, matrix) + workspace1 := mustTempDir("llar-kodo-e2e-concurrent-") + workspace2 := mustTempDir("llar-kodo-e2e-concurrent-") + c1 := s.newCache(workspace1) + c2 := s.newCache(workspace2) + + results := make(chan buildResult, 2) + start := make(chan struct{}) + go func() { + <-start + got, err := s.build(ctx, s.cfg.target, matrix, workspace1, c1) + results <- buildResult{result: got, err: err} + }() + go func() { + <-start + got, err := s.build(ctx, s.cfg.target, matrix, workspace2, c2) + results <- buildResult{result: got, err: err} + }() + close(start) + + first, err := waitBuildResult(ctx, results) + if err != nil { + return err + } + second, err := waitBuildResult(ctx, results) + if err != nil { + return err + } + if first.err != nil { + return fmt.Errorf("first build: %w", first.err) + } + if second.err != nil { + return fmt.Errorf("second build: %w", second.err) + } + if first.result.Metadata != second.result.Metadata { + return fmt.Errorf("concurrent metadata = %q, want %q", second.result.Metadata, first.result.Metadata) + } + if err := assertZlibOutput(first.result.OutputDir); err != nil { + return err + } + if err := assertZlibOutput(second.result.OutputDir); err != nil { + return err + } + if total := c1.totalPuts() + c2.totalPuts(); total != 2 { + return fmt.Errorf("cache Put calls = %d, want 2", total) + } + if _, err := s.assertStoredArtifact(ctx, key, "-lz"); err != nil { + return err + } + return nil +} + +func (s *suite) concurrentSharedDependency(ctx context.Context) error { + matrix := s.cfg.matrix + "-shareddep" + workspace := mustTempDir("llar-kodo-e2e-shared-") + c := s.newCache(workspace) + + results := make(chan namedBuildResult, len(s.cfg.sharedTargets)) + start := make(chan struct{}) + for _, target := range s.cfg.sharedTargets { + go func(target module.Version) { + <-start + got, err := s.build(ctx, target, matrix, workspace, c) + results <- namedBuildResult{target: target, result: got, err: err} + }(target) + } + close(start) + + gotByTarget := make(map[string]build.Result, len(s.cfg.sharedTargets)) + for range s.cfg.sharedTargets { + result, err := waitNamedBuildResult(ctx, results) + if err != nil { + return err + } + if result.err != nil { + return fmt.Errorf("build %s@%s: %w", result.target.Path, result.target.Version, result.err) + } + gotByTarget[targetKey(result.target)] = result.result + } + if c.totalPuts() != 4 { + return fmt.Errorf("cache Put calls = %d, want 4", c.totalPuts()) + } + + zlib := module.Version{Path: "madler/zlib", Version: "v1.3.1"} + if c.putCount(cacheKey(zlib, matrix)) != 2 { + return fmt.Errorf("shared dependency Put calls = %d, want 2", c.putCount(cacheKey(zlib, matrix))) + } + if _, err := s.assertStoredArtifact(ctx, cacheKey(zlib, matrix), "-lz"); err != nil { + return err + } + + for _, target := range s.cfg.sharedTargets { + got, ok := gotByTarget[targetKey(target)] + if !ok { + return fmt.Errorf("missing result for %s", targetKey(target)) + } + wantMetadata := map[string]string{ + "DaveGamble/cJSON": "-lcjson", + "pnggroup/libpng": "-lpng", + }[target.Path] + if got.Metadata != wantMetadata { + return fmt.Errorf("%s metadata = %q, want %q", targetKey(target), got.Metadata, wantMetadata) + } + if _, err := s.assertStoredArtifact(ctx, cacheKey(target, matrix), wantMetadata); err != nil { + return err + } + } + return nil +} + +func (s *suite) build(ctx context.Context, target module.Version, matrix, workspaceDir string, c buildcache.Cache) (build.Result, error) { + mods, err := modules.Load(ctx, target, modules.Options{FormulaStore: s.formulas}) + if err != nil { + return build.Result{}, fmt.Errorf("modules.Load %s: %w", targetKey(target), err) + } + builder, err := build.NewBuilder(build.Options{ + Store: s.formulas, + MatrixStr: matrix, + WorkspaceDir: workspaceDir, + Cache: c, + }) + if err != nil { + return build.Result{}, fmt.Errorf("NewBuilder: %w", err) + } + results, err := builder.Build(ctx, mods) + if err != nil { + return build.Result{}, fmt.Errorf("Build %s: %w", targetKey(target), err) + } + if len(results) == 0 { + return build.Result{}, fmt.Errorf("Build %s returned no results", targetKey(target)) + } + return results[len(results)-1], nil +} + +func (s *suite) newCache(workspaceDir string) *countingCache { + return &countingCache{ + inner: buildcache.NewKodo(buildcache.KodoConfig{ + AccessKey: s.cfg.accessKey, + SecretKey: s.cfg.secretKey, + Bucket: s.cfg.bucket, + PublicDomain: s.cfg.publicDomain, + Prefix: s.cfg.prefix, + WorkspaceDir: workspaceDir, + Artifacts: s.artifacts, + }), + } +} + +func (s *suite) assertStoredArtifact(ctx context.Context, key buildcache.Key, metadata string) (artifact.Artifact, error) { + got, err := s.artifacts.Get(ctx, artifact.Key{ + Module: key.Module.Path, + Version: key.Module.Version, + MatrixStr: key.Matrix, + }) + if err != nil { + return artifact.Artifact{}, fmt.Errorf("Get stored artifact %s: %w", keyString(key), err) + } + if got.Source.Type != "kodo" { + return artifact.Artifact{}, fmt.Errorf("source type = %q, want kodo", got.Source.Type) + } + if got.Type != "tar.gz" { + return artifact.Artifact{}, fmt.Errorf("artifact type = %q, want tar.gz", got.Type) + } + if got.Metadata != metadata { + return artifact.Artifact{}, fmt.Errorf("artifact metadata = %q, want %q", got.Metadata, metadata) + } + if len(got.Checksum) != 64 { + return artifact.Artifact{}, fmt.Errorf("artifact checksum = %q, want sha256 hex", got.Checksum) + } + + objectName, err := parseKodoSourceURL(got.Source.URL) + if err != nil { + return artifact.Artifact{}, err + } + wantObject := objectNameFor(s.cfg.prefix, key) + if objectName != wantObject { + return artifact.Artifact{}, fmt.Errorf("artifact object = %q, want %q", objectName, wantObject) + } + wantURL := publicURL(s.cfg.publicDomain, wantObject) + if got.Source.URL != wantURL { + return artifact.Artifact{}, fmt.Errorf("artifact source url = %q, want %q", got.Source.URL, wantURL) + } + + if err := s.kodo.assertChecksum(ctx, objectName, got.Checksum); err != nil { + return artifact.Artifact{}, err + } + if err := assertPublicURLChecksum(ctx, got.Source.URL, got.Checksum); err != nil { + return artifact.Artifact{}, err + } + return got, nil +} + +type localFormulaStore struct { + root string +} + +func newLocalFormulaStore(root string) *localFormulaStore { + return &localFormulaStore{root: root} +} + +func (s *localFormulaStore) ModuleFS(ctx context.Context, modPath string) (fs.FS, error) { + dir := filepath.Join(s.root, filepath.FromSlash(modPath)) + if _, err := os.Stat(filepath.Join(dir, "versions.json")); err != nil { + return nil, fmt.Errorf("local formula %s: %w", modPath, err) + } + return os.DirFS(dir), nil +} + +func (s *localFormulaStore) LockModule(modPath string) (func(), error) { + if modPath == "" { + return nil, fmt.Errorf("empty module path") + } + return func() {}, nil +} + +type countingCache struct { + inner buildcache.Cache + mu sync.Mutex + puts map[buildcache.Key]int +} + +func (c *countingCache) Get(ctx context.Context, key buildcache.Key) (buildcache.Entry, bool, error) { + return c.inner.Get(ctx, key) +} + +func (c *countingCache) Put(ctx context.Context, key buildcache.Key, output fs.FS, entry buildcache.Entry) (buildcache.Entry, error) { + got, err := c.inner.Put(ctx, key, output, entry) + if err != nil { + return buildcache.Entry{}, err + } + c.mu.Lock() + if c.puts == nil { + c.puts = make(map[buildcache.Key]int) + } + c.puts[key]++ + c.mu.Unlock() + return got, nil +} + +func (c *countingCache) putCount(key buildcache.Key) int { + c.mu.Lock() + defer c.mu.Unlock() + return c.puts[key] +} + +func (c *countingCache) totalPuts() int { + c.mu.Lock() + defer c.mu.Unlock() + var n int + for _, count := range c.puts { + n += count + } + return n +} + +type kodoClient struct { + bucket string + objects *objects.ObjectsManager + downloader *qiniudownloader.DownloadManager +} + +func newKodoClient(cfg config) *kodoClient { + cred := credentials.NewCredentials(cfg.accessKey, cfg.secretKey) + options := httpclient.Options{Credentials: cred} + return &kodoClient{ + bucket: cfg.bucket, + objects: objects.NewObjectsManager(&objects.ObjectsManagerOptions{ + Options: options, + }), + downloader: qiniudownloader.NewDownloadManager(&qiniudownloader.DownloadManagerOptions{ + Options: options, + }), + } +} + +func (c *kodoClient) assertChecksum(ctx context.Context, objectName, checksum string) error { + file, err := os.CreateTemp("", "llar-kodo-e2e-download-*.tar.gz") + if err != nil { + return err + } + name := file.Name() + if err := file.Close(); err != nil { + _ = os.Remove(name) + return err + } + defer os.Remove(name) + + if _, err := c.downloader.DownloadToFile(ctx, objectName, name, &qiniudownloader.ObjectOptions{ + GenerateOptions: qiniudownloader.GenerateOptions{ + BucketName: c.bucket, + }, + }); err != nil { + return fmt.Errorf("download kodo object %s: %w", objectName, err) + } + got, err := fileSHA256(name) + if err != nil { + return err + } + if got != checksum { + return fmt.Errorf("kodo object %s checksum = %s, want %s", objectName, got, checksum) + } + return nil +} + +func (c *kodoClient) objectCount(ctx context.Context, prefix string) (int64, error) { + lister := c.objects.Bucket(c.bucket).List(ctx, &objects.ListObjectsOptions{Prefix: prefix}) + defer lister.Close() + var count int64 + var details objects.ObjectDetails + for lister.Next(&details) { + count++ + } + if err := lister.Error(); err != nil { + return 0, fmt.Errorf("list kodo prefix %s: %w", prefix, err) + } + return count, nil +} + +func (c *kodoClient) deletePrefix(ctx context.Context, prefix string) error { + lister := c.objects.Bucket(c.bucket).List(ctx, &objects.ListObjectsOptions{Prefix: prefix}) + var names []string + var details objects.ObjectDetails + for lister.Next(&details) { + names = append(names, details.Name) + } + if err := lister.Close(); err != nil { + return fmt.Errorf("list kodo prefix %s: %w", prefix, err) + } + for _, name := range names { + if err := c.objects.Bucket(c.bucket).Object(name).Delete().Call(ctx); err != nil && !isKodoObjectNotFound(err) { + return fmt.Errorf("delete kodo object %s: %w", name, err) + } + } + return nil +} + +func openDatabase(dsn string) (*gorm.DB, func(), error) { + var dial gorm.Dialector + var cleanup func() + if strings.TrimSpace(dsn) == "" { + name := filepath.Join(mustTempDir("llar-kodo-e2e-db-"), "artifacts.db") + dial = sqlite.Open(name) + cleanup = func() {} + } else { + dial = postgres.Open(dsn) + cleanup = func() {} + } + db, err := gorm.Open(dial, &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + return nil, cleanup, fmt.Errorf("open database: %w", err) + } + sqlDB, err := db.DB() + if err != nil { + return nil, cleanup, fmt.Errorf("database handle: %w", err) + } + return db, func() { + _ = sqlDB.Close() + cleanup() + }, nil +} + +func resetDatabase(ctx context.Context, db *gorm.DB) error { + if err := db.WithContext(ctx).Exec("DROP TABLE IF EXISTS artifacts").Error; err != nil { + return fmt.Errorf("drop artifacts table: %w", err) + } + return nil +} + +func artifactCount(ctx context.Context, db *gorm.DB) int64 { + var count int64 + if err := db.WithContext(ctx).Table("artifacts").Count(&count).Error; err != nil { + return -1 + } + return count +} + +func validateLocalFormula(root string, target module.Version) error { + dir := filepath.Join(root, filepath.FromSlash(target.Path)) + if _, err := os.Stat(filepath.Join(dir, "versions.json")); err != nil { + return fmt.Errorf("local formula for %s: %w", target.Path, err) + } + return nil +} + +func parseTarget(value string) (module.Version, error) { + path, version, ok := strings.Cut(strings.TrimSpace(value), "@") + if !ok || path == "" || version == "" { + return module.Version{}, fmt.Errorf("target must be module@version, got %q", value) + } + return module.Version{Path: path, Version: version}, nil +} + +func parseTargets(value string) ([]module.Version, error) { + parts := strings.Split(value, ",") + targets := make([]module.Version, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + target, err := parseTarget(part) + if err != nil { + return nil, err + } + targets = append(targets, target) + } + return targets, nil +} + +func cacheKey(target module.Version, matrix string) buildcache.Key { + return buildcache.Key{Module: target, Matrix: matrix} +} + +func keyString(key buildcache.Key) string { + return fmt.Sprintf("%s@%s matrix=%s", key.Module.Path, key.Module.Version, key.Matrix) +} + +func targetKey(target module.Version) string { + return target.Path + "@" + target.Version +} + +func objectNameFor(prefix string, key buildcache.Key) string { + parts := make([]string, 0, 4) + if prefix != "" { + parts = append(parts, strings.Trim(prefix, "/")) + } + parts = append(parts, strings.Trim(key.Module.Path, "/"), strings.Trim(key.Module.Version, "/"), key.Matrix+".tar.gz") + return strings.Join(parts, "/") +} + +func normalizePublicDomain(domain string) string { + domain = strings.TrimRight(strings.TrimSpace(domain), "/") + if domain == "" || strings.Contains(domain, "://") { + return domain + } + return "http://" + domain +} + +func parseHTTPURL(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil { + return nil, err + } + if u.Scheme != "http" && u.Scheme != "https" || u.Host == "" { + return nil, fmt.Errorf("must be http(s), got %q", raw) + } + return u, nil +} + +func parseKodoSourceURL(raw string) (string, error) { + u, err := parseHTTPURL(raw) + if err != nil { + return "", err + } + objectName, err := url.PathUnescape(strings.TrimPrefix(u.EscapedPath(), "/")) + if err != nil { + return "", err + } + if objectName == "" { + return "", fmt.Errorf("invalid kodo source url %q", raw) + } + return objectName, nil +} + +func publicURL(domain, objectName string) string { + u, _ := parseHTTPURL(domain) + u.Path = "/" + objectName + u.RawPath = "" + u.RawQuery = "" + u.Fragment = "" + return u.String() +} + +func assertZlibOutput(dir string) error { + for _, name := range []string{ + filepath.Join("include", "zlib.h"), + filepath.Join("lib", "libz.a"), + } { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + return fmt.Errorf("zlib output %s missing in %s: %w", name, dir, err) + } + } + return nil +} + +func isKodoObjectNotFound(err error) bool { + var info *qiniuclient.ErrorInfo + return errors.As(err, &info) && info.Code == 612 +} + +func fileSHA256(name string) (string, error) { + file, err := os.Open(name) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func assertPublicURLChecksum(ctx context.Context, rawURL, checksum string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET %s: %s", rawURL, resp.Status) + } + hash := sha256.New() + if _, err := io.Copy(hash, resp.Body); err != nil { + return err + } + got := hex.EncodeToString(hash.Sum(nil)) + if got != checksum { + return fmt.Errorf("GET %s checksum = %s, want %s", rawURL, got, checksum) + } + return nil +} + +func mustTempDir(pattern string) string { + dir, err := os.MkdirTemp("", pattern) + if err != nil { + panic(err) + } + return dir +} + +func envOrDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func hostMatrix() string { + return runtime.GOARCH + "-" + runtime.GOOS +} + +type buildResult struct { + result build.Result + err error +} + +type namedBuildResult struct { + target module.Version + result build.Result + err error +} + +func waitBuildResult(ctx context.Context, ch <-chan buildResult) (buildResult, error) { + select { + case result := <-ch: + return result, nil + case <-ctx.Done(): + return buildResult{}, ctx.Err() + } +} + +func waitNamedBuildResult(ctx context.Context, ch <-chan namedBuildResult) (namedBuildResult, error) { + select { + case result := <-ch: + return result, nil + case <-ctx.Done(): + return namedBuildResult{}, ctx.Err() + } +}