From 6d380caa8a2ed702cd027dc31b373be43f4be8b4 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 15:29:00 +0800 Subject: [PATCH 1/2] feat(build): add kodo artifact store --- .github/workflows/kodo.yml | 54 ++++ go.mod | 7 + go.sum | 27 +- internal/{artfact => artifact}/artifact.go | 0 internal/{artfact => artifact}/gorm.go | 0 internal/{artfact => artifact}/gorm_test.go | 38 +++ internal/artifact/kodo.go | 122 +++++++ internal/artifact/kodo_test.go | 342 ++++++++++++++++++++ 8 files changed, 588 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/kodo.yml rename internal/{artfact => artifact}/artifact.go (100%) rename internal/{artfact => artifact}/gorm.go (100%) rename internal/{artfact => artifact}/gorm_test.go (85%) create mode 100644 internal/artifact/kodo.go create mode 100644 internal/artifact/kodo_test.go diff --git a/.github/workflows/kodo.yml b/.github/workflows/kodo.yml new file mode 100644 index 00000000..fa0f3df5 --- /dev/null +++ b/.github/workflows/kodo.yml @@ -0,0 +1,54 @@ +name: Kodo E2E + +on: + push: + branches: + - "**" + - "!dependabot/**" + - "!xgopilot/**" + pull_request_target: + branches: [ "**" ] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +jobs: + kodo-artifact: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.x + + - name: Download Go modules + run: go mod download + + - name: Check Kodo secrets + env: + QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} + QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} + QINIU_BUCKET: ${{ secrets.QINIU_BUCKET }} + run: | + test -n "$QINIU_ACCESS_KEY" + test -n "$QINIU_SECRET_KEY" + test -n "$QINIU_BUCKET" + + - name: Run Kodo artifact E2E + env: + QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} + QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} + QINIU_BUCKET: ${{ secrets.QINIU_BUCKET }} + QINIU_PREFIX: ${{ secrets.QINIU_PREFIX }} + run: go test -v -ldflags="-checklinkname=0" ./internal/artifact -run '^TestKodoArtifactE2E$' -count=1 diff --git a/go.mod b/go.mod index 3eca0e7e..87ed41a4 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/goplus/xgo v1.6.1 github.com/jessevdk/go-flags v1.6.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 + github.com/qiniu/go-sdk/v7 v7.26.14 github.com/qiniu/x v1.16.0 github.com/spf13/cobra v1.10.2 golang.org/x/mod v0.32.0 @@ -18,10 +19,14 @@ require ( ) require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/containerd/stargz-snapshotter/estargz v0.18.1 // indirect github.com/docker/cli v29.0.3+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect + github.com/gammazero/toposort v0.1.1 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/goplus/gogen v1.20.6 // indirect github.com/goplus/reflectx v1.5.0 // indirect @@ -29,6 +34,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/klauspost/compress v1.18.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -44,4 +50,5 @@ require ( golang.org/x/sync v0.19.0 // indirect golang.org/x/text v0.20.0 // indirect golang.org/x/tools v0.40.0 // indirect + modernc.org/fileutil v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index f0f9236c..b1e1febd 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,11 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8= github.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -10,6 +15,10 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= +github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-containerregistry v0.20.7 h1:24VGNpS0IwrOZ2ms2P1QE3Xa5X9p4phx0aUgzYzHW6I= @@ -27,6 +36,8 @@ github.com/goplus/reflectx v1.5.0 h1:d6THLWzrQJCIAnNzyXJDA7J1sxj8ELzKH+Wj17X/dtE github.com/goplus/reflectx v1.5.0/go.mod h1:wHOS9ilbB4zrecI0W1dMmkW9JMcpXV7VjALVbNU9xfM= github.com/goplus/xgo v1.6.1 h1:xe4ezrXkBvK5317inkPIqsKVZR3/J0oyT4lcvT9mYPc= github.com/goplus/xgo v1.6.1/go.mod h1:vEnp8PO5JCJBNXYjnsSTJ1hgAEvbP/IUWzC17pVz4R8= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= @@ -37,10 +48,14 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -51,8 +66,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qiniu/go-sdk/v7 v7.26.14 h1:kV9zdvTOM0w9/lgeYffDDw0Fj8FH14/XgCjk4Wb80k8= +github.com/qiniu/go-sdk/v7 v7.26.14/go.mod h1:ri7fGwbio0pRDFr8EK5TUpx0DbnpIMJ2bMSDxGWfCbk= github.com/qiniu/x v1.16.0 h1:W2VOecyIT3Uxwjm6vJinUR7G3gpwgUgHZA9OpeHArdE= github.com/qiniu/x v1.16.0/go.mod h1:AiovSOCaRijaf3fj+0CBOpR1457pn24b0Vdb1JpwhII= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -95,6 +114,8 @@ golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -104,3 +125,5 @@ gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= diff --git a/internal/artfact/artifact.go b/internal/artifact/artifact.go similarity index 100% rename from internal/artfact/artifact.go rename to internal/artifact/artifact.go diff --git a/internal/artfact/gorm.go b/internal/artifact/gorm.go similarity index 100% rename from internal/artfact/gorm.go rename to internal/artifact/gorm.go diff --git a/internal/artfact/gorm_test.go b/internal/artifact/gorm_test.go similarity index 85% rename from internal/artfact/gorm_test.go rename to internal/artifact/gorm_test.go index e1c62a0f..032be941 100644 --- a/internal/artfact/gorm_test.go +++ b/internal/artifact/gorm_test.go @@ -156,6 +156,44 @@ func TestGormStoreDelete(t *testing.T) { } } +func TestGormStoreDatabaseErrors(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + t.Fatalf("gorm.Open: %v", err) + } + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("db.DB: %v", err) + } + store, err := NewGormStore(db) + if err != nil { + t.Fatalf("NewGormStore: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("db.Close: %v", err) + } + + ctx := context.Background() + key := Key{Module: "madler/zlib", Version: "v1.3.1", MatrixStr: "amd64-linux"} + value := Artifact{ + Source: Source{Type: "ghcr", URL: "https://ghcr.io/v2/meteorsliu/llar/blobs/sha256:abc"}, + Type: "tar.gz", + Metadata: "-lz", + Checksum: "abc", + } + if _, _, err := store.Get(ctx, key); err == nil { + t.Fatal("Get with closed database = nil, want error") + } + if _, err := store.Put(ctx, key, value); err == nil { + t.Fatal("Put with closed database = nil, want error") + } + if err := store.Delete(ctx, key); err == nil { + t.Fatal("Delete with closed database = nil, want error") + } +} + func newTestGormStore(t *testing.T) (*GormStore, *sql.DB) { t.Helper() return newTestGormStoreWithLogger(t, logger.Default.LogMode(logger.Silent)) diff --git a/internal/artifact/kodo.go b/internal/artifact/kodo.go new file mode 100644 index 00000000..938d306a --- /dev/null +++ b/internal/artifact/kodo.go @@ -0,0 +1,122 @@ +package artifact + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + + qiniuclient "github.com/qiniu/go-sdk/v7/client" + "github.com/qiniu/go-sdk/v7/storagev2/credentials" + httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/go-sdk/v7/storagev2/objects" +) + +const kodoArtifactMetadataKey = "llar-artifact" + +type KodoArtifactConfig struct { + AccessKey string + SecretKey string + Bucket string + Prefix string +} + +type kodoArtifact struct { + bucket string + prefix string + objects *objects.ObjectsManager +} + +func NewKodoArtifact(cfg KodoArtifactConfig) Store { + cred := credentials.NewCredentials(cfg.AccessKey, cfg.SecretKey) + return &kodoArtifact{ + bucket: cfg.Bucket, + prefix: strings.Trim(cfg.Prefix, "/"), + objects: objects.NewObjectsManager(&objects.ObjectsManagerOptions{ + Options: httpclient.Options{Credentials: cred}, + }), + } +} + +func (s *kodoArtifact) Get(ctx context.Context, key Key) (Artifact, bool, error) { + objectName := s.objectName(key) + object, err := s.objects.Bucket(s.bucket).Object(objectName).Stat().Call(ctx) + if err != nil { + if kodoArtifactObjectNotFound(err) { + return Artifact{}, false, nil + } + return Artifact{}, false, err + } + got, ok := kodoArtifactFromMetadata(object.Metadata) + if !ok { + return Artifact{}, false, fmt.Errorf("read kodo artifact metadata for %s", objectName) + } + return got, true, nil +} + +func (s *kodoArtifact) Put(ctx context.Context, key Key, art Artifact) (Artifact, error) { + got, ok, err := s.Get(ctx, key) + if err != nil { + return art, err + } + if !ok { + return art, fmt.Errorf("kodo artifact object %s missing", s.objectName(key)) + } + return got, nil +} + +func (s *kodoArtifact) Delete(ctx context.Context, key Key) error { + err := s.objects.Bucket(s.bucket).Object(s.objectName(key)).Delete().Call(ctx) + if err != nil && !kodoArtifactObjectNotFound(err) { + return err + } + return nil +} + +func (s *kodoArtifact) objectName(key Key) string { + parts := make([]string, 0, 4) + if s.prefix != "" { + parts = append(parts, s.prefix) + } + parts = append(parts, strings.Trim(key.Module, "/"), strings.Trim(key.Version, "/"), key.MatrixStr+".tar.gz") + return strings.Join(parts, "/") +} + +func encodeKodoArtifact(art Artifact) (string, error) { + data, err := json.Marshal(art) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(data), nil +} + +func kodoArtifactFromMetadata(metadata map[string]string) (Artifact, bool) { + raw := kodoArtifactMetadataValue(metadata, kodoArtifactMetadataKey) + if raw == "" { + return Artifact{}, false + } + data, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil { + return Artifact{}, false + } + var art Artifact + if err := json.Unmarshal(data, &art); err != nil { + return Artifact{}, false + } + return art, true +} + +func kodoArtifactMetadataValue(metadata map[string]string, key string) string { + value := metadata[key] + if value == "" { + value = metadata["x-qn-meta-"+key] + } + return value +} + +func kodoArtifactObjectNotFound(err error) bool { + var info *qiniuclient.ErrorInfo + return errors.As(err, &info) && info.Code == 612 +} diff --git a/internal/artifact/kodo_test.go b/internal/artifact/kodo_test.go new file mode 100644 index 00000000..e6b44b36 --- /dev/null +++ b/internal/artifact/kodo_test.go @@ -0,0 +1,342 @@ +package artifact + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + qiniuclient "github.com/qiniu/go-sdk/v7/client" + "github.com/qiniu/go-sdk/v7/storagev2/apis/stat_object" + "github.com/qiniu/go-sdk/v7/storagev2/credentials" + httpclient "github.com/qiniu/go-sdk/v7/storagev2/http_client" + "github.com/qiniu/go-sdk/v7/storagev2/objects" + "github.com/qiniu/go-sdk/v7/storagev2/region" + "github.com/qiniu/go-sdk/v7/storagev2/uploader" + "github.com/qiniu/go-sdk/v7/storagev2/uptoken" +) + +func TestKodoArtifactObjectName(t *testing.T) { + store := NewKodoArtifact(KodoArtifactConfig{Prefix: "/cache/"}).(*kodoArtifact) + key := Key{Module: "madler/zlib", Version: "v1.3.2", MatrixStr: "amd64-linux"} + if got, want := store.objectName(key), "cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"; got != want { + t.Fatalf("object name = %q, want %q", got, want) + } +} + +func TestKodoArtifactMetadataRoundTrip(t *testing.T) { + want := Artifact{ + Source: Source{Type: "kodo", URL: "https://example.com/cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"}, + Type: "tar.gz", + Metadata: "-lz", + Checksum: "abc", + } + raw, err := encodeKodoArtifact(want) + if err != nil { + t.Fatalf("encode: %v", err) + } + for _, metadata := range []map[string]string{ + {kodoArtifactMetadataKey: raw}, + {"x-qn-meta-" + kodoArtifactMetadataKey: raw}, + } { + got, ok := kodoArtifactFromMetadata(metadata) + if !ok { + t.Fatalf("decode missed metadata %+v", metadata) + } + if got != want { + t.Fatalf("decode = %+v, want %+v", got, want) + } + } +} + +func TestKodoArtifactMetadataInvalid(t *testing.T) { + for _, metadata := range []map[string]string{ + {}, + {kodoArtifactMetadataKey: "not-base64"}, + {kodoArtifactMetadataKey: base64.RawURLEncoding.EncodeToString([]byte("{"))}, + } { + if got, ok := kodoArtifactFromMetadata(metadata); ok { + t.Fatalf("decode %+v = %+v, true; want false", metadata, got) + } + } +} + +func TestKodoArtifactObjectNotFound(t *testing.T) { + if !kodoArtifactObjectNotFound(&qiniuclient.ErrorInfo{Code: 612}) { + t.Fatal("612 should be object not found") + } + if kodoArtifactObjectNotFound(&qiniuclient.ErrorInfo{Code: 500}) { + t.Fatal("500 should not be object not found") + } + if kodoArtifactObjectNotFound(context.Canceled) { + t.Fatal("context.Canceled should not be object not found") + } +} + +func TestKodoArtifactGetPutDeleteWithFakeKodo(t *testing.T) { + key := Key{Module: "madler/zlib", Version: "v1.3.2", MatrixStr: "amd64-linux"} + want := Artifact{ + Source: Source{Type: "kodo", URL: "https://example.com/cache/madler/zlib/v1.3.2/amd64-linux.tar.gz"}, + Type: "tar.gz", + Metadata: "-lz", + Checksum: "abc", + } + raw, err := encodeKodoArtifact(want) + if err != nil { + t.Fatalf("encode: %v", err) + } + objectName := "cache/madler/zlib/v1.3.2/amd64-linux.tar.gz" + server := newFakeKodoObjectServer(t, "bucket", objectName, map[string]string{ + "x-qn-meta-" + kodoArtifactMetadataKey: raw, + }) + defer server.Close() + store := newFakeKodoArtifactStore(server.URL, "bucket", "cache") + + got, ok, err := store.Get(context.Background(), key) + if err != nil { + t.Fatalf("Get: %v", err) + } + if !ok { + t.Fatal("Get missed object") + } + if got != want { + t.Fatalf("Get = %+v, want %+v", got, want) + } + + conflict := Artifact{ + Source: Source{Type: "kodo", URL: "https://example.com/conflict"}, + Type: "tar.gz", + Metadata: "-lz-conflict", + Checksum: "conflict", + } + got, err = store.Put(context.Background(), key, conflict) + if err != nil { + t.Fatalf("Put: %v", err) + } + if got != want { + t.Fatalf("Put = %+v, want %+v", got, want) + } + + if err := store.Delete(context.Background(), key); err != nil { + t.Fatalf("Delete: %v", err) + } +} + +func TestKodoArtifactGetErrors(t *testing.T) { + key := Key{Module: "madler/zlib", Version: "v1.3.2", MatrixStr: "amd64-linux"} + + t.Run("not found", func(t *testing.T) { + server := newFakeKodoObjectServer(t, "bucket", "other", nil) + defer server.Close() + store := newFakeKodoArtifactStore(server.URL, "bucket", "") + + got, ok, err := store.Get(context.Background(), key) + if err != nil { + t.Fatalf("Get: %v", err) + } + if ok { + t.Fatalf("Get = %+v, true; want miss", got) + } + }) + + t.Run("missing metadata", func(t *testing.T) { + objectName := "madler/zlib/v1.3.2/amd64-linux.tar.gz" + server := newFakeKodoObjectServer(t, "bucket", objectName, nil) + defer server.Close() + store := newFakeKodoArtifactStore(server.URL, "bucket", "") + + if got, ok, err := store.Get(context.Background(), key); err == nil { + t.Fatalf("Get = %+v, %v, nil; want metadata error", got, ok) + } + }) +} + +func TestKodoArtifactPutDeleteErrors(t *testing.T) { + key := Key{Module: "madler/zlib", Version: "v1.3.2", MatrixStr: "amd64-linux"} + value := Artifact{Source: Source{Type: "kodo", URL: "https://example.com"}, Type: "tar.gz"} + + t.Run("put missing object", func(t *testing.T) { + server := newFakeKodoObjectServer(t, "bucket", "other", nil) + defer server.Close() + store := newFakeKodoArtifactStore(server.URL, "bucket", "") + + if got, err := store.Put(context.Background(), key, value); err == nil { + t.Fatalf("Put = %+v, nil; want missing object error", got) + } + }) + + t.Run("delete missing object", func(t *testing.T) { + server := newFakeKodoObjectServer(t, "bucket", "other", nil) + defer server.Close() + store := newFakeKodoArtifactStore(server.URL, "bucket", "") + + if err := store.Delete(context.Background(), key); err != nil { + t.Fatalf("Delete missing object: %v", err) + } + }) + + t.Run("delete server error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error":"boom"}`)) + })) + defer server.Close() + store := newFakeKodoArtifactStore(server.URL, "bucket", "") + + if err := store.Delete(context.Background(), key); err == nil { + t.Fatal("Delete server error = nil, want error") + } + }) +} + +func TestKodoArtifactE2E(t *testing.T) { + accessKey := os.Getenv("QINIU_ACCESS_KEY") + secretKey := os.Getenv("QINIU_SECRET_KEY") + bucket := os.Getenv("QINIU_BUCKET") + if accessKey == "" || secretKey == "" || bucket == "" { + t.Skip("QINIU_ACCESS_KEY, QINIU_SECRET_KEY, and QINIU_BUCKET are required") + } + + prefix := strings.Trim(os.Getenv("QINIU_PREFIX"), "/") + if prefix != "" { + prefix += "/" + } + prefix += fmt.Sprintf("llar-kodo-artifact-e2e/%d", time.Now().UnixNano()) + store := NewKodoArtifact(KodoArtifactConfig{ + AccessKey: accessKey, + SecretKey: secretKey, + Bucket: bucket, + Prefix: prefix, + }).(*kodoArtifact) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + key := Key{Module: "madler/zlib", Version: "v1.3.2", MatrixStr: "amd64-linux"} + objectName := store.objectName(key) + want := Artifact{ + Source: Source{Type: "kodo", URL: "https://example.com/" + objectName}, + Type: "tar.gz", + Metadata: "-lz", + Checksum: "sha256-test", + } + if err := uploadKodoArtifactObject(ctx, t, accessKey, secretKey, bucket, objectName, want); err != nil { + t.Fatalf("upload object: %v", err) + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := store.Delete(cleanupCtx, key); err != nil { + t.Errorf("delete object: %v", err) + } + }) + + got, ok, err := store.Get(ctx, key) + if err != nil { + t.Fatalf("Get: %v", err) + } + if !ok { + t.Fatal("Get missed uploaded object") + } + if got != want { + t.Fatalf("Get = %+v, want %+v", got, want) + } + + conflict := Artifact{ + Source: Source{Type: "kodo", URL: "https://example.com/conflict"}, + Type: "tar.gz", + Metadata: "-lz-conflict", + Checksum: "sha256-conflict", + } + got, err = store.Put(ctx, key, conflict) + if err != nil { + t.Fatalf("Put conflict: %v", err) + } + if got != want { + t.Fatalf("Put conflict = %+v, want canonical %+v", got, want) + } +} + +func newFakeKodoArtifactStore(rsURL, bucket, prefix string) *kodoArtifact { + cred := credentials.NewCredentials("testak", "testsk") + return &kodoArtifact{ + bucket: bucket, + prefix: strings.Trim(prefix, "/"), + objects: objects.NewObjectsManager(&objects.ObjectsManagerOptions{ + Options: httpclient.Options{ + Credentials: cred, + Regions: ®ion.Region{Rs: region.Endpoints{Preferred: []string{rsURL}}}, + }, + }), + } +} + +func newFakeKodoObjectServer(t *testing.T, bucket, objectName string, metadata map[string]string) *httptest.Server { + t.Helper() + + entry := base64.URLEncoding.EncodeToString([]byte(bucket + ":" + objectName)) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Reqid", "fake-reqid") + switch { + case r.Method == http.MethodGet && r.URL.RequestURI() == "/stat/"+entry: + response := stat_object.Response{ + Hash: "etag", + MimeType: "application/gzip", + PutTime: time.Now().UnixNano() / 100, + Metadata: metadata, + } + data, err := json.Marshal(&response) + if err != nil { + t.Fatalf("marshal stat response: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(data) + case r.Method == http.MethodPost && r.URL.RequestURI() == "/delete/"+entry: + w.WriteHeader(http.StatusOK) + case strings.HasPrefix(r.URL.Path, "/stat/"), strings.HasPrefix(r.URL.Path, "/delete/"): + w.WriteHeader(612) + _, _ = w.Write([]byte(`{"error":"not found"}`)) + default: + t.Fatalf("unexpected %s %s", r.Method, r.URL.RequestURI()) + } + })) +} + +func uploadKodoArtifactObject(ctx context.Context, t *testing.T, accessKey, secretKey, bucket, objectName string, art Artifact) error { + t.Helper() + + raw, err := encodeKodoArtifact(art) + if err != nil { + return err + } + file := filepath.Join(t.TempDir(), "tar.gz") + if err := os.WriteFile(file, []byte("artifact"), 0o644); err != nil { + return err + } + cred := credentials.NewCredentials(accessKey, secretKey) + options := httpclient.Options{Credentials: cred} + manager := uploader.NewUploadManager(&uploader.UploadManagerOptions{Options: options}) + putPolicy, err := uptoken.NewPutPolicyWithKey(bucket, objectName, time.Now().Add(time.Hour)) + if err != nil { + return err + } + putPolicy.SetInsertOnly(1) + return manager.UploadFile(ctx, file, &uploader.ObjectOptions{ + BucketName: bucket, + ObjectName: &objectName, + FileName: path.Base(objectName), + ContentType: "application/gzip", + UpToken: uptoken.NewSigner(putPolicy, cred), + Metadata: map[string]string{ + kodoArtifactMetadataKey: raw, + }, + }, nil) +} From 02762e212397e3650e9a7398e1e94642be7b46e9 Mon Sep 17 00:00:00 2001 From: Rick Guo Date: Fri, 3 Jul 2026 16:34:34 +0800 Subject: [PATCH 2/2] fix(artifact): write kodo metadata on put --- internal/artifact/kodo.go | 16 ++++--- internal/artifact/kodo_test.go | 81 +++++++++++++++------------------- 2 files changed, 47 insertions(+), 50 deletions(-) diff --git a/internal/artifact/kodo.go b/internal/artifact/kodo.go index 938d306a..b4170eb7 100644 --- a/internal/artifact/kodo.go +++ b/internal/artifact/kodo.go @@ -14,7 +14,10 @@ import ( "github.com/qiniu/go-sdk/v7/storagev2/objects" ) -const kodoArtifactMetadataKey = "llar-artifact" +const ( + kodoArtifactContentType = "application/gzip" + kodoArtifactMetadataKey = "llar-artifact" +) type KodoArtifactConfig struct { AccessKey string @@ -57,14 +60,17 @@ func (s *kodoArtifact) Get(ctx context.Context, key Key) (Artifact, bool, error) } func (s *kodoArtifact) Put(ctx context.Context, key Key, art Artifact) (Artifact, error) { - got, ok, err := s.Get(ctx, key) + raw, err := encodeKodoArtifact(art) if err != nil { return art, err } - if !ok { - return art, fmt.Errorf("kodo artifact object %s missing", s.objectName(key)) + err = s.objects.Bucket(s.bucket).Object(s.objectName(key)).SetMetadata(kodoArtifactContentType). + Metadata(map[string]string{kodoArtifactMetadataKey: raw}). + Call(ctx) + if err != nil { + return art, err } - return got, nil + return art, nil } func (s *kodoArtifact) Delete(ctx context.Context, key Key) error { diff --git a/internal/artifact/kodo_test.go b/internal/artifact/kodo_test.go index e6b44b36..b33390f1 100644 --- a/internal/artifact/kodo_test.go +++ b/internal/artifact/kodo_test.go @@ -89,17 +89,19 @@ func TestKodoArtifactGetPutDeleteWithFakeKodo(t *testing.T) { Metadata: "-lz", Checksum: "abc", } - raw, err := encodeKodoArtifact(want) - if err != nil { - t.Fatalf("encode: %v", err) - } objectName := "cache/madler/zlib/v1.3.2/amd64-linux.tar.gz" - server := newFakeKodoObjectServer(t, "bucket", objectName, map[string]string{ - "x-qn-meta-" + kodoArtifactMetadataKey: raw, - }) + server := newFakeKodoObjectServer(t, "bucket", objectName, nil) defer server.Close() store := newFakeKodoArtifactStore(server.URL, "bucket", "cache") + got, err := store.Put(context.Background(), key, want) + if err != nil { + t.Fatalf("Put: %v", err) + } + if got != want { + t.Fatalf("Put = %+v, want %+v", got, want) + } + got, ok, err := store.Get(context.Background(), key) if err != nil { t.Fatalf("Get: %v", err) @@ -111,20 +113,6 @@ func TestKodoArtifactGetPutDeleteWithFakeKodo(t *testing.T) { t.Fatalf("Get = %+v, want %+v", got, want) } - conflict := Artifact{ - Source: Source{Type: "kodo", URL: "https://example.com/conflict"}, - Type: "tar.gz", - Metadata: "-lz-conflict", - Checksum: "conflict", - } - got, err = store.Put(context.Background(), key, conflict) - if err != nil { - t.Fatalf("Put: %v", err) - } - if got != want { - t.Fatalf("Put = %+v, want %+v", got, want) - } - if err := store.Delete(context.Background(), key); err != nil { t.Fatalf("Delete: %v", err) } @@ -228,7 +216,7 @@ func TestKodoArtifactE2E(t *testing.T) { Metadata: "-lz", Checksum: "sha256-test", } - if err := uploadKodoArtifactObject(ctx, t, accessKey, secretKey, bucket, objectName, want); err != nil { + if err := uploadKodoArtifactObject(ctx, t, accessKey, secretKey, bucket, objectName); err != nil { t.Fatalf("upload object: %v", err) } t.Cleanup(func() { @@ -239,6 +227,14 @@ func TestKodoArtifactE2E(t *testing.T) { } }) + got, err := store.Put(ctx, key, want) + if err != nil { + t.Fatalf("Put: %v", err) + } + if got != want { + t.Fatalf("Put = %+v, want %+v", got, want) + } + got, ok, err := store.Get(ctx, key) if err != nil { t.Fatalf("Get: %v", err) @@ -249,20 +245,6 @@ func TestKodoArtifactE2E(t *testing.T) { if got != want { t.Fatalf("Get = %+v, want %+v", got, want) } - - conflict := Artifact{ - Source: Source{Type: "kodo", URL: "https://example.com/conflict"}, - Type: "tar.gz", - Metadata: "-lz-conflict", - Checksum: "sha256-conflict", - } - got, err = store.Put(ctx, key, conflict) - if err != nil { - t.Fatalf("Put conflict: %v", err) - } - if got != want { - t.Fatalf("Put conflict = %+v, want canonical %+v", got, want) - } } func newFakeKodoArtifactStore(rsURL, bucket, prefix string) *kodoArtifact { @@ -301,7 +283,23 @@ func newFakeKodoObjectServer(t *testing.T, bucket, objectName string, metadata m _, _ = w.Write(data) case r.Method == http.MethodPost && r.URL.RequestURI() == "/delete/"+entry: w.WriteHeader(http.StatusOK) - case strings.HasPrefix(r.URL.Path, "/stat/"), strings.HasPrefix(r.URL.Path, "/delete/"): + case r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/chgm/"+entry+"/mime/"): + parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/chgm/"+entry+"/mime/"), "/") + if len(parts) < 3 || len(parts)%2 != 1 { + t.Fatalf("unexpected chgm path: %s", r.URL.Path) + } + if metadata == nil { + metadata = map[string]string{} + } + for i := 1; i < len(parts); i += 2 { + value, err := base64.URLEncoding.DecodeString(parts[i+1]) + if err != nil { + t.Fatalf("decode metadata value: %v", err) + } + metadata[parts[i]] = string(value) + } + w.WriteHeader(http.StatusOK) + case strings.HasPrefix(r.URL.Path, "/stat/"), strings.HasPrefix(r.URL.Path, "/delete/"), strings.HasPrefix(r.URL.Path, "/chgm/"): w.WriteHeader(612) _, _ = w.Write([]byte(`{"error":"not found"}`)) default: @@ -310,13 +308,9 @@ func newFakeKodoObjectServer(t *testing.T, bucket, objectName string, metadata m })) } -func uploadKodoArtifactObject(ctx context.Context, t *testing.T, accessKey, secretKey, bucket, objectName string, art Artifact) error { +func uploadKodoArtifactObject(ctx context.Context, t *testing.T, accessKey, secretKey, bucket, objectName string) error { t.Helper() - raw, err := encodeKodoArtifact(art) - if err != nil { - return err - } file := filepath.Join(t.TempDir(), "tar.gz") if err := os.WriteFile(file, []byte("artifact"), 0o644); err != nil { return err @@ -335,8 +329,5 @@ func uploadKodoArtifactObject(ctx context.Context, t *testing.T, accessKey, secr FileName: path.Base(objectName), ContentType: "application/gzip", UpToken: uptoken.NewSigner(putPolicy, cred), - Metadata: map[string]string{ - kodoArtifactMetadataKey: raw, - }, }, nil) }