From 5bacb93a5a390876b79fab64dc17c404251b3c62 Mon Sep 17 00:00:00 2001 From: BK Date: Sun, 4 Jun 2023 18:55:09 -0700 Subject: [PATCH 1/2] initial Azure blob storage --- WORKSPACE | 63 +++++++++++ add-bazel-dep.sh | 1 + go.mod | 9 ++ go.sum | 19 ++++ internal/cli/BUILD.bazel | 1 + internal/cli/helpers.go | 12 +++ internal/cli/init.go | 24 ++++- internal/cli/root.go | 5 + internal/config/config.go | 15 +++ internal/fileutils/BUILD.bazel | 9 ++ internal/fileutils/fileutils.go | 25 +++++ internal/stores/BUILD.bazel | 9 ++ internal/stores/azure.go | 185 ++++++++++++++++++++++++++++++++ internal/stores/azure_test.go | 166 ++++++++++++++++++++++++++++ internal/stores/stores.go | 10 ++ internal/testutils/memfs.go | 1 + 16 files changed, 550 insertions(+), 4 deletions(-) create mode 100644 internal/fileutils/BUILD.bazel create mode 100644 internal/fileutils/fileutils.go create mode 100644 internal/stores/azure.go create mode 100644 internal/stores/azure_test.go diff --git a/WORKSPACE b/WORKSPACE index b8baa092..4c050e3e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -110,6 +110,69 @@ go_repository( version = "v1.1.0", ) +go_repository( + name = "com_github_azure_azure_sdk_for_go", + importpath = "github.com/Azure/azure-sdk-for-go", + sum = "h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=", + version = "v68.0.0+incompatible", +) + +go_repository( + name = "com_github_azure_azure_sdk_for_go_sdk_storage_azblob", + importpath = "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob", + sum = "h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=", + version = "v1.0.0", +) + +go_repository( + name = "com_github_azure_azure_sdk_for_go_sdk_azidentity", + importpath = "github.com/Azure/azure-sdk-for-go/sdk/azidentity", + sum = "h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=", + version = "v1.3.0", +) + +go_repository( + name = "com_github_azure_azure_sdk_for_go_sdk_azcore", + importpath = "github.com/Azure/azure-sdk-for-go/sdk/azcore", + sum = "h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=", + version = "v1.6.0", +) + +go_repository( + name = "com_github_azuread_microsoft_authentication_library_for_go", + importpath = "github.com/AzureAD/microsoft-authentication-library-for-go", + sum = "h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=", + version = "v1.0.0", +) + +go_repository( + name = "com_github_azure_azure_sdk_for_go_sdk_internal", + importpath = "github.com/Azure/azure-sdk-for-go/sdk/internal", + sum = "h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=", + version = "v1.3.0", +) + +go_repository( + name = "com_github_kylelemons_godebug", + importpath = "github.com/kylelemons/godebug", + sum = "h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=", + version = "v1.1.0", +) + +go_repository( + name = "com_github_pkg_browser", + importpath = "github.com/pkg/browser", + sum = "h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=", + version = "v0.0.0-20210911075715-681adbf594b8", +) + +go_repository( + name = "com_github_golang_jwt_jwt_v4", + importpath = "github.com/golang-jwt/jwt/v4", + sum = "h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=", + version = "v4.5.0", +) + bazel_skylib_workspace() load("//:repositories.bzl", "go_repositories") diff --git a/add-bazel-dep.sh b/add-bazel-dep.sh index cfd1f1fd..c2a3d0d7 100755 --- a/add-bazel-dep.sh +++ b/add-bazel-dep.sh @@ -1 +1,2 @@ +go get $1 bazel run //:gazelle -- update-repos $1 \ No newline at end of file diff --git a/go.mod b/go.mod index a6caee4e..69bee9cd 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,12 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.0.1 // indirect cloud.google.com/go/pubsub v1.30.1 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/aws/aws-sdk-go v1.44.256 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect @@ -44,6 +50,7 @@ require ( github.com/felixge/httpsnoop v1.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fsouza/fake-gcs-server v1.45.1 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect @@ -60,9 +67,11 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/xattr v0.4.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect diff --git a/go.sum b/go.sum index f7ff9202..add5bd8c 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,18 @@ cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -137,6 +149,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -257,6 +271,8 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -265,6 +281,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= @@ -478,6 +496,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/cli/BUILD.bazel b/internal/cli/BUILD.bazel index bd4b1755..8c976184 100644 --- a/internal/cli/BUILD.bazel +++ b/internal/cli/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//internal/metadata", "//internal/program", "//internal/stores", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob", "@com_github_google_logger//:logger", "@com_github_spf13_afero//:afero", "@com_github_spf13_cobra//:cobra", diff --git a/internal/cli/helpers.go b/internal/cli/helpers.go index f6b9094a..1a14d0b1 100644 --- a/internal/cli/helpers.go +++ b/internal/cli/helpers.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/discentem/cavorite/internal/config" "github.com/discentem/cavorite/internal/stores" "github.com/google/logger" @@ -54,6 +55,17 @@ func initStoreFromConfig(ctx context.Context, cfg config.Config, fsys afero.Fs, return nil, fmt.Errorf("improper stores.GCSClient init: %v", err) } s = stores.Store(gcs) + case stores.StoreTypeAzureBlob: + az, err := stores.NewAzureBlobStore( + ctx, + fsys, + opts, + azblob.ClientOptions{}, + ) + if err != nil { + return nil, fmt.Errorf("improper stores.AzureBlobStore init: %v", err) + } + s = stores.Store(az) default: return nil, fmt.Errorf("type %s is not supported", cfg.StoreType) } diff --git a/internal/cli/init.go b/internal/cli/init.go index 34384343..a1a14ad7 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -14,10 +14,18 @@ import ( func initCmd() *cobra.Command { initCmd := &cobra.Command{ - Use: "init", - Short: fmt.Sprintf("Initialize a new %s repo", program.Name), - Long: fmt.Sprintf("Initialize a new %s repo", program.Name), - Args: cobra.ExactArgs(1), + Use: "init", + Short: fmt.Sprintf("Initialize a new %s repo", program.Name), + Long: fmt.Sprintf("Initialize a new %s repo", program.Name), + // Args: cobra.ExactArgs(1), + Args: func(cmd *cobra.Command, args []string) error { + fn := cobra.ExactArgs(1) + err := fn(cmd, args) + if err != nil { + return errors.New("you must specify a path to a repo you want cavorite to track") + } + return nil + }, PreRunE: initPreExecFn, RunE: initFn, } @@ -99,6 +107,14 @@ func initFn(cmd *cobra.Command, args []string) error { backendAddress, opts, ) + case stores.StoreTypeAzureBlob: + cfg = config.InitializeStoreTypeAzureBlob( + cmd.Context(), + fsys, + repoToInit, + backendAddress, + opts, + ) default: return config.ErrUnsupportedStore } diff --git a/internal/cli/root.go b/internal/cli/root.go index 3220ea43..23c5bb6d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -8,6 +8,7 @@ import ( "github.com/discentem/cavorite/internal/metadata" "github.com/discentem/cavorite/internal/program" "github.com/discentem/cavorite/internal/stores" + "github.com/google/logger" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -74,6 +75,10 @@ func rootCmd() *cobra.Command { viper.SetDefault("store_type", stores.StoreTypeUndefined) viper.SetDefault("metadata_file_extension", metadata.MetadataFileExtension) + if vv { + logger.SetLevel(2) + } + // Import subCmds into the rootCmd rootCmd.AddCommand( initCmd(), diff --git a/internal/config/config.go b/internal/config/config.go index 7701e5e0..fc260871 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,6 +64,21 @@ func InitializeStoreTypeGCS( } } +func InitializeStoreTypeAzureBlob( + ctx context.Context, + fsys afero.Fs, + sourceRepo, backendAddress string, + opts stores.Options, +) Config { + return Config{ + StoreType: stores.StoreTypeAzureBlob, + Options: opts, + Validate: func() error { + return nil + }, + } +} + func (c *Config) Write(fsys afero.Fs, sourceRepo string) error { if c.Validate == nil { return ErrValidateNil diff --git a/internal/fileutils/BUILD.bazel b/internal/fileutils/BUILD.bazel new file mode 100644 index 00000000..ebaab3b6 --- /dev/null +++ b/internal/fileutils/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "fileutils", + srcs = ["fileutils.go"], + importpath = "github.com/discentem/cavorite/internal/fileutils", + visibility = ["//:__subpackages__"], + deps = ["@com_github_spf13_afero//:afero"], +) diff --git a/internal/fileutils/fileutils.go b/internal/fileutils/fileutils.go new file mode 100644 index 00000000..96f458b5 --- /dev/null +++ b/internal/fileutils/fileutils.go @@ -0,0 +1,25 @@ +package fileutils + +import ( + "fmt" + "io" + + "github.com/spf13/afero" +) + +func BytesFromAferoFile(f afero.File) ([]byte, error) { + _, err := f.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + objInfo, err := f.Stat() + if err != nil { + return nil, err + } + b := make([]byte, objInfo.Size()) + _, err = f.Read(b) + if err != nil { + return nil, fmt.Errorf("failed to read bytes from objectHandle: %w", err) + } + return b, nil +} diff --git a/internal/stores/BUILD.bazel b/internal/stores/BUILD.bazel index d26c3fce..f723763e 100644 --- a/internal/stores/BUILD.bazel +++ b/internal/stores/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "stores", srcs = [ + "azure.go", "gcs.go", "options.go", "s3.go", @@ -16,6 +17,10 @@ go_library( "@com_github_aws_aws_sdk_go_v2_config//:config", "@com_github_aws_aws_sdk_go_v2_feature_s3_manager//:manager", "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", + "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//blob", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//blockblob", "@com_github_google_logger//:logger", "@com_github_hashicorp_go_multierror//:go-multierror", "@com_github_spf13_afero//:afero", @@ -27,6 +32,7 @@ go_library( go_test( name = "stores_test", srcs = [ + "azure_test.go", "gcs_test.go", "s3_test.go", "stores_test.go", @@ -36,9 +42,12 @@ go_test( "//internal/testutils", "@com_github_aws_aws_sdk_go_v2_feature_s3_manager//:manager", "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//blob", "@com_github_fsouza_fake_gcs_server//fakestorage", "@com_github_spf13_afero//:afero", "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", "@com_google_cloud_go_storage//:storage", "@org_golang_google_api//option", ], diff --git a/internal/stores/azure.go b/internal/stores/azure.go new file mode 100644 index 00000000..6474cbf7 --- /dev/null +++ b/internal/stores/azure.go @@ -0,0 +1,185 @@ +package stores + +import ( + "context" + "fmt" + "io" + "net/url" + "path" + "path/filepath" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/discentem/cavorite/internal/metadata" + "github.com/google/logger" + "github.com/spf13/afero" +) + +// azureBlobishClient is derived from https://github.com/Azure/azure-sdk-for-go/blob/sdk/storage/azblob/v1.0.0/sdk/storage/azblob/client.go#L34 +type azureBlobishClient interface { + // DownloadBuffer(ctx context.Context, containerName string, blobName string, buffer []byte, o *azblob.DownloadBufferOptions) (int64, error) + // UploadBuffer(ctx context.Context, containerName string, blobName string, buffer []byte, o *azblob.UploadBufferOptions) (azblob.UploadBufferResponse, error) + UploadStream(ctx context.Context, containerName string, blobName string, body io.Reader, o *azblob.UploadStreamOptions) (azblob.UploadStreamResponse, error) + DownloadStream(ctx context.Context, containerName string, blobName string, o *azblob.DownloadStreamOptions) (azblob.DownloadStreamResponse, error) +} + +type AzureBlobStore struct { + Options Options + containerClient azureBlobishClient + fsys afero.Fs +} + +func (s *AzureBlobStore) GetOptions() Options { return s.Options } +func (s *AzureBlobStore) GetFsys() afero.Fs { return s.fsys } + +func (s *AzureBlobStore) Upload(ctx context.Context, objects ...string) error { + for _, o := range objects { + f, err := s.fsys.Open(o) + if err != nil { + return err + } + // cleanupFn is function that can be called if + // uploading to blob storage fails. cleanupFn deletes the cfile + // so that we don't retain a cfile without a corresponding binary + cleanupFn, err := WriteMetadataToFsys(s, o, f) + if err != nil { + return err + } + containerName := path.Base(s.Options.BackendAddress) + _, err = s.containerClient.UploadStream( + ctx, + containerName, + o, + f, + &blockblob.UploadStreamOptions{ + Concurrency: 25, + }, + ) + if err != nil { + if err := cleanupFn(); err != nil { + return err + } + return err + } + if err := f.Close(); err != nil { + return err + } + } + + return nil +} +func (s *AzureBlobStore) Retrieve(ctx context.Context, objects ...string) error { + for _, o := range objects { + // For Retrieve, the object is the cfile itself, which we derive the actual filename from + objectPath := strings.TrimSuffix(o, filepath.Ext(o)) + // We will either read the file that already exists or download it because it + // is missing + f, err := openOrCreateFile(s.fsys, objectPath) + if err != nil { + return err + } + fileInfo, err := f.Stat() + if err != nil { + return err + } + if fileInfo.Size() > 0 { + logger.Infof("%s already exists", objectPath) + } else { + containerName := path.Base(s.Options.BackendAddress) + logger.Infof("containerName: %s", containerName) + // Download the file + resp, err := s.containerClient.DownloadStream( + ctx, + containerName, + objectPath, + &blob.DownloadStreamOptions{}) + if err != nil { + return err + } + if resp.Body == nil { + return fmt.Errorf("blob %q was nil", o) + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + _, err = f.Write(b) + if err != nil { + return err + } + + } + // Get the hash for the downloaded file + hash, err := metadata.SHA256FromReader(f) + if err != nil { + return err + } + // Get the metadata from the metadata file + m, err := metadata.ParseCfile(s.fsys, o) + if err != nil { + return err + } + // If the hash of the downloaded file does not match the retrieved file, return an error + if hash != m.Checksum { + logger.Infof("Hash mismatch, got %s but expected %s", hash, m.Checksum) + if err := s.fsys.Remove(objectPath); err != nil { + return err + } + return ErrRetrieveFailureHashMismatch + } + if err := f.Close(); err != nil { + return err + } + } + return nil + +} + +func newAzureContainerClient(serviceURL string, options azblob.ClientOptions) (*azblob.Client, error) { + // We only support Azure CLI authentication. + // In the future we could support multiple types with + // azidentity.NewChainedTokenCredential() + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return nil, err + } + container, err := azblob.NewClient( + serviceURL, + cred, + // &azblob.ClientOptions{ + // ClientOptions: policy.ClientOptions{ + // Retry: policy.RetryOptions{ + // TryTimeout: time.Second * 5, + // MaxRetryDelay: time.Second * 10, + // }, + // }, + // }, + nil, + ) + if err != nil { + return nil, err + } + return container, nil +} + +func NewAzureBlobStore(ctx context.Context, fsys afero.Fs, storeOpts Options, azureBlobOptions azblob.ClientOptions) (*AzureBlobStore, error) { + u, err := url.Parse(storeOpts.BackendAddress) + if err != nil { + return nil, err + } + containerClient, err := newAzureContainerClient( + fmt.Sprintf("https://%s/", u.Host), + azureBlobOptions, + ) + if err != nil { + return nil, err + } + return &AzureBlobStore{ + Options: storeOpts, + containerClient: containerClient, + fsys: fsys, + }, nil +} diff --git a/internal/stores/azure_test.go b/internal/stores/azure_test.go new file mode 100644 index 00000000..f5b60d7d --- /dev/null +++ b/internal/stores/azure_test.go @@ -0,0 +1,166 @@ +package stores + +import ( + "context" + "fmt" + "io" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/discentem/cavorite/internal/testutils" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type aferoAzureBlobServer struct { + containers map[string]afero.Fs +} + +var ( + // ensure test server meets interface + _ = azureBlobishClient(aferoAzureBlobServer{}) +) + +func (s aferoAzureBlobServer) UploadStream(ctx context.Context, containerName string, blobName string, body io.Reader, o *azblob.UploadStreamOptions) (azblob.UploadStreamResponse, error) { + + // check if the containerName exists in s.containers + _, ok := s.containers[containerName] + if !ok { + return azblob.UploadStreamResponse{}, + fmt.Errorf( + "%s does not exist in this aferoAzureBlobServer", + containerName, + ) + } + + b, err := io.ReadAll(body) + if err != nil { + return azblob.UploadStreamResponse{}, err + } + + // create a filesystem for container referenced in input + containerfs, _ := testutils.MemMapFsWith(map[string]testutils.MapFile{ + blobName: { + // write input body to bucketfs + Content: b, + }, + }) + // write containerfs to associated "container" + s.containers[containerName] = *containerfs + + return azblob.UploadStreamResponse{}, nil +} + +func (s aferoAzureBlobServer) DownloadStream(ctx context.Context, containerName string, blobName string, o *azblob.DownloadStreamOptions) (azblob.DownloadStreamResponse, error) { + _, ok := s.containers[containerName] + if !ok { + return azblob.DownloadStreamResponse{}, + fmt.Errorf( + "%s does not exist in this aferoAzureBlobServer", + containerName, + ) + } + + objectHandle, err := s.containers[containerName].Open(blobName) + if err != nil { + return azblob.DownloadStreamResponse{}, + fmt.Errorf("could not find %s in container %s: %w", blobName, containerName, err) + } + _, err = objectHandle.Seek(0, io.SeekStart) + if err != nil { + return azblob.DownloadStreamResponse{}, err + } + + return azblob.DownloadStreamResponse{ + DownloadResponse: blob.DownloadResponse{ + Body: io.NopCloser(objectHandle), + }, + }, nil +} + +func TestAzureBlobStoreUpload(t *testing.T) { + mTime, _ := time.Parse("2006-01-02T15:04:05.000Z", "2014-11-12T11:45:26.371Z") + memfs, err := testutils.MemMapFsWith(map[string]testutils.MapFile{ + "test": { + Content: []byte("tree"), + ModTime: &mTime, + }, + }) + assert.NoError(t, err) + + fakeAzureBlobServer := aferoAzureBlobServer{ + containers: map[string]afero.Fs{ + // create a bucket in our fake azure blob server + "test": afero.NewMemMapFs(), + }, + } + store := AzureBlobStore{ + Options: Options{ + BackendAddress: "http://whatever/test", + MetadataFileExtension: "cfile", + }, + fsys: *memfs, + containerClient: fakeAzureBlobServer, + } + err = store.Upload(context.Background(), "test") + require.NoError(t, err) + b, _ := afero.ReadFile(*memfs, "test.cfile") + assert.Equal(t, `{ + "name": "test", + "checksum": "dc9c5edb8b2d479e697b4b0b8ab874f32b325138598ce9e7b759eb8292110622", + "date_modified": "2014-11-12T11:45:26.371Z" +}`, string(b)) +} + +func TestAzureBlobStoreRetrieve(t *testing.T) { + mTime, _ := time.Parse("2006-01-02T15:04:05.000Z", "2014-11-12T11:45:26.371Z") + // create bucket content + bucketfs, err := testutils.MemMapFsWith(map[string]testutils.MapFile{ + "someObject": { + Content: []byte("tla"), + ModTime: &mTime, + }, + }) + assert.NoError(t, err) + // bfs := *bucketfs + // mfs := bfs.(*afero.MemMapFs) + + fakeAzureBlobServer := aferoAzureBlobServer{ + containers: map[string]afero.Fs{ + // create a bucket in our fake azure blob server + "aFakeBucket": *bucketfs, + }, + } + + localFs, err := testutils.MemMapFsWith(map[string]testutils.MapFile{ + "someObject.cfile": { + Content: []byte(`{ + "name": "someObject", + "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "date_modified": "2014-11-12T11:45:26.371Z" + }`), + ModTime: &mTime, + }, + }) + assert.NoError(t, err) + + store := AzureBlobStore{ + Options: Options{ + BackendAddress: "http://whatever/aFakeBucket", + MetadataFileExtension: "cfile", + }, + fsys: *localFs, + containerClient: fakeAzureBlobServer, + } + + err = store.Retrieve(context.Background(), "someObject.cfile") + assert.NoError(t, err) + + // ensure the content of the file is correct + b, _ := afero.ReadFile(*localFs, "someObject") + assert.Equal(t, `tla`, string(b)) + +} diff --git a/internal/stores/stores.go b/internal/stores/stores.go index 851532b5..faba08e7 100644 --- a/internal/stores/stores.go +++ b/internal/stores/stores.go @@ -20,6 +20,7 @@ const ( StoreTypeUndefined StoreType = "undefined" StoreTypeS3 StoreType = "s3" StoreTypeGCS StoreType = "gcs" + StoreTypeAzureBlob StoreType = "azure" ) var ( @@ -33,6 +34,15 @@ type Store interface { GetFsys() afero.Fs } +var ( + // ensure AzureblobStore meets Store interface + _ = Store(&AzureBlobStore{}) + // ensure S3Store meets Store interface + _ = Store(&S3Store{}) + // ensure GCSStore meets Store interface + _ = Store(&GCSStore{}) +) + func openOrCreateFile(fsys afero.Fs, filename string) (afero.File, error) { file, err := fsys.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0644) if err != nil { diff --git a/internal/testutils/memfs.go b/internal/testutils/memfs.go index 74148ffe..df008342 100644 --- a/internal/testutils/memfs.go +++ b/internal/testutils/memfs.go @@ -36,5 +36,6 @@ func MemMapFsWith(files map[string]MapFile) (*afero.Fs, error) { } } } + return &memfsys, nil } From 0472810af5419edc99a767b85e25d24018d23923 Mon Sep 17 00:00:00 2001 From: BK Date: Tue, 6 Jun 2023 09:02:27 -0700 Subject: [PATCH 2/2] progres for azure blob uploads --- WORKSPACE | 28 +++++++++ go.mod | 5 ++ go.sum | 18 ++++++ internal/stores/BUILD.bazel | 3 + internal/stores/azure.go | 104 +++++++++++++++++++++------------- internal/stores/azure_test.go | 4 ++ 6 files changed, 123 insertions(+), 39 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 4c050e3e..7819d247 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -173,6 +173,34 @@ go_repository( version = "v4.5.0", ) +go_repository( + name = "com_github_schollz_progressbar_v3", + importpath = "github.com/schollz/progressbar/v3", + sum = "h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=", + version = "v3.13.1", +) + +go_repository( + name = "com_github_mattn_go_runewidth", + importpath = "github.com/mattn/go-runewidth", + sum = "h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=", + version = "v0.0.14", +) + +go_repository( + name = "com_github_mitchellh_colorstring", + importpath = "github.com/mitchellh/colorstring", + sum = "h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=", + version = "v0.0.0-20190213212951-d06e56a500db", +) + +go_repository( + name = "com_github_rivo_uniseg", + importpath = "github.com/rivo/uniseg", + sum = "h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=", + version = "v0.4.4", +) + bazel_skylib_workspace() load("//:repositories.bzl", "go_repositories") diff --git a/go.mod b/go.mod index 69bee9cd..84457cb9 100644 --- a/go.mod +++ b/go.mod @@ -69,12 +69,16 @@ require ( github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/xattr v0.4.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/schollz/progressbar/v3 v3.13.1 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -88,6 +92,7 @@ require ( golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index add5bd8c..daacf6d8 100644 --- a/go.sum +++ b/go.sum @@ -264,6 +264,7 @@ github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 h1:O7syWuYG github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -275,6 +276,11 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -290,6 +296,10 @@ github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6kt github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= @@ -297,6 +307,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= +github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= @@ -320,6 +332,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -501,11 +514,13 @@ golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -513,7 +528,10 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/stores/BUILD.bazel b/internal/stores/BUILD.bazel index f723763e..43815a49 100644 --- a/internal/stores/BUILD.bazel +++ b/internal/stores/BUILD.bazel @@ -12,6 +12,7 @@ go_library( importpath = "github.com/discentem/cavorite/internal/stores", visibility = ["//:__subpackages__"], deps = [ + "//internal/fileutils", "//internal/metadata", "@com_github_aws_aws_sdk_go_v2//aws", "@com_github_aws_aws_sdk_go_v2_config//:config", @@ -21,8 +22,10 @@ go_library( "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//:azblob", "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//blob", "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//blockblob", + "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//container", "@com_github_google_logger//:logger", "@com_github_hashicorp_go_multierror//:go-multierror", + "@com_github_schollz_progressbar_v3//:progressbar", "@com_github_spf13_afero//:afero", "@com_google_cloud_go_storage//:storage", "@org_golang_google_api//option", diff --git a/internal/stores/azure.go b/internal/stores/azure.go index 6474cbf7..4537bd6e 100644 --- a/internal/stores/azure.go +++ b/internal/stores/azure.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/url" + "os" "path" "path/filepath" "strings" @@ -13,9 +14,13 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/discentem/cavorite/internal/fileutils" "github.com/discentem/cavorite/internal/metadata" "github.com/google/logger" "github.com/spf13/afero" + + progressbar "github.com/schollz/progressbar/v3" ) // azureBlobishClient is derived from https://github.com/Azure/azure-sdk-for-go/blob/sdk/storage/azblob/v1.0.0/sdk/storage/azblob/client.go#L34 @@ -24,17 +29,25 @@ type azureBlobishClient interface { // UploadBuffer(ctx context.Context, containerName string, blobName string, buffer []byte, o *azblob.UploadBufferOptions) (azblob.UploadBufferResponse, error) UploadStream(ctx context.Context, containerName string, blobName string, body io.Reader, o *azblob.UploadStreamOptions) (azblob.UploadStreamResponse, error) DownloadStream(ctx context.Context, containerName string, blobName string, o *azblob.DownloadStreamOptions) (azblob.DownloadStreamResponse, error) + // NewBlobClient(blobName string) *blob.Client } type AzureBlobStore struct { Options Options - containerClient azureBlobishClient + containerClient *container.Client fsys afero.Fs } func (s *AzureBlobStore) GetOptions() Options { return s.Options } func (s *AzureBlobStore) GetFsys() afero.Fs { return s.fsys } +func bytesTransferredFn(w io.Writer, progbar *progressbar.ProgressBar) func(bytesTransferred int64) { + return func(bytesTransferred int64) { + progbar.Set64(bytesTransferred) + w.Write([]byte(progbar.String())) + } +} + func (s *AzureBlobStore) Upload(ctx context.Context, objects ...string) error { for _, o := range objects { f, err := s.fsys.Open(o) @@ -48,16 +61,27 @@ func (s *AzureBlobStore) Upload(ctx context.Context, objects ...string) error { if err != nil { return err } - containerName := path.Base(s.Options.BackendAddress) - _, err = s.containerClient.UploadStream( - ctx, - containerName, - o, - f, - &blockblob.UploadStreamOptions{ - Concurrency: 25, - }, - ) + + blobClient := s.containerClient.NewBlockBlobClient(o) + stat, err := f.Stat() + if err != nil { + return err + } + + if err != nil { + return err + } + b, err := fileutils.BytesFromAferoFile(f) + if err != nil { + return err + } + progbar := progressbar.DefaultBytesSilent(stat.Size(), o) + + blobClient.UploadBuffer(ctx, b, &blockblob.UploadBufferOptions{ + Progress: bytesTransferredFn(os.Stdout, progbar), + }) + fmt.Println(progbar.String()) + if err != nil { if err := cleanupFn(); err != nil { return err @@ -90,19 +114,27 @@ func (s *AzureBlobStore) Retrieve(ctx context.Context, objects ...string) error } else { containerName := path.Base(s.Options.BackendAddress) logger.Infof("containerName: %s", containerName) - // Download the file - resp, err := s.containerClient.DownloadStream( - ctx, - containerName, - objectPath, - &blob.DownloadStreamOptions{}) + + blobClient := s.containerClient.NewBlobClient(o) + blobProps, err := blobClient.GetProperties(ctx, nil) + size := blobProps.ContentLength if err != nil { return err } - if resp.Body == nil { - return fmt.Errorf("blob %q was nil", o) + if err := f.Truncate(*size); err != nil { + return err } - b, err := io.ReadAll(resp.Body) + + var b []byte + progbar := progressbar.DefaultBytesSilent(*size, o) + + // Download the file + _, err = blobClient.DownloadBuffer( + ctx, + b, + &blob.DownloadBufferOptions{ + Progress: bytesTransferredFn(os.Stdout, progbar), + }) if err != nil { return err } @@ -138,7 +170,11 @@ func (s *AzureBlobStore) Retrieve(ctx context.Context, objects ...string) error } -func newAzureContainerClient(serviceURL string, options azblob.ClientOptions) (*azblob.Client, error) { +func newAZContainerClient(backendAddress string) (*container.Client, error) { + u, err := url.Parse(backendAddress) + if err != nil { + return nil, err + } // We only support Azure CLI authentication. // In the future we could support multiple types with // azidentity.NewChainedTokenCredential() @@ -146,18 +182,13 @@ func newAzureContainerClient(serviceURL string, options azblob.ClientOptions) (* if err != nil { return nil, err } - container, err := azblob.NewClient( - serviceURL, + containerURL := fmt.Sprintf("https://%s/%s", u.Host, path.Base(backendAddress)) + + container, err := container.NewClient( + // Construct container url + containerURL, cred, - // &azblob.ClientOptions{ - // ClientOptions: policy.ClientOptions{ - // Retry: policy.RetryOptions{ - // TryTimeout: time.Second * 5, - // MaxRetryDelay: time.Second * 10, - // }, - // }, - // }, - nil, + &container.ClientOptions{}, ) if err != nil { return nil, err @@ -166,13 +197,8 @@ func newAzureContainerClient(serviceURL string, options azblob.ClientOptions) (* } func NewAzureBlobStore(ctx context.Context, fsys afero.Fs, storeOpts Options, azureBlobOptions azblob.ClientOptions) (*AzureBlobStore, error) { - u, err := url.Parse(storeOpts.BackendAddress) - if err != nil { - return nil, err - } - containerClient, err := newAzureContainerClient( - fmt.Sprintf("https://%s/", u.Host), - azureBlobOptions, + containerClient, err := newAZContainerClient( + storeOpts.BackendAddress, ) if err != nil { return nil, err diff --git a/internal/stores/azure_test.go b/internal/stores/azure_test.go index f5b60d7d..a588232f 100644 --- a/internal/stores/azure_test.go +++ b/internal/stores/azure_test.go @@ -81,6 +81,10 @@ func (s aferoAzureBlobServer) DownloadStream(ctx context.Context, containerName }, nil } +func (s aferoAzureBlobServer) NewBlobClient(blobName string) *blob.Client { + return nil +} + func TestAzureBlobStoreUpload(t *testing.T) { mTime, _ := time.Parse("2006-01-02T15:04:05.000Z", "2014-11-12T11:45:26.371Z") memfs, err := testutils.MemMapFsWith(map[string]testutils.MapFile{