From f6c727d564f4f126f72f49706a1ef681ad2cd2f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 06:44:17 +0000 Subject: [PATCH] Bump github.com/google/go-containerregistry from 0.21.5 to 0.21.6 Bumps [github.com/google/go-containerregistry](https://github.com/google/go-containerregistry) from 0.21.5 to 0.21.6. - [Release notes](https://github.com/google/go-containerregistry/releases) - [Commits](https://github.com/google/go-containerregistry/compare/v0.21.5...v0.21.6) --- updated-dependencies: - dependency-name: github.com/google/go-containerregistry dependency-version: 0.21.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 +- go.sum | 8 +- .../stargz-snapshotter/estargz/LICENSE | 202 -- .../stargz-snapshotter/estargz/build.go | 756 ------ .../estargz/errorutil/errors.go | 40 - .../stargz-snapshotter/estargz/estargz.go | 1232 --------- .../stargz-snapshotter/estargz/gzip.go | 237 -- .../stargz-snapshotter/estargz/testutil.go | 2403 ----------------- .../stargz-snapshotter/estargz/types.go | 342 --- .../internal/estargz/estargz.go | 54 - .../pkg/authn/keychain.go | 3 +- .../go-containerregistry/pkg/name/registry.go | 4 +- .../go-containerregistry/pkg/v1/manifest.go | 2 + .../pkg/v1/mutate/image.go | 141 +- .../pkg/v1/mutate/mutate.go | 123 +- .../pkg/v1/partial/with.go | 13 +- .../pkg/v1/remote/fetcher.go | 128 +- .../pkg/v1/remote/image.go | 40 +- .../pkg/v1/remote/index.go | 8 +- .../pkg/v1/remote/limiter.go | 55 + .../pkg/v1/remote/options.go | 3 + .../pkg/v1/remote/pusher.go | 8 +- .../pkg/v1/remote/referrers.go | 2 +- .../pkg/v1/remote/transport/bearer.go | 42 +- .../pkg/v1/remote/transport/error.go | 15 +- .../pkg/v1/remote/transport/scope.go | 6 +- .../pkg/v1/remote/write.go | 63 +- .../pkg/v1/tarball/layer.go | 57 +- .../pkg/v1/validate/image.go | 18 +- vendor/github.com/vbatts/tar-split/LICENSE | 28 - .../vbatts/tar-split/archive/tar/common.go | 724 ----- .../vbatts/tar-split/archive/tar/format.go | 307 --- .../vbatts/tar-split/archive/tar/reader.go | 944 ------- .../tar-split/archive/tar/stat_actime1.go | 20 - .../tar-split/archive/tar/stat_actime2.go | 20 - .../vbatts/tar-split/archive/tar/stat_unix.go | 96 - .../vbatts/tar-split/archive/tar/strconv.go | 326 --- .../vbatts/tar-split/archive/tar/writer.go | 656 ----- vendor/modules.txt | 10 +- 39 files changed, 515 insertions(+), 8625 deletions(-) delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/build.go delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go delete mode 100644 vendor/github.com/containerd/stargz-snapshotter/estargz/types.go delete mode 100644 vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go delete mode 100644 vendor/github.com/vbatts/tar-split/LICENSE delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/common.go delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/format.go delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/reader.go delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/strconv.go delete mode 100644 vendor/github.com/vbatts/tar-split/archive/tar/writer.go diff --git a/go.mod b/go.mod index efa988ed..6a55e27a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/buildpacks/lifecycle v0.21.12 github.com/buildpacks/pack v0.40.6 github.com/cespare/xxhash/v2 v2.3.0 - github.com/google/go-containerregistry v0.21.5 + github.com/google/go-containerregistry v0.21.6 github.com/jarcoal/httpmock v1.4.1 github.com/moby/moby/api v1.54.2 github.com/onsi/ginkgo/v2 v2.29.0 @@ -105,7 +105,6 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v1.0.0-rc.4 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/curioswitch/go-reassign v0.3.0 // indirect @@ -285,7 +284,6 @@ require ( github.com/ultraware/whitespace v0.2.0 // indirect github.com/uudashr/gocognit v1.2.0 // indirect github.com/uudashr/iface v1.4.1 // indirect - github.com/vbatts/tar-split v0.12.2 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xen0n/gosmopolitan v1.3.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect diff --git a/go.sum b/go.sum index eb44594f..d43a0a4a 100644 --- a/go.sum +++ b/go.sum @@ -213,8 +213,6 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0Q5b3op97T4= github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A= -github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw= -github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY= github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= @@ -394,8 +392,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.21.5 h1:KTJG9Pn/jC0VdZR6ctV3/jcN+q6/Iqlx0sTVz3ywZlM= -github.com/google/go-containerregistry v0.21.5/go.mod h1:ySvMuiWg+dOsRW0Hw8GYwfMwBlNRTmpYBFJPlkco5zU= +github.com/google/go-containerregistry v0.21.6 h1:T+yqQIlJXKrM98Om4DlW3GoWQAmhZuLMwoDOvVrtiUM= +github.com/google/go-containerregistry v0.21.6/go.mod h1:U7MMSBIJynke2MVQrQk19NP9k/uQsGz/h0amIFSHMbo= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= @@ -792,8 +790,6 @@ github.com/uudashr/gocognit v1.2.0 h1:3BU9aMr1xbhPlvJLSydKwdLN3tEUUrzPSSM8S4hDYR github.com/uudashr/gocognit v1.2.0/go.mod h1:k/DdKPI6XBZO1q7HgoV2juESI2/Ofj9AcHPZhBBdrTU= github.com/uudashr/iface v1.4.1 h1:J16Xl1wyNX9ofhpHmQ9h9gk5rnv2A6lX/2+APLTo0zU= github.com/uudashr/iface v1.4.1/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= -github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= -github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE b/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE deleted file mode 100644 index d6456956..00000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go deleted file mode 100644 index a9e1b72b..00000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/build.go +++ /dev/null @@ -1,756 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "errors" - "fmt" - "io" - "os" - "path" - "runtime" - "strings" - "sync" - "sync/atomic" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - "github.com/klauspost/compress/zstd" - digest "github.com/opencontainers/go-digest" - "golang.org/x/sync/errgroup" -) - -type GzipHelperFunc func(io.Reader) (io.ReadCloser, error) - -type options struct { - chunkSize int - compressionLevel int - prioritizedFiles []string - missedPrioritizedFiles *[]string - compression Compression - ctx context.Context - minChunkSize int - gzipHelperFunc GzipHelperFunc -} - -type Option func(o *options) error - -// WithChunkSize option specifies the chunk size of eStargz blob to build. -func WithChunkSize(chunkSize int) Option { - return func(o *options) error { - o.chunkSize = chunkSize - return nil - } -} - -// WithCompressionLevel option specifies the gzip compression level. -// The default is gzip.BestCompression. -// This option will be ignored if WithCompression option is used. -// See also: https://godoc.org/compress/gzip#pkg-constants -func WithCompressionLevel(level int) Option { - return func(o *options) error { - o.compressionLevel = level - return nil - } -} - -// WithPrioritizedFiles option specifies the list of prioritized files. -// These files must be complete paths that are absolute or relative to "/" -// For example, all of "foo/bar", "/foo/bar", "./foo/bar" and "../foo/bar" -// are treated as "/foo/bar". -func WithPrioritizedFiles(files []string) Option { - return func(o *options) error { - o.prioritizedFiles = files - return nil - } -} - -// WithAllowPrioritizeNotFound makes Build continue the execution even if some -// of prioritized files specified by WithPrioritizedFiles option aren't found -// in the input tar. Instead, this records all missed file names to the passed -// slice. -func WithAllowPrioritizeNotFound(missedFiles *[]string) Option { - return func(o *options) error { - if missedFiles == nil { - return fmt.Errorf("WithAllowPrioritizeNotFound: slice must be passed") - } - o.missedPrioritizedFiles = missedFiles - return nil - } -} - -// WithCompression specifies compression algorithm to be used. -// Default is gzip. -func WithCompression(compression Compression) Option { - return func(o *options) error { - o.compression = compression - return nil - } -} - -// WithContext specifies a context that can be used for clean canceleration. -func WithContext(ctx context.Context) Option { - return func(o *options) error { - o.ctx = ctx - return nil - } -} - -// WithMinChunkSize option specifies the minimal number of bytes of data -// must be written in one gzip stream. -// By increasing this number, one gzip stream can contain multiple files -// and it hopefully leads to smaller result blob. -// NOTE: This adds a TOC property that old reader doesn't understand. -func WithMinChunkSize(minChunkSize int) Option { - return func(o *options) error { - o.minChunkSize = minChunkSize - return nil - } -} - -// WithGzipHelperFunc option specifies a custom function to decompress gzip-compressed layers. -// When a gzip-compressed layer is detected, this function will be used instead of the -// Go standard library gzip decompression for better performance. -// The function should take an io.Reader as input and return an io.ReadCloser. -// If nil, the Go standard library gzip.NewReader will be used. -func WithGzipHelperFunc(gzipHelperFunc GzipHelperFunc) Option { - return func(o *options) error { - o.gzipHelperFunc = gzipHelperFunc - return nil - } -} - -// Blob is an eStargz blob. -type Blob struct { - io.ReadCloser - diffID digest.Digester - tocDigest digest.Digest - readCompleted *atomic.Bool - uncompressedSize *atomic.Int64 -} - -// DiffID returns the digest of uncompressed blob. -// It is only valid to call DiffID after Close. -func (b *Blob) DiffID() digest.Digest { - return b.diffID.Digest() -} - -// TOCDigest returns the digest of uncompressed TOC JSON. -func (b *Blob) TOCDigest() digest.Digest { - return b.tocDigest -} - -// UncompressedSize returns the size of uncompressed blob. -// UncompressedSize should only be called after the blob has been fully read. -func (b *Blob) UncompressedSize() (int64, error) { - switch { - case b.uncompressedSize == nil || b.readCompleted == nil: - return -1, fmt.Errorf("readCompleted or uncompressedSize is not initialized") - case !b.readCompleted.Load(): - return -1, fmt.Errorf("called UncompressedSize before the blob has been fully read") - default: - return b.uncompressedSize.Load(), nil - } -} - -// Build builds an eStargz blob which is an extended version of stargz, from a blob (gzip, zstd -// or plain tar) passed through the argument. If there are some prioritized files are listed in -// the option, these files are grouped as "prioritized" and can be used for runtime optimization -// (e.g. prefetch). This function builds a blob in parallel, with dividing that blob into several -// (at least the number of runtime.GOMAXPROCS(0)) sub-blobs. -func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) { - var opts options - opts.compressionLevel = gzip.BestCompression // BestCompression by default - for _, o := range opt { - if err := o(&opts); err != nil { - return nil, err - } - } - if opts.compression == nil { - opts.compression = newGzipCompressionWithLevel(opts.compressionLevel) - } - layerFiles := newTempFiles() - ctx := opts.ctx - if ctx == nil { - ctx = context.Background() - } - done := make(chan struct{}) - defer close(done) - go func() { - select { - case <-done: - // nop - case <-ctx.Done(): - layerFiles.CleanupAll() - } - }() - defer func() { - if rErr != nil { - if err := layerFiles.CleanupAll(); err != nil { - rErr = fmt.Errorf("failed to cleanup tmp files: %v: %w", err, rErr) - } - } - if cErr := ctx.Err(); cErr != nil { - rErr = fmt.Errorf("error from context %q: %w", cErr, rErr) - } - }() - tarBlob, err := decompressBlob(tarBlob, layerFiles, opts.gzipHelperFunc) - if err != nil { - return nil, err - } - entries, err := sortEntries(tarBlob, opts.prioritizedFiles, opts.missedPrioritizedFiles) - if err != nil { - return nil, err - } - var tarParts [][]*entry - if opts.minChunkSize > 0 { - // Each entry needs to know the size of the current gzip stream so they - // cannot be processed in parallel. - tarParts = [][]*entry{entries} - } else { - tarParts = divideEntries(entries, runtime.GOMAXPROCS(0)) - } - writers := make([]*Writer, len(tarParts)) - payloads := make([]*os.File, len(tarParts)) - var mu sync.Mutex - var eg errgroup.Group - for i, parts := range tarParts { - i, parts := i, parts - // builds verifiable stargz sub-blobs - eg.Go(func() error { - esgzFile, err := layerFiles.TempFile("", "esgzdata") - if err != nil { - return err - } - sw := NewWriterWithCompressor(esgzFile, opts.compression) - sw.ChunkSize = opts.chunkSize - sw.MinChunkSize = opts.minChunkSize - if sw.needsOpenGzEntries == nil { - sw.needsOpenGzEntries = make(map[string]struct{}) - } - for _, f := range []string{PrefetchLandmark, NoPrefetchLandmark} { - sw.needsOpenGzEntries[f] = struct{}{} - } - if err := sw.AppendTar(readerFromEntries(parts...)); err != nil { - return err - } - mu.Lock() - writers[i] = sw - payloads[i] = esgzFile - mu.Unlock() - return nil - }) - } - if err := eg.Wait(); err != nil { - rErr = err - return nil, err - } - tocAndFooter, tocDgst, err := closeWithCombine(writers...) - if err != nil { - rErr = err - return nil, err - } - var rs []io.Reader - for _, p := range payloads { - fs, err := fileSectionReader(p) - if err != nil { - return nil, err - } - rs = append(rs, fs) - } - diffID := digest.Canonical.Digester() - pr, pw := io.Pipe() - readCompleted := new(atomic.Bool) - uncompressedSize := new(atomic.Int64) - go func() { - var size int64 - var decompressFunc func(io.Reader) (io.ReadCloser, error) - if _, ok := opts.compression.(*gzipCompression); ok && opts.gzipHelperFunc != nil { - decompressFunc = opts.gzipHelperFunc - } else { - decompressFunc = opts.compression.Reader - } - decompressR, err := decompressFunc(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw)) - if err != nil { - pw.CloseWithError(err) - return - } - defer decompressR.Close() - if size, err = io.Copy(diffID.Hash(), decompressR); err != nil { - pw.CloseWithError(err) - return - } - uncompressedSize.Store(size) - readCompleted.Store(true) - pw.Close() - }() - return &Blob{ - ReadCloser: readCloser{ - Reader: pr, - closeFunc: layerFiles.CleanupAll, - }, - tocDigest: tocDgst, - diffID: diffID, - readCompleted: readCompleted, - uncompressedSize: uncompressedSize, - }, nil -} - -// closeWithCombine takes unclosed Writers and close them. This also returns the -// toc that combined all Writers into. -// Writers doesn't write TOC and footer to the underlying writers so they can be -// combined into a single eStargz and tocAndFooter returned by this function can -// be appended at the tail of that combined blob. -func closeWithCombine(ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) { - if len(ws) == 0 { - return nil, "", fmt.Errorf("at least one writer must be passed") - } - for _, w := range ws { - if w.closed { - return nil, "", fmt.Errorf("writer must be unclosed") - } - defer func(w *Writer) { w.closed = true }(w) - if err := w.closeGz(); err != nil { - return nil, "", err - } - if err := w.bw.Flush(); err != nil { - return nil, "", err - } - } - var ( - mtoc = new(JTOC) - currentOffset int64 - ) - mtoc.Version = ws[0].toc.Version - for _, w := range ws { - for _, e := range w.toc.Entries { - // Recalculate Offset of non-empty files/chunks - if (e.Type == "reg" && e.Size > 0) || e.Type == "chunk" { - e.Offset += currentOffset - } - mtoc.Entries = append(mtoc.Entries, e) - } - if w.toc.Version > mtoc.Version { - mtoc.Version = w.toc.Version - } - currentOffset += w.cw.n - } - - return tocAndFooter(ws[0].compressor, mtoc, currentOffset) -} - -func tocAndFooter(compressor Compressor, toc *JTOC, offset int64) (io.Reader, digest.Digest, error) { - buf := new(bytes.Buffer) - tocDigest, err := compressor.WriteTOCAndFooter(buf, offset, toc, nil) - if err != nil { - return nil, "", err - } - return buf, tocDigest, nil -} - -// divideEntries divides passed entries to the parts at least the number specified by the -// argument. -func divideEntries(entries []*entry, minPartsNum int) (set [][]*entry) { - var estimatedSize int64 - for _, e := range entries { - estimatedSize += e.header.Size - } - unitSize := estimatedSize / int64(minPartsNum) - var ( - nextEnd = unitSize - offset int64 - ) - set = append(set, []*entry{}) - for _, e := range entries { - set[len(set)-1] = append(set[len(set)-1], e) - offset += e.header.Size - if offset > nextEnd { - set = append(set, []*entry{}) - nextEnd += unitSize - } - } - return -} - -var errNotFound = errors.New("not found") - -// sortEntries reads the specified tar blob and returns a list of tar entries. -// If some of prioritized files are specified, the list starts from these -// files with keeping the order specified by the argument. -func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]string) ([]*entry, error) { - - // Import tar file. - intar, err := importTar(in) - if err != nil { - return nil, fmt.Errorf("failed to sort: %w", err) - } - - // Sort the tar file respecting to the prioritized files list. - sorted := &tarFile{} - picked := make(map[string]struct{}) - for _, l := range prioritized { - if err := moveRec(l, intar, sorted, picked); err != nil { - if errors.Is(err, errNotFound) && missedPrioritized != nil { - *missedPrioritized = append(*missedPrioritized, l) - continue // allow not found - } - return nil, fmt.Errorf("failed to sort tar entries: %w", err) - } - } - if len(prioritized) == 0 { - sorted.add(&entry{ - header: &tar.Header{ - Name: NoPrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }, - payload: bytes.NewReader([]byte{landmarkContents}), - }) - } else { - sorted.add(&entry{ - header: &tar.Header{ - Name: PrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - }, - payload: bytes.NewReader([]byte{landmarkContents}), - }) - } - - // Dump prioritized entries followed by the rest entries while skipping picked ones. - return append(sorted.dump(nil), intar.dump(picked)...), nil -} - -// readerFromEntries returns a reader of tar archive that contains entries passed -// through the arguments. -func readerFromEntries(entries ...*entry) io.Reader { - pr, pw := io.Pipe() - go func() { - tw := tar.NewWriter(pw) - defer tw.Close() - for _, entry := range entries { - if err := tw.WriteHeader(entry.header); err != nil { - pw.CloseWithError(fmt.Errorf("failed to write tar header: %v", err)) - return - } - if _, err := io.Copy(tw, entry.payload); err != nil { - pw.CloseWithError(fmt.Errorf("failed to write tar payload: %v", err)) - return - } - } - pw.Close() - }() - return pr -} - -func importTar(in io.ReaderAt) (*tarFile, error) { - tf := &tarFile{} - pw, err := newCountReadSeeker(in) - if err != nil { - return nil, fmt.Errorf("failed to make position watcher: %w", err) - } - tr := tar.NewReader(pw) - - // Walk through all nodes. - for { - // Fetch and parse next header. - h, err := tr.Next() - if err != nil { - if err == io.EOF { - break - } - return nil, fmt.Errorf("failed to parse tar file, %w", err) - } - switch cleanEntryName(h.Name) { - case PrefetchLandmark, NoPrefetchLandmark: - // Ignore existing landmark - continue - } - - // Add entry. If it already exists, replace it. - if _, ok := tf.get(h.Name); ok { - tf.remove(h.Name) - } - tf.add(&entry{ - header: h, - payload: io.NewSectionReader(in, pw.currentPos(), h.Size), - }) - } - - return tf, nil -} - -func moveRec(name string, in *tarFile, out *tarFile, picked map[string]struct{}) error { - name = cleanEntryName(name) - if name == "" { // root directory. stop recursion. - if e, ok := in.get(name); ok { - // entry of the root directory exists. we should move it as well. - // this case will occur if tar entries are prefixed with "./", "/", etc. - if _, done := picked[name]; !done { - out.add(e) - picked[name] = struct{}{} - } - } - return nil - } - - _, okIn := in.get(name) - _, okOut := out.get(name) - _, okPicked := picked[name] - if !okIn && !okOut && !okPicked { - return fmt.Errorf("file: %q: %w", name, errNotFound) - } - - parent, _ := path.Split(strings.TrimSuffix(name, "/")) - if err := moveRec(parent, in, out, picked); err != nil { - return err - } - if e, ok := in.get(name); ok && e.header.Typeflag == tar.TypeLink { - if err := moveRec(e.header.Linkname, in, out, picked); err != nil { - return err - } - } - if _, done := picked[name]; done { - return nil - } - if e, ok := in.get(name); ok { - out.add(e) - picked[name] = struct{}{} - } - return nil -} - -type entry struct { - header *tar.Header - payload io.ReadSeeker -} - -type tarFile struct { - index map[string]*entry - stream []*entry -} - -func (f *tarFile) add(e *entry) { - if f.index == nil { - f.index = make(map[string]*entry) - } - f.index[cleanEntryName(e.header.Name)] = e - f.stream = append(f.stream, e) -} - -func (f *tarFile) remove(name string) { - name = cleanEntryName(name) - if f.index != nil { - delete(f.index, name) - } - var filtered []*entry - for _, e := range f.stream { - if cleanEntryName(e.header.Name) == name { - continue - } - filtered = append(filtered, e) - } - f.stream = filtered -} - -func (f *tarFile) get(name string) (e *entry, ok bool) { - if f.index == nil { - return nil, false - } - e, ok = f.index[cleanEntryName(name)] - return -} - -func (f *tarFile) dump(skip map[string]struct{}) []*entry { - if len(skip) == 0 { - return f.stream - } - var out []*entry - for _, e := range f.stream { - if _, ok := skip[cleanEntryName(e.header.Name)]; ok { - continue - } - out = append(out, e) - } - return out -} - -type readCloser struct { - io.Reader - closeFunc func() error -} - -func (rc readCloser) Close() error { - return rc.closeFunc() -} - -func fileSectionReader(file *os.File) (*io.SectionReader, error) { - info, err := file.Stat() - if err != nil { - return nil, err - } - return io.NewSectionReader(file, 0, info.Size()), nil -} - -func newTempFiles() *tempFiles { - return &tempFiles{} -} - -type tempFiles struct { - files []*os.File - filesMu sync.Mutex - cleanupOnce sync.Once -} - -func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) { - f, err := os.CreateTemp(dir, pattern) - if err != nil { - return nil, err - } - tf.filesMu.Lock() - tf.files = append(tf.files, f) - tf.filesMu.Unlock() - return f, nil -} - -func (tf *tempFiles) CleanupAll() (err error) { - tf.cleanupOnce.Do(func() { - err = tf.cleanupAll() - }) - return -} - -func (tf *tempFiles) cleanupAll() error { - tf.filesMu.Lock() - defer tf.filesMu.Unlock() - var allErr []error - for _, f := range tf.files { - if err := f.Close(); err != nil { - allErr = append(allErr, err) - } - if err := os.Remove(f.Name()); err != nil { - allErr = append(allErr, err) - } - } - tf.files = nil - return errorutil.Aggregate(allErr) -} - -func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) { - pos := int64(0) - return &countReadSeeker{r: r, cPos: &pos}, nil -} - -type countReadSeeker struct { - r io.ReaderAt - cPos *int64 - - mu sync.Mutex -} - -func (cr *countReadSeeker) Read(p []byte) (int, error) { - cr.mu.Lock() - defer cr.mu.Unlock() - - n, err := cr.r.ReadAt(p, *cr.cPos) - if err == nil { - *cr.cPos += int64(n) - } - return n, err -} - -func (cr *countReadSeeker) Seek(offset int64, whence int) (int64, error) { - cr.mu.Lock() - defer cr.mu.Unlock() - - switch whence { - default: - return 0, fmt.Errorf("unknown whence: %v", whence) - case io.SeekStart: - case io.SeekCurrent: - offset += *cr.cPos - case io.SeekEnd: - return 0, fmt.Errorf("unsupported whence: %v", whence) - } - - if offset < 0 { - return 0, fmt.Errorf("invalid offset") - } - *cr.cPos = offset - return offset, nil -} - -func (cr *countReadSeeker) currentPos() int64 { - cr.mu.Lock() - defer cr.mu.Unlock() - - return *cr.cPos -} - -func decompressBlob(org *io.SectionReader, tmp *tempFiles, gzipHelperFunc GzipHelperFunc) (*io.SectionReader, error) { - if org.Size() < 4 { - return org, nil - } - src := make([]byte, 4) - if _, err := org.Read(src); err != nil && err != io.EOF { - return nil, err - } - var dR io.Reader - if bytes.Equal([]byte{0x1F, 0x8B, 0x08}, src[:3]) { - // gzip - var dgR io.ReadCloser - var err error - if gzipHelperFunc != nil { - dgR, err = gzipHelperFunc(io.NewSectionReader(org, 0, org.Size())) - } else { - dgR, err = gzip.NewReader(io.NewSectionReader(org, 0, org.Size())) - } - if err != nil { - return nil, err - } - defer dgR.Close() - dR = io.Reader(dgR) - } else if bytes.Equal([]byte{0x28, 0xb5, 0x2f, 0xfd}, src[:4]) { - // zstd - dzR, err := zstd.NewReader(io.NewSectionReader(org, 0, org.Size())) - if err != nil { - return nil, err - } - defer dzR.Close() - dR = io.Reader(dzR) - } else { - // uncompressed - return io.NewSectionReader(org, 0, org.Size()), nil - } - b, err := tmp.TempFile("", "uncompresseddata") - if err != nil { - return nil, err - } - if _, err := io.Copy(b, dR); err != nil { - return nil, err - } - return fileSectionReader(b) -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go deleted file mode 100644 index 6de78b02..00000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/errorutil/errors.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package errorutil - -import ( - "errors" - "fmt" - "strings" -) - -// Aggregate combines a list of errors into a single new error. -func Aggregate(errs []error) error { - switch len(errs) { - case 0: - return nil - case 1: - return errs[0] - default: - points := make([]string, len(errs)+1) - points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs)) - for i, err := range errs { - points[i+1] = fmt.Sprintf("* %s", err) - } - return errors.New(strings.Join(points, "\n\t")) - } -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go deleted file mode 100644 index 69373042..00000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go +++ /dev/null @@ -1,1232 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "bufio" - "bytes" - "compress/gzip" - "crypto/sha256" - "errors" - "fmt" - "hash" - "io" - "os" - "path" - "sort" - "strings" - "sync" - "time" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - digest "github.com/opencontainers/go-digest" - "github.com/vbatts/tar-split/archive/tar" -) - -// A Reader permits random access reads from a stargz file. -type Reader struct { - sr *io.SectionReader - toc *JTOC - tocDigest digest.Digest - - // m stores all non-chunk entries, keyed by name. - m map[string]*TOCEntry - - // chunks stores all TOCEntry values for regular files that - // are split up. For a file with a single chunk, it's only - // stored in m. - chunks map[string][]*TOCEntry - - decompressor Decompressor -} - -type openOpts struct { - tocOffset int64 - decompressors []Decompressor - telemetry *Telemetry -} - -// OpenOption is an option used during opening the layer -type OpenOption func(o *openOpts) error - -// WithTOCOffset option specifies the offset of TOC -func WithTOCOffset(tocOffset int64) OpenOption { - return func(o *openOpts) error { - o.tocOffset = tocOffset - return nil - } -} - -// WithDecompressors option specifies decompressors to use. -// Default is gzip-based decompressor. -func WithDecompressors(decompressors ...Decompressor) OpenOption { - return func(o *openOpts) error { - o.decompressors = decompressors - return nil - } -} - -// WithTelemetry option specifies the telemetry hooks -func WithTelemetry(telemetry *Telemetry) OpenOption { - return func(o *openOpts) error { - o.telemetry = telemetry - return nil - } -} - -// MeasureLatencyHook is a func which takes start time and records the diff -type MeasureLatencyHook func(time.Time) - -// Telemetry is a struct which defines telemetry hooks. By implementing these hooks you should be able to record -// the latency metrics of the respective steps of estargz open operation. To be used with estargz.OpenWithTelemetry(...) -type Telemetry struct { - GetFooterLatency MeasureLatencyHook // measure time to get stargz footer (in milliseconds) - GetTocLatency MeasureLatencyHook // measure time to GET TOC JSON (in milliseconds) - DeserializeTocLatency MeasureLatencyHook // measure time to deserialize TOC JSON (in milliseconds) -} - -// Open opens a stargz file for reading. -// The behavior is configurable using options. -// -// Note that each entry name is normalized as the path that is relative to root. -func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { - var opts openOpts - for _, o := range opt { - if err := o(&opts); err != nil { - return nil, err - } - } - - gzipCompressors := []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)} - decompressors := append(gzipCompressors, opts.decompressors...) - - // Determine the size to fetch. Try to fetch as many bytes as possible. - fetchSize := maxFooterSize(sr.Size(), decompressors...) - if maybeTocOffset := opts.tocOffset; maybeTocOffset > fetchSize { - if maybeTocOffset > sr.Size() { - return nil, fmt.Errorf("blob size %d is smaller than the toc offset", sr.Size()) - } - fetchSize = sr.Size() - maybeTocOffset - } - - start := time.Now() // before getting layer footer - footer := make([]byte, fetchSize) - if _, err := sr.ReadAt(footer, sr.Size()-fetchSize); err != nil { - return nil, fmt.Errorf("error reading footer: %v", err) - } - if opts.telemetry != nil && opts.telemetry.GetFooterLatency != nil { - opts.telemetry.GetFooterLatency(start) - } - - var allErr []error - var found bool - var r *Reader - for _, d := range decompressors { - fSize := d.FooterSize() - fOffset := positive(int64(len(footer)) - fSize) - maybeTocBytes := footer[:fOffset] - _, tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:]) - if err != nil { - allErr = append(allErr, err) - continue - } - if tocOffset >= 0 && tocSize <= 0 { - tocSize = sr.Size() - tocOffset - fSize - } - if tocOffset >= 0 && tocSize < int64(len(maybeTocBytes)) { - maybeTocBytes = maybeTocBytes[:tocSize] - } - r, err = parseTOC(d, sr, tocOffset, tocSize, maybeTocBytes, opts) - if err == nil { - found = true - break - } - allErr = append(allErr, err) - } - if !found { - return nil, errorutil.Aggregate(allErr) - } - if err := r.initFields(); err != nil { - return nil, fmt.Errorf("failed to initialize fields of entries: %v", err) - } - return r, nil -} - -// OpenFooter extracts and parses footer from the given blob. -// only supports gzip-based eStargz. -func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr error) { - if sr.Size() < FooterSize && sr.Size() < legacyFooterSize { - return 0, 0, fmt.Errorf("blob size %d is smaller than the footer size", sr.Size()) - } - var footer [FooterSize]byte - if _, err := sr.ReadAt(footer[:], sr.Size()-FooterSize); err != nil { - return 0, 0, fmt.Errorf("error reading footer: %v", err) - } - var allErr []error - for _, d := range []Decompressor{new(GzipDecompressor), new(LegacyGzipDecompressor)} { - fSize := d.FooterSize() - fOffset := positive(int64(len(footer)) - fSize) - _, tocOffset, _, err := d.ParseFooter(footer[fOffset:]) - if err == nil { - return tocOffset, fSize, err - } - allErr = append(allErr, err) - } - return 0, 0, errorutil.Aggregate(allErr) -} - -// initFields populates the Reader from r.toc after decoding it from -// JSON. -// -// Unexported fields are populated and TOCEntry fields that were -// implicit in the JSON are populated. -func (r *Reader) initFields() error { - r.m = make(map[string]*TOCEntry, len(r.toc.Entries)) - r.chunks = make(map[string][]*TOCEntry) - var lastPath string - uname := map[int]string{} - gname := map[int]string{} - var lastRegEnt *TOCEntry - var chunkTopIndex int - for i, ent := range r.toc.Entries { - ent.Name = cleanEntryName(ent.Name) - switch ent.Type { - case "reg", "chunk": - if ent.Offset != r.toc.Entries[chunkTopIndex].Offset { - chunkTopIndex = i - } - ent.chunkTopIndex = chunkTopIndex - } - if ent.Type == "reg" { - lastRegEnt = ent - } - if ent.Type == "chunk" { - ent.Name = lastPath - r.chunks[ent.Name] = append(r.chunks[ent.Name], ent) - if ent.ChunkSize == 0 && lastRegEnt != nil { - ent.ChunkSize = lastRegEnt.Size - ent.ChunkOffset - } - } else { - lastPath = ent.Name - - if ent.Uname != "" { - uname[ent.UID] = ent.Uname - } else { - ent.Uname = uname[ent.UID] - } - if ent.Gname != "" { - gname[ent.GID] = ent.Gname - } else { - ent.Gname = gname[ent.GID] - } - - ent.modTime, _ = time.Parse(time.RFC3339, ent.ModTime3339) - - if ent.Type == "dir" { - ent.NumLink++ // Parent dir links to this directory - } - r.m[ent.Name] = ent - } - if ent.Type == "reg" && ent.ChunkSize > 0 && ent.ChunkSize < ent.Size { - r.chunks[ent.Name] = make([]*TOCEntry, 0, ent.Size/ent.ChunkSize+1) - r.chunks[ent.Name] = append(r.chunks[ent.Name], ent) - } - if ent.ChunkSize == 0 && ent.Size != 0 { - ent.ChunkSize = ent.Size - } - } - - // Populate children, add implicit directories: - for _, ent := range r.toc.Entries { - if ent.Type == "chunk" { - continue - } - // add "foo/": - // add "foo" child to "" (creating "" if necessary) - // - // add "foo/bar/": - // add "bar" child to "foo" (creating "foo" if necessary) - // - // add "foo/bar.txt": - // add "bar.txt" child to "foo" (creating "foo" if necessary) - // - // add "a/b/c/d/e/f.txt": - // create "a/b/c/d/e" node - // add "f.txt" child to "e" - - name := ent.Name - pdirName := parentDir(name) - if name == pdirName { - // This entry and its parent are the same. - // Ignore this for avoiding infinite loop of the reference. - // The example case where this can occur is when tar contains the root - // directory itself (e.g. "./", "/"). - continue - } - pdir := r.getOrCreateDir(pdirName) - ent.NumLink++ // at least one name(ent.Name) references this entry. - if ent.Type == "hardlink" { - org, err := r.getSource(ent) - if err != nil { - return err - } - org.NumLink++ // original entry is referenced by this ent.Name. - ent = org - } - pdir.addChild(path.Base(name), ent) - } - - lastOffset := r.sr.Size() - for i := len(r.toc.Entries) - 1; i >= 0; i-- { - e := r.toc.Entries[i] - if e.isDataType() { - e.nextOffset = lastOffset - } - if e.Offset != 0 && e.InnerOffset == 0 { - lastOffset = e.Offset - } - } - - if len(r.m) == 0 { - r.m[""] = &TOCEntry{ - Name: "", - Type: "dir", - Mode: 0755, - NumLink: 1, - } - } - - return nil -} - -func (r *Reader) getSource(ent *TOCEntry) (_ *TOCEntry, err error) { - if ent.Type == "hardlink" { - org, ok := r.m[cleanEntryName(ent.LinkName)] - if !ok { - return nil, fmt.Errorf("%q is a hardlink but the linkname %q isn't found", ent.Name, ent.LinkName) - } - ent, err = r.getSource(org) - if err != nil { - return nil, err - } - } - return ent, nil -} - -func parentDir(p string) string { - dir, _ := path.Split(p) - return strings.TrimSuffix(dir, "/") -} - -func (r *Reader) getOrCreateDir(d string) *TOCEntry { - e, ok := r.m[d] - if !ok { - e = &TOCEntry{ - Name: d, - Type: "dir", - Mode: 0755, - NumLink: 2, // The directory itself(.) and the parent link to this directory. - } - r.m[d] = e - if d != "" { - pdir := r.getOrCreateDir(parentDir(d)) - pdir.addChild(path.Base(d), e) - } - } - return e -} - -func (r *Reader) TOCDigest() digest.Digest { - return r.tocDigest -} - -// VerifyTOC checks that the TOC JSON in the passed blob matches the -// passed digests and that the TOC JSON contains digests for all chunks -// contained in the blob. If the verification succceeds, this function -// returns TOCEntryVerifier which holds all chunk digests in the stargz blob. -func (r *Reader) VerifyTOC(tocDigest digest.Digest) (TOCEntryVerifier, error) { - // Verify the digest of TOC JSON - if r.tocDigest != tocDigest { - return nil, fmt.Errorf("invalid TOC JSON %q; want %q", r.tocDigest, tocDigest) - } - return r.Verifiers() -} - -// Verifiers returns TOCEntryVerifier of this chunk. Use VerifyTOC instead in most cases -// because this doesn't verify TOC. -func (r *Reader) Verifiers() (TOCEntryVerifier, error) { - chunkDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the chunk digest - regDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the reg file digest - var chunkDigestMapIncomplete bool - var regDigestMapIncomplete bool - var containsChunk bool - for _, e := range r.toc.Entries { - if e.Type != "reg" && e.Type != "chunk" { - continue - } - - // offset must be unique in stargz blob - _, dOK := chunkDigestMap[e.Offset] - _, rOK := regDigestMap[e.Offset] - if dOK || rOK { - return nil, fmt.Errorf("offset %d found twice", e.Offset) - } - - if e.Type == "reg" { - if e.Size == 0 { - continue // ignores empty file - } - - // record the digest of regular file payload - if e.Digest != "" { - d, err := digest.Parse(e.Digest) - if err != nil { - return nil, fmt.Errorf("failed to parse regular file digest %q: %w", e.Digest, err) - } - regDigestMap[e.Offset] = d - } else { - regDigestMapIncomplete = true - } - } else { - containsChunk = true // this layer contains "chunk" entries. - } - - // "reg" also can contain ChunkDigest (e.g. when "reg" is the first entry of - // chunked file) - if e.ChunkDigest != "" { - d, err := digest.Parse(e.ChunkDigest) - if err != nil { - return nil, fmt.Errorf("failed to parse chunk digest %q: %w", e.ChunkDigest, err) - } - chunkDigestMap[e.Offset] = d - } else { - chunkDigestMapIncomplete = true - } - } - - if chunkDigestMapIncomplete { - // Though some chunk digests are not found, if this layer doesn't contain - // "chunk"s and all digest of "reg" files are recorded, we can use them instead. - if !containsChunk && !regDigestMapIncomplete { - return &verifier{digestMap: regDigestMap}, nil - } - return nil, fmt.Errorf("some ChunkDigest not found in TOC JSON") - } - - return &verifier{digestMap: chunkDigestMap}, nil -} - -// verifier is an implementation of TOCEntryVerifier which holds verifiers keyed by -// offset of the chunk. -type verifier struct { - digestMap map[int64]digest.Digest - digestMapMu sync.Mutex -} - -// Verifier returns a content verifier specified by TOCEntry. -func (v *verifier) Verifier(ce *TOCEntry) (digest.Verifier, error) { - v.digestMapMu.Lock() - defer v.digestMapMu.Unlock() - d, ok := v.digestMap[ce.Offset] - if !ok { - return nil, fmt.Errorf("verifier for offset=%d,size=%d hasn't been registered", - ce.Offset, ce.ChunkSize) - } - return d.Verifier(), nil -} - -// ChunkEntryForOffset returns the TOCEntry containing the byte of the -// named file at the given offset within the file. -// Name must be absolute path or one that is relative to root. -func (r *Reader) ChunkEntryForOffset(name string, offset int64) (e *TOCEntry, ok bool) { - name = cleanEntryName(name) - e, ok = r.Lookup(name) - if !ok || !e.isDataType() { - return nil, false - } - ents := r.chunks[name] - if len(ents) < 2 { - if offset >= e.ChunkSize { - return nil, false - } - return e, true - } - i := sort.Search(len(ents), func(i int) bool { - e := ents[i] - return e.ChunkOffset >= offset || (offset > e.ChunkOffset && offset < e.ChunkOffset+e.ChunkSize) - }) - if i == len(ents) { - return nil, false - } - return ents[i], true -} - -// Lookup returns the Table of Contents entry for the given path. -// -// To get the root directory, use the empty string. -// Path must be absolute path or one that is relative to root. -func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) { - path = cleanEntryName(path) - if r == nil { - return - } - e, ok = r.m[path] - if ok && e.Type == "hardlink" { - var err error - e, err = r.getSource(e) - if err != nil { - return nil, false - } - } - return -} - -// OpenFile returns the reader of the specified file payload. -// -// Name must be absolute path or one that is relative to root. -func (r *Reader) OpenFile(name string) (*io.SectionReader, error) { - fr, err := r.newFileReader(name) - if err != nil { - return nil, err - } - return io.NewSectionReader(fr, 0, fr.size), nil -} - -func (r *Reader) newFileReader(name string) (*fileReader, error) { - name = cleanEntryName(name) - ent, ok := r.Lookup(name) - if !ok { - // TODO: come up with some error plan. This is lazy: - return nil, &os.PathError{ - Path: name, - Op: "OpenFile", - Err: os.ErrNotExist, - } - } - if ent.Type != "reg" { - return nil, &os.PathError{ - Path: name, - Op: "OpenFile", - Err: errors.New("not a regular file"), - } - } - return &fileReader{ - r: r, - size: ent.Size, - ents: r.getChunks(ent), - }, nil -} - -func (r *Reader) OpenFileWithPreReader(name string, preRead func(*TOCEntry, io.Reader) error) (*io.SectionReader, error) { - fr, err := r.newFileReader(name) - if err != nil { - return nil, err - } - fr.preRead = preRead - return io.NewSectionReader(fr, 0, fr.size), nil -} - -func (r *Reader) getChunks(ent *TOCEntry) []*TOCEntry { - if ents, ok := r.chunks[ent.Name]; ok { - return ents - } - return []*TOCEntry{ent} -} - -type fileReader struct { - r *Reader - size int64 - ents []*TOCEntry // 1 or more reg/chunk entries - preRead func(*TOCEntry, io.Reader) error -} - -func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) { - if off >= fr.size { - return 0, io.EOF - } - if off < 0 { - return 0, errors.New("invalid offset") - } - var i int - if len(fr.ents) > 1 { - i = sort.Search(len(fr.ents), func(i int) bool { - return fr.ents[i].ChunkOffset >= off - }) - if i == len(fr.ents) { - i = len(fr.ents) - 1 - } - } - ent := fr.ents[i] - if ent.ChunkOffset > off { - if i == 0 { - return 0, errors.New("internal error; first chunk offset is non-zero") - } - ent = fr.ents[i-1] - } - - // If ent is a chunk of a large file, adjust the ReadAt - // offset by the chunk's offset. - off -= ent.ChunkOffset - - finalEnt := fr.ents[len(fr.ents)-1] - compressedOff := ent.Offset - // compressedBytesRemain is the number of compressed bytes in this - // file remaining, over 1+ chunks. - compressedBytesRemain := finalEnt.NextOffset() - compressedOff - - sr := io.NewSectionReader(fr.r.sr, compressedOff, compressedBytesRemain) - - const maxRead = 2 << 20 - var bufSize = maxRead - if compressedBytesRemain < maxRead { - bufSize = int(compressedBytesRemain) - } - - br := bufio.NewReaderSize(sr, bufSize) - if _, err := br.Peek(bufSize); err != nil { - return 0, fmt.Errorf("fileReader.ReadAt.peek: %v", err) - } - - dr, err := fr.r.decompressor.Reader(br) - if err != nil { - return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err) - } - defer dr.Close() - - if fr.preRead == nil { - if n, err := io.CopyN(io.Discard, dr, ent.InnerOffset+off); n != ent.InnerOffset+off || err != nil { - return 0, fmt.Errorf("discard of %d bytes != %v, %v", ent.InnerOffset+off, n, err) - } - return io.ReadFull(dr, p) - } - - var retN int - var retErr error - var found bool - var nr int64 - for _, e := range fr.r.toc.Entries[ent.chunkTopIndex:] { - if !e.isDataType() { - continue - } - if e.Offset != fr.r.toc.Entries[ent.chunkTopIndex].Offset { - break - } - if in, err := io.CopyN(io.Discard, dr, e.InnerOffset-nr); err != nil || in != e.InnerOffset-nr { - return 0, fmt.Errorf("discard of remaining %d bytes != %v, %v", e.InnerOffset-nr, in, err) - } - nr = e.InnerOffset - if e == ent { - found = true - if n, err := io.CopyN(io.Discard, dr, off); n != off || err != nil { - return 0, fmt.Errorf("discard of offset %d bytes != %v, %v", off, n, err) - } - retN, retErr = io.ReadFull(dr, p) - nr += off + int64(retN) - continue - } - cr := &countReader{r: io.LimitReader(dr, e.ChunkSize)} - if err := fr.preRead(e, cr); err != nil { - return 0, fmt.Errorf("failed to pre read: %w", err) - } - nr += cr.n - } - if !found { - return 0, fmt.Errorf("fileReader.ReadAt: target entry not found") - } - return retN, retErr -} - -// A Writer writes stargz files. -// -// Use NewWriter to create a new Writer. -type Writer struct { - bw *bufio.Writer - cw *countWriter - toc *JTOC - diffHash hash.Hash // SHA-256 of uncompressed tar - - closed bool - gz io.WriteCloser - lastUsername map[int]string - lastGroupname map[int]string - compressor Compressor - - uncompressedCounter *countWriteFlusher - - // ChunkSize optionally controls the maximum number of bytes - // of data of a regular file that can be written in one gzip - // stream before a new gzip stream is started. - // Zero means to use a default, currently 4 MiB. - ChunkSize int - - // MinChunkSize optionally controls the minimum number of bytes - // of data must be written in one gzip stream before a new gzip - // NOTE: This adds a TOC property that stargz snapshotter < v0.13.0 doesn't understand. - MinChunkSize int - - needsOpenGzEntries map[string]struct{} -} - -// currentCompressionWriter writes to the current w.gz field, which can -// change throughout writing a tar entry. -// -// Additionally, it updates w's SHA-256 of the uncompressed bytes -// of the tar file. -type currentCompressionWriter struct{ w *Writer } - -func (ccw currentCompressionWriter) Write(p []byte) (int, error) { - ccw.w.diffHash.Write(p) - if ccw.w.gz == nil { - if err := ccw.w.condOpenGz(); err != nil { - return 0, err - } - } - return ccw.w.gz.Write(p) -} - -func (w *Writer) chunkSize() int { - if w.ChunkSize <= 0 { - return 4 << 20 - } - return w.ChunkSize -} - -// Unpack decompresses the given estargz blob and returns a ReadCloser of the tar blob. -// TOC JSON and footer are removed. -func Unpack(sr *io.SectionReader, c Decompressor) (io.ReadCloser, error) { - footerSize := c.FooterSize() - if sr.Size() < footerSize { - return nil, fmt.Errorf("blob is too small; %d < %d", sr.Size(), footerSize) - } - footerOffset := sr.Size() - footerSize - footer := make([]byte, footerSize) - if _, err := sr.ReadAt(footer, footerOffset); err != nil { - return nil, err - } - blobPayloadSize, _, _, err := c.ParseFooter(footer) - if err != nil { - return nil, fmt.Errorf("failed to parse footer: %w", err) - } - if blobPayloadSize < 0 { - blobPayloadSize = sr.Size() - } - return c.Reader(io.LimitReader(sr, blobPayloadSize)) -} - -// NewWriter returns a new stargz writer (gzip-based) writing to w. -// -// The writer must be closed to write its trailing table of contents. -func NewWriter(w io.Writer) *Writer { - return NewWriterLevel(w, gzip.BestCompression) -} - -// NewWriterLevel returns a new stargz writer (gzip-based) writing to w. -// The compression level is configurable. -// -// The writer must be closed to write its trailing table of contents. -func NewWriterLevel(w io.Writer, compressionLevel int) *Writer { - return NewWriterWithCompressor(w, NewGzipCompressorWithLevel(compressionLevel)) -} - -// NewWriterWithCompressor returns a new stargz writer writing to w. -// The compression method is configurable. -// -// The writer must be closed to write its trailing table of contents. -func NewWriterWithCompressor(w io.Writer, c Compressor) *Writer { - bw := bufio.NewWriter(w) - cw := &countWriter{w: bw} - return &Writer{ - bw: bw, - cw: cw, - toc: &JTOC{Version: 1}, - diffHash: sha256.New(), - compressor: c, - uncompressedCounter: &countWriteFlusher{}, - } -} - -// Close writes the stargz's table of contents and flushes all the -// buffers, returning any error. -func (w *Writer) Close() (digest.Digest, error) { - if w.closed { - return "", nil - } - defer func() { w.closed = true }() - - if err := w.closeGz(); err != nil { - return "", err - } - - // Write the TOC index and footer. - tocDigest, err := w.compressor.WriteTOCAndFooter(w.cw, w.cw.n, w.toc, w.diffHash) - if err != nil { - return "", err - } - if err := w.bw.Flush(); err != nil { - return "", err - } - - return tocDigest, nil -} - -func (w *Writer) closeGz() error { - if w.closed { - return errors.New("write on closed Writer") - } - if w.gz != nil { - if err := w.gz.Close(); err != nil { - return err - } - w.gz = nil - } - return nil -} - -func (w *Writer) flushGz() error { - if w.closed { - return errors.New("flush on closed Writer") - } - if w.gz != nil { - if f, ok := w.gz.(interface { - Flush() error - }); ok { - return f.Flush() - } - } - return nil -} - -// nameIfChanged returns name, unless it was the already the value of (*mp)[id], -// in which case it returns the empty string. -func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string { - if name == "" { - return "" - } - if *mp == nil { - *mp = make(map[int]string) - } - if (*mp)[id] == name { - return "" - } - (*mp)[id] = name - return name -} - -func (w *Writer) condOpenGz() (err error) { - if w.gz == nil { - w.gz, err = w.compressor.Writer(w.cw) - if w.gz != nil { - w.gz = w.uncompressedCounter.register(w.gz) - } - } - return -} - -// AppendTar reads the tar or tar.gz file from r and appends -// each of its contents to w. -// -// The input r can optionally be gzip compressed but the output will -// always be compressed by the specified compressor. -func (w *Writer) AppendTar(r io.Reader) error { - return w.appendTar(r, false) -} - -// AppendTarLossLess reads the tar or tar.gz file from r and appends -// each of its contents to w. -// -// The input r can optionally be gzip compressed but the output will -// always be compressed by the specified compressor. -// -// The difference of this func with AppendTar is that this writes -// the input tar stream into w without any modification (e.g. to header bytes). -// -// Note that if the input tar stream already contains TOC JSON, this returns -// error because w cannot overwrite the TOC JSON to the one generated by w without -// lossy modification. To avoid this error, if the input stream is known to be stargz/estargz, -// you shoud decompress it and remove TOC JSON in advance. -func (w *Writer) AppendTarLossLess(r io.Reader) error { - return w.appendTar(r, true) -} - -func (w *Writer) appendTar(r io.Reader, lossless bool) error { - var src io.Reader - br := bufio.NewReader(r) - if isGzip(br) { - zr, _ := gzip.NewReader(br) - src = zr - } else { - src = io.Reader(br) - } - dst := currentCompressionWriter{w} - var tw *tar.Writer - if !lossless { - tw = tar.NewWriter(dst) // use tar writer only when this isn't lossless mode. - } - tr := tar.NewReader(src) - if lossless { - tr.RawAccounting = true - } - prevOffset := w.cw.n - var prevOffsetUncompressed int64 - for { - h, err := tr.Next() - if err == io.EOF { - if lossless { - if remain := tr.RawBytes(); len(remain) > 0 { - // Collect the remaining null bytes. - // https://github.com/vbatts/tar-split/blob/80a436fd6164c557b131f7c59ed69bd81af69761/concept/main.go#L49-L53 - if _, err := dst.Write(remain); err != nil { - return err - } - } - } - break - } - if err != nil { - return fmt.Errorf("error reading from source tar: tar.Reader.Next: %v", err) - } - if cleanEntryName(h.Name) == TOCTarName { - // It is possible for a layer to be "stargzified" twice during the - // distribution lifecycle. So we reserve "TOCTarName" here to avoid - // duplicated entries in the resulting layer. - if lossless { - // We cannot handle this in lossless way. - return fmt.Errorf("existing TOC JSON is not allowed; decompress layer before append") - } - continue - } - - xattrs := make(map[string][]byte) - const xattrPAXRecordsPrefix = "SCHILY.xattr." - if h.PAXRecords != nil { - for k, v := range h.PAXRecords { - if strings.HasPrefix(k, xattrPAXRecordsPrefix) { - xattrs[k[len(xattrPAXRecordsPrefix):]] = []byte(v) - } - } - } - ent := &TOCEntry{ - Name: h.Name, - Mode: h.Mode, - UID: h.Uid, - GID: h.Gid, - Uname: w.nameIfChanged(&w.lastUsername, h.Uid, h.Uname), - Gname: w.nameIfChanged(&w.lastGroupname, h.Gid, h.Gname), - ModTime3339: formatModtime(h.ModTime), - Xattrs: xattrs, - } - if err := w.condOpenGz(); err != nil { - return err - } - if tw != nil { - if err := tw.WriteHeader(h); err != nil { - return err - } - } else { - if _, err := dst.Write(tr.RawBytes()); err != nil { - return err - } - } - switch h.Typeflag { - case tar.TypeLink: - ent.Type = "hardlink" - ent.LinkName = h.Linkname - case tar.TypeSymlink: - ent.Type = "symlink" - ent.LinkName = h.Linkname - case tar.TypeDir: - ent.Type = "dir" - case tar.TypeReg: - ent.Type = "reg" - ent.Size = h.Size - case tar.TypeChar: - ent.Type = "char" - ent.DevMajor = int(h.Devmajor) - ent.DevMinor = int(h.Devminor) - case tar.TypeBlock: - ent.Type = "block" - ent.DevMajor = int(h.Devmajor) - ent.DevMinor = int(h.Devminor) - case tar.TypeFifo: - ent.Type = "fifo" - default: - return fmt.Errorf("unsupported input tar entry %q", h.Typeflag) - } - - // We need to keep a reference to the TOC entry for regular files, so that we - // can fill the digest later. - var regFileEntry *TOCEntry - var payloadDigest digest.Digester - if h.Typeflag == tar.TypeReg { - regFileEntry = ent - payloadDigest = digest.Canonical.Digester() - } - - if h.Typeflag == tar.TypeReg && ent.Size > 0 { - var written int64 - totalSize := ent.Size // save it before we destroy ent - tee := io.TeeReader(tr, payloadDigest.Hash()) - for written < totalSize { - chunkSize := int64(w.chunkSize()) - remain := totalSize - written - if remain < chunkSize { - chunkSize = remain - } else { - ent.ChunkSize = chunkSize - } - - // We flush the underlying compression writer here to correctly calculate "w.cw.n". - if err := w.flushGz(); err != nil { - return err - } - if w.needsOpenGz(ent) || w.cw.n-prevOffset >= int64(w.MinChunkSize) { - if err := w.closeGz(); err != nil { - return err - } - ent.Offset = w.cw.n - prevOffset = ent.Offset - prevOffsetUncompressed = w.uncompressedCounter.n - } else { - ent.Offset = prevOffset - ent.InnerOffset = w.uncompressedCounter.n - prevOffsetUncompressed - } - - ent.ChunkOffset = written - chunkDigest := digest.Canonical.Digester() - - if err := w.condOpenGz(); err != nil { - return err - } - - teeChunk := io.TeeReader(tee, chunkDigest.Hash()) - var out io.Writer - if tw != nil { - out = tw - } else { - out = dst - } - if _, err := io.CopyN(out, teeChunk, chunkSize); err != nil { - return fmt.Errorf("error copying %q: %v", h.Name, err) - } - ent.ChunkDigest = chunkDigest.Digest().String() - w.toc.Entries = append(w.toc.Entries, ent) - written += chunkSize - ent = &TOCEntry{ - Name: h.Name, - Type: "chunk", - } - } - } else { - w.toc.Entries = append(w.toc.Entries, ent) - } - if payloadDigest != nil { - regFileEntry.Digest = payloadDigest.Digest().String() - } - if tw != nil { - if err := tw.Flush(); err != nil { - return err - } - } - } - remainDest := io.Discard - if lossless { - remainDest = dst // Preserve the remaining bytes in lossless mode - } - _, err := io.Copy(remainDest, src) - return err -} - -func (w *Writer) needsOpenGz(ent *TOCEntry) bool { - if ent.Type != "reg" { - return false - } - if w.needsOpenGzEntries == nil { - return false - } - _, ok := w.needsOpenGzEntries[ent.Name] - return ok -} - -// DiffID returns the SHA-256 of the uncompressed tar bytes. -// It is only valid to call DiffID after Close. -func (w *Writer) DiffID() string { - return fmt.Sprintf("sha256:%x", w.diffHash.Sum(nil)) -} - -func maxFooterSize(blobSize int64, decompressors ...Decompressor) (res int64) { - for _, d := range decompressors { - if s := d.FooterSize(); res < s && s <= blobSize { - res = s - } - } - return -} - -func parseTOC(d Decompressor, sr *io.SectionReader, tocOff, tocSize int64, tocBytes []byte, opts openOpts) (*Reader, error) { - if tocOff < 0 { - // This means that TOC isn't contained in the blob. - // We pass nil reader to ParseTOC and expect that ParseTOC acquire TOC from - // the external location. - start := time.Now() - toc, tocDgst, err := d.ParseTOC(nil) - if err != nil { - return nil, err - } - if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil { - opts.telemetry.GetTocLatency(start) - } - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil - } - if len(tocBytes) > 0 { - start := time.Now() - toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) - if err == nil { - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil - } - } - - start := time.Now() - tocBytes = make([]byte, tocSize) - if _, err := sr.ReadAt(tocBytes, tocOff); err != nil { - return nil, fmt.Errorf("error reading %d byte TOC targz: %v", len(tocBytes), err) - } - if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil { - opts.telemetry.GetTocLatency(start) - } - start = time.Now() - toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes)) - if err != nil { - return nil, err - } - if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil { - opts.telemetry.DeserializeTocLatency(start) - } - return &Reader{ - sr: sr, - toc: toc, - tocDigest: tocDgst, - decompressor: d, - }, nil -} - -func formatModtime(t time.Time) string { - if t.IsZero() || t.Unix() == 0 { - return "" - } - return t.UTC().Round(time.Second).Format(time.RFC3339) -} - -func cleanEntryName(name string) string { - // Use path.Clean to consistently deal with path separators across platforms. - return strings.TrimPrefix(path.Clean("/"+name), "/") -} - -// countWriter counts how many bytes have been written to its wrapped -// io.Writer. -type countWriter struct { - w io.Writer - n int64 -} - -func (cw *countWriter) Write(p []byte) (n int, err error) { - n, err = cw.w.Write(p) - cw.n += int64(n) - return -} - -type countWriteFlusher struct { - io.WriteCloser - n int64 -} - -func (wc *countWriteFlusher) register(w io.WriteCloser) io.WriteCloser { - wc.WriteCloser = w - return wc -} - -func (wc *countWriteFlusher) Write(p []byte) (n int, err error) { - n, err = wc.WriteCloser.Write(p) - wc.n += int64(n) - return -} - -func (wc *countWriteFlusher) Flush() error { - if f, ok := wc.WriteCloser.(interface { - Flush() error - }); ok { - return f.Flush() - } - return nil -} - -func (wc *countWriteFlusher) Close() error { - err := wc.WriteCloser.Close() - wc.WriteCloser = nil - return err -} - -// isGzip reports whether br is positioned right before an upcoming gzip stream. -// It does not consume any bytes from br. -func isGzip(br *bufio.Reader) bool { - const ( - gzipID1 = 0x1f - gzipID2 = 0x8b - gzipDeflate = 8 - ) - peek, _ := br.Peek(3) - return len(peek) >= 3 && peek[0] == gzipID1 && peek[1] == gzipID2 && peek[2] == gzipDeflate -} - -func positive(n int64) int64 { - if n < 0 { - return 0 - } - return n -} - -type countReader struct { - r io.Reader - n int64 -} - -func (cr *countReader) Read(p []byte) (n int, err error) { - n, err = cr.r.Read(p) - cr.n += int64(n) - return -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go deleted file mode 100644 index 88fa13b1..00000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go +++ /dev/null @@ -1,237 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "encoding/binary" - "encoding/json" - "fmt" - "hash" - "io" - "strconv" - - digest "github.com/opencontainers/go-digest" -) - -type gzipCompression struct { - *GzipCompressor - *GzipDecompressor -} - -func newGzipCompressionWithLevel(level int) Compression { - return &gzipCompression{ - &GzipCompressor{level}, - &GzipDecompressor{}, - } -} - -func NewGzipCompressor() *GzipCompressor { - return &GzipCompressor{gzip.BestCompression} -} - -func NewGzipCompressorWithLevel(level int) *GzipCompressor { - return &GzipCompressor{level} -} - -type GzipCompressor struct { - compressionLevel int -} - -func (gc *GzipCompressor) Writer(w io.Writer) (WriteFlushCloser, error) { - return gzip.NewWriterLevel(w, gc.compressionLevel) -} - -func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) { - tocJSON, err := json.MarshalIndent(toc, "", "\t") - if err != nil { - return "", err - } - gz, _ := gzip.NewWriterLevel(w, gc.compressionLevel) - gw := io.Writer(gz) - if diffHash != nil { - gw = io.MultiWriter(gz, diffHash) - } - tw := tar.NewWriter(gw) - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: TOCTarName, - Size: int64(len(tocJSON)), - }); err != nil { - return "", err - } - if _, err := tw.Write(tocJSON); err != nil { - return "", err - } - - if err := tw.Close(); err != nil { - return "", err - } - if err := gz.Close(); err != nil { - return "", err - } - if _, err := w.Write(gzipFooterBytes(off)); err != nil { - return "", err - } - return digest.FromBytes(tocJSON), nil -} - -// gzipFooterBytes returns the 51 bytes footer. -func gzipFooterBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, FooterSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes - - // Extra header indicating the offset of TOCJSON - // https://tools.ietf.org/html/rfc1952#section-2.3.1.1 - header := make([]byte, 4) - header[0], header[1] = 'S', 'G' - subfield := fmt.Sprintf("%016xSTARGZ", tocOff) - binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952 - gz.Extra = append(header, []byte(subfield)...) - gz.Close() - if buf.Len() != FooterSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize)) - } - return buf.Bytes() -} - -type GzipDecompressor struct{} - -func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) -} - -func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - return parseTOCEStargz(r) -} - -func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { - if len(p) != FooterSize { - return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, 0, 0, err - } - defer zr.Close() - extra := zr.Extra - si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] - if si1 != 'S' || si2 != 'G' { - return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) - } - if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) { - return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ")) - } - if string(subfield[16:]) != "STARGZ" { - return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield") - } - tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err) - } - return tocOffset, tocOffset, 0, nil -} - -func (gz *GzipDecompressor) FooterSize() int64 { - return FooterSize -} - -func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) { - return decompressTOCEStargz(r) -} - -type LegacyGzipDecompressor struct{} - -func (gz *LegacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) -} - -func (gz *LegacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - return parseTOCEStargz(r) -} - -func (gz *LegacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { - if len(p) != legacyFooterSize { - return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p)) - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to get footer gzip reader: %w", err) - } - defer zr.Close() - extra := zr.Extra - if len(extra) != 16+len("STARGZ") { - return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size") - } - if string(extra[16:]) != "STARGZ" { - return 0, 0, 0, fmt.Errorf("legacy: magic string STARGZ not found") - } - tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64) - if err != nil { - return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err) - } - return tocOffset, tocOffset, 0, nil -} - -func (gz *LegacyGzipDecompressor) FooterSize() int64 { - return legacyFooterSize -} - -func (gz *LegacyGzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) { - return decompressTOCEStargz(r) -} - -func parseTOCEStargz(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) { - tr, err := decompressTOCEStargz(r) - if err != nil { - return nil, "", err - } - dgstr := digest.Canonical.Digester() - toc = new(JTOC) - if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil { - return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err) - } - if err := tr.Close(); err != nil { - return nil, "", err - } - return toc, dgstr.Digest(), nil -} - -func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) { - zr, err := gzip.NewReader(r) - if err != nil { - return nil, fmt.Errorf("malformed TOC gzip header: %v", err) - } - zr.Multistream(false) - tr := tar.NewReader(zr) - h, err := tr.Next() - if err != nil { - return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err) - } - if h.Name != TOCTarName { - return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName) - } - return readCloser{tr, zr.Close}, nil -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go deleted file mode 100644 index ff165e09..00000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go +++ /dev/null @@ -1,2403 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "crypto/rand" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "math/big" - "os" - "path/filepath" - "reflect" - "sort" - "strings" - "time" - - "github.com/containerd/stargz-snapshotter/estargz/errorutil" - "github.com/klauspost/compress/zstd" - digest "github.com/opencontainers/go-digest" -) - -// TestingController is Compression with some helper methods necessary for testing. -type TestingController interface { - Compression - TestStreams(t TestingT, b []byte, streams []int64) - DiffIDOf(TestingT, []byte) string - String() string -} - -// TestingT is the minimal set of testing.T required to run the -// tests defined in CompressionTestSuite. This interface exists to prevent -// leaking the testing package from being exposed outside tests. -type TestingT interface { - Errorf(format string, args ...any) - FailNow() - Failed() bool - Fatal(args ...any) - Fatalf(format string, args ...any) - Logf(format string, args ...any) - Parallel() -} - -// Runner allows running subtests of TestingT. This exists instead of adding -// a Run method to TestingT interface because the Run implementation of -// testing.T would not satisfy the interface. -type Runner func(t TestingT, name string, fn func(t TestingT)) - -type TestRunner struct { - TestingT - Runner Runner -} - -func (r *TestRunner) Run(name string, run func(*TestRunner)) { - r.Runner(r.TestingT, name, func(t TestingT) { - run(&TestRunner{TestingT: t, Runner: r.Runner}) - }) -} - -// CompressionTestSuite tests this pkg with controllers can build valid eStargz blobs and parse them. -func CompressionTestSuite(t *TestRunner, controllers ...TestingControllerFactory) { - t.Run("testBuild", func(t *TestRunner) { t.Parallel(); testBuild(t, controllers...) }) - t.Run("testDigestAndVerify", func(t *TestRunner) { - t.Parallel() - testDigestAndVerify(t, controllers...) - }) - t.Run("testWriteAndOpen", func(t *TestRunner) { t.Parallel(); testWriteAndOpen(t, controllers...) }) -} - -type TestingControllerFactory func() TestingController - -const ( - uncompressedType int = iota - gzipType - zstdType -) - -var srcCompressions = []int{ - uncompressedType, - gzipType, - zstdType, -} - -var allowedPrefix = [4]string{"", "./", "/", "../"} - -// testBuild tests the resulting stargz blob built by this pkg has the same -// contents as the normal stargz blob. -func testBuild(t *TestRunner, controllers ...TestingControllerFactory) { - tests := []struct { - name string - chunkSize int - minChunkSize []int - in []tarEntry - }{ - { - name: "regfiles and directories", - chunkSize: 4, - in: tarOf( - file("foo", "test1"), - dir("foo2/"), - file("foo2/bar", "test2", xAttr(map[string]string{"test": "sample"})), - ), - }, - { - name: "empty files", - chunkSize: 4, - in: tarOf( - file("foo", "tttttt"), - file("foo_empty", ""), - file("foo2", "tttttt"), - file("foo_empty2", ""), - file("foo3", "tttttt"), - file("foo_empty3", ""), - file("foo4", "tttttt"), - file("foo_empty4", ""), - file("foo5", "tttttt"), - file("foo_empty5", ""), - file("foo6", "tttttt"), - ), - }, - { - name: "various files", - chunkSize: 4, - minChunkSize: []int{0, 64000}, - in: tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo1.txt", "a"), - file("bar/foo2.txt", "b"), - file("foo3.txt", "c"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "testbartestbar", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - { - name: "no contents", - chunkSize: 4, - in: tarOf( - file("baz.txt", ""), - symlink("barlink", "test/bar.txt"), - dir("test/"), - dir("dev/"), - blockdev("dev/testblock", 3, 4), - fifo("dev/testfifo"), - chardev("dev/testchar1", 5, 6), - file("test/bar.txt", "", xAttr(map[string]string{"test2": "sample2"})), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - chardev("dev/testchar2", 1, 2), - ), - }, - } - for _, tt := range tests { - if len(tt.minChunkSize) == 0 { - tt.minChunkSize = []int{0} - } - for _, srcCompression := range srcCompressions { - srcCompression := srcCompression - for _, newCL := range controllers { - newCL := newCL - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, prefix := range allowedPrefix { - prefix := prefix - for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *TestRunner) { - tarBlob := buildTar(t, tt.in, prefix, srcTarFormat) - // Test divideEntries() - entries, err := sortEntries(tarBlob, nil, nil) // identical order - if err != nil { - t.Fatalf("failed to parse tar: %v", err) - } - var merged []*entry - for _, part := range divideEntries(entries, 4) { - merged = append(merged, part...) - } - if !reflect.DeepEqual(entries, merged) { - for _, e := range entries { - t.Logf("Original: %v", e.header) - } - for _, e := range merged { - t.Logf("Merged: %v", e.header) - } - t.Errorf("divided entries couldn't be merged") - return - } - - // Prepare sample data - cl1 := newCL() - wantBuf := new(bytes.Buffer) - sw := NewWriterWithCompressor(wantBuf, cl1) - sw.MinChunkSize = minChunkSize - sw.ChunkSize = tt.chunkSize - if err := sw.AppendTar(tarBlob); err != nil { - t.Fatalf("failed to append tar to want stargz: %v", err) - } - if _, err := sw.Close(); err != nil { - t.Fatalf("failed to prepare want stargz: %v", err) - } - wantData := wantBuf.Bytes() - want, err := Open(io.NewSectionReader( - bytes.NewReader(wantData), 0, int64(len(wantData))), - WithDecompressors(cl1), - ) - if err != nil { - t.Fatalf("failed to parse the want stargz: %v", err) - } - - // Prepare testing data - var opts []Option - if minChunkSize > 0 { - opts = append(opts, WithMinChunkSize(minChunkSize)) - } - cl2 := newCL() - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - append(opts, WithChunkSize(tt.chunkSize), WithCompression(cl2))...) - if err != nil { - t.Fatalf("failed to build stargz: %v", err) - } - defer rc.Close() - gotBuf := new(bytes.Buffer) - if _, err := io.Copy(gotBuf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - gotData := gotBuf.Bytes() - got, err := Open(io.NewSectionReader( - bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))), - WithDecompressors(cl2), - ) - if err != nil { - t.Fatalf("failed to parse the got stargz: %v", err) - } - - // Check DiffID is properly calculated - rc.Close() - diffID := rc.DiffID() - wantDiffID := cl2.DiffIDOf(t, gotData) - if diffID.String() != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - // Compare as stargz - if !isSameVersion(t, cl1, wantData, cl2, gotData) { - t.Errorf("built stargz hasn't same json") - return - } - if !isSameEntries(t, want, got) { - t.Errorf("built stargz isn't same as the original") - return - } - - // Compare as tar.gz - if !isSameTarGz(t, cl1, wantData, cl2, gotData) { - t.Errorf("built stargz isn't same tar.gz") - return - } - }) - } - } - } - } - } - } -} - -func isSameTarGz(t TestingT, cla TestingController, a []byte, clb TestingController, b []byte) bool { - aGz, err := cla.Reader(bytes.NewReader(a)) - if err != nil { - t.Fatalf("failed to read A") - } - defer aGz.Close() - bGz, err := clb.Reader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("failed to read B") - } - defer bGz.Close() - - // Same as tar's Next() method but ignores landmarks and TOCJSON file - next := func(r *tar.Reader) (h *tar.Header, err error) { - for { - if h, err = r.Next(); err != nil { - return - } - if h.Name != PrefetchLandmark && - h.Name != NoPrefetchLandmark && - h.Name != TOCTarName { - return - } - } - } - - aTar := tar.NewReader(aGz) - bTar := tar.NewReader(bGz) - for { - // Fetch and parse next header. - aH, aErr := next(aTar) - bH, bErr := next(bTar) - if aErr != nil || bErr != nil { - if aErr == io.EOF && bErr == io.EOF { - break - } - t.Fatalf("Failed to parse tar file: A: %v, B: %v", aErr, bErr) - } - if !reflect.DeepEqual(aH, bH) { - t.Logf("different header (A = %v; B = %v)", aH, bH) - return false - - } - aFile, err := io.ReadAll(aTar) - if err != nil { - t.Fatal("failed to read tar payload of A") - } - bFile, err := io.ReadAll(bTar) - if err != nil { - t.Fatal("failed to read tar payload of B") - } - if !bytes.Equal(aFile, bFile) { - t.Logf("different tar payload (A = %q; B = %q)", string(a), string(b)) - return false - } - } - - return true -} - -func isSameVersion(t TestingT, cla TestingController, a []byte, clb TestingController, b []byte) bool { - aJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a))), cla) - if err != nil { - t.Fatalf("failed to parse A: %v", err) - } - bJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), clb) - if err != nil { - t.Fatalf("failed to parse B: %v", err) - } - t.Logf("A: TOCJSON: %v", dumpTOCJSON(t, aJTOC)) - t.Logf("B: TOCJSON: %v", dumpTOCJSON(t, bJTOC)) - return aJTOC.Version == bJTOC.Version -} - -func isSameEntries(t TestingT, a, b *Reader) bool { - aroot, ok := a.Lookup("") - if !ok { - t.Fatalf("failed to get root of A") - } - broot, ok := b.Lookup("") - if !ok { - t.Fatalf("failed to get root of B") - } - aEntry := stargzEntry{aroot, a} - bEntry := stargzEntry{broot, b} - return contains(t, aEntry, bEntry) && contains(t, bEntry, aEntry) -} - -func compressBlob(t TestingT, src *io.SectionReader, srcCompression int) *io.SectionReader { - buf := new(bytes.Buffer) - var w io.WriteCloser - var err error - switch srcCompression { - case gzipType: - w = gzip.NewWriter(buf) - case zstdType: - w, err = zstd.NewWriter(buf) - if err != nil { - t.Fatalf("failed to init zstd writer: %v", err) - } - default: - return src - } - src.Seek(0, io.SeekStart) - if _, err := io.Copy(w, src); err != nil { - t.Fatalf("failed to compress source") - } - if err := w.Close(); err != nil { - t.Fatalf("failed to finalize compress source") - } - data := buf.Bytes() - return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) - -} - -type stargzEntry struct { - e *TOCEntry - r *Reader -} - -// contains checks if all child entries in "b" are also contained in "a". -// This function also checks if the files/chunks contain the same contents among "a" and "b". -func contains(t TestingT, a, b stargzEntry) bool { - ae, ar := a.e, a.r - be, br := b.e, b.r - t.Logf("Comparing: %q vs %q", ae.Name, be.Name) - if !equalEntry(ae, be) { - t.Logf("%q != %q: entry: a: %v, b: %v", ae.Name, be.Name, ae, be) - return false - } - if ae.Type == "dir" { - t.Logf("Directory: %q vs %q: %v vs %v", ae.Name, be.Name, - allChildrenName(ae), allChildrenName(be)) - iscontain := true - ae.ForeachChild(func(aBaseName string, aChild *TOCEntry) bool { - // Walk through all files on this stargz file. - - if aChild.Name == PrefetchLandmark || - aChild.Name == NoPrefetchLandmark { - return true // Ignore landmarks - } - - // Ignore a TOCEntry of "./" (formated as "" by stargz lib) on root directory - // because this points to the root directory itself. - if aChild.Name == "" && ae.Name == "" { - return true - } - - bChild, ok := be.LookupChild(aBaseName) - if !ok { - t.Logf("%q (base: %q): not found in b: %v", - ae.Name, aBaseName, allChildrenName(be)) - iscontain = false - return false - } - - childcontain := contains(t, stargzEntry{aChild, a.r}, stargzEntry{bChild, b.r}) - if !childcontain { - t.Logf("%q != %q: non-equal dir", ae.Name, be.Name) - iscontain = false - return false - } - return true - }) - return iscontain - } else if ae.Type == "reg" { - af, err := ar.OpenFile(ae.Name) - if err != nil { - t.Fatalf("failed to open file %q on A: %v", ae.Name, err) - } - bf, err := br.OpenFile(be.Name) - if err != nil { - t.Fatalf("failed to open file %q on B: %v", be.Name, err) - } - - var nr int64 - for nr < ae.Size { - abytes, anext, aok := readOffset(t, af, nr, a) - bbytes, bnext, bok := readOffset(t, bf, nr, b) - if !aok && !bok { - break - } else if !aok || !bok || anext != bnext { - t.Logf("%q != %q (offset=%d): chunk existence a=%v vs b=%v, anext=%v vs bnext=%v", - ae.Name, be.Name, nr, aok, bok, anext, bnext) - return false - } - nr = anext - if !bytes.Equal(abytes, bbytes) { - t.Logf("%q != %q: different contents %v vs %v", - ae.Name, be.Name, string(abytes), string(bbytes)) - return false - } - } - return true - } - - return true -} - -func allChildrenName(e *TOCEntry) (children []string) { - e.ForeachChild(func(baseName string, _ *TOCEntry) bool { - children = append(children, baseName) - return true - }) - return -} - -func equalEntry(a, b *TOCEntry) bool { - // Here, we selectively compare fileds that we are interested in. - return a.Name == b.Name && - a.Type == b.Type && - a.Size == b.Size && - a.ModTime3339 == b.ModTime3339 && - a.Stat().ModTime().Equal(b.Stat().ModTime()) && // modTime time.Time - a.LinkName == b.LinkName && - a.Mode == b.Mode && - a.UID == b.UID && - a.GID == b.GID && - a.Uname == b.Uname && - a.Gname == b.Gname && - (a.Offset >= 0) == (b.Offset >= 0) && - (a.NextOffset() > 0) == (b.NextOffset() > 0) && - a.DevMajor == b.DevMajor && - a.DevMinor == b.DevMinor && - a.NumLink == b.NumLink && - reflect.DeepEqual(a.Xattrs, b.Xattrs) && - // chunk-related infomations aren't compared in this function. - // ChunkOffset int64 `json:"chunkOffset,omitempty"` - // ChunkSize int64 `json:"chunkSize,omitempty"` - // children map[string]*TOCEntry - a.Digest == b.Digest -} - -func readOffset(t TestingT, r *io.SectionReader, offset int64, e stargzEntry) ([]byte, int64, bool) { - ce, ok := e.r.ChunkEntryForOffset(e.e.Name, offset) - if !ok { - return nil, 0, false - } - data := make([]byte, ce.ChunkSize) - t.Logf("Offset: %v, NextOffset: %v", ce.Offset, ce.NextOffset()) - n, err := r.ReadAt(data, ce.ChunkOffset) - if err != nil { - t.Fatalf("failed to read file payload of %q (offset:%d,size:%d): %v", - e.e.Name, ce.ChunkOffset, ce.ChunkSize, err) - } - if int64(n) != ce.ChunkSize { - t.Fatalf("unexpected copied data size %d; want %d", - n, ce.ChunkSize) - } - return data[:n], offset + ce.ChunkSize, true -} - -func dumpTOCJSON(t TestingT, tocJSON *JTOC) string { - jtocData, err := json.Marshal(*tocJSON) - if err != nil { - t.Fatalf("failed to marshal TOC JSON: %v", err) - } - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, bytes.NewReader(jtocData)); err != nil { - t.Fatalf("failed to read toc json blob: %v", err) - } - return buf.String() -} - -const chunkSize = 3 - -type check func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) - -// testDigestAndVerify runs specified checks against sample stargz blobs. -func testDigestAndVerify(t *TestRunner, controllers ...TestingControllerFactory) { - tests := []struct { - name string - tarInit func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) - checks []check - minChunkSize []int - }{ - { - name: "no-regfile", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - dir("test/"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - dir("test2/"), // modified - ), allowedPrefix[0])), - }, - }, - { - name: "small-files", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "bbb", dgstMap), - ) - }, - minChunkSize: []int{0, 64000}, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", ""), - file("foo.txt", "M"), // modified - dir("test/"), - file("test/bar.txt", "bbb"), - ), allowedPrefix[0])), - // checkVerifyInvalidTOCEntryFail("foo.txt"), // TODO - checkVerifyBrokenContentFail("foo.txt"), - }, - }, - { - name: "big-files", - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", "bazbazbazMMMbazbazbaz"), // modified - file("foo.txt", "a"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - { - name: "with-non-regfiles", - minChunkSize: []int{0, 64000}, - tarInit: func(t TestingT, dgstMap map[string]digest.Digest) (blob []tarEntry) { - return tarOf( - regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap), - regDigest(t, "foo.txt", "a", dgstMap), - regDigest(t, "bar/foo2.txt", "b", dgstMap), - regDigest(t, "foo3.txt", "c", dgstMap), - symlink("barlink", "test/bar.txt"), - dir("test/"), - regDigest(t, "test/bar.txt", "testbartestbar", dgstMap), - dir("test2/"), - link("test2/bazlink", "baz.txt"), - ) - }, - checks: []check{ - checkStargzTOC, - checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTar(t, tarOf( - file("baz.txt", "bazbazbazbazbazbazbaz"), - file("foo.txt", "a"), - file("bar/foo2.txt", "b"), - file("foo3.txt", "c"), - symlink("barlink", "test/bar.txt"), - dir("test/"), - file("test/bar.txt", "testbartestbar"), - dir("test2/"), - link("test2/bazlink", "foo.txt"), // modified - ), allowedPrefix[0])), - checkVerifyInvalidTOCEntryFail("test/bar.txt"), - checkVerifyBrokenContentFail("test/bar.txt"), - }, - }, - } - - for _, tt := range tests { - if len(tt.minChunkSize) == 0 { - tt.minChunkSize = []int{0} - } - for _, srcCompression := range srcCompressions { - srcCompression := srcCompression - for _, newCL := range controllers { - newCL := newCL - for _, prefix := range allowedPrefix { - prefix := prefix - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, minChunkSize := range tt.minChunkSize { - minChunkSize := minChunkSize - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *TestRunner) { - // Get original tar file and chunk digests - dgstMap := make(map[string]digest.Digest) - tarBlob := buildTar(t, tt.tarInit(t, dgstMap), prefix, srcTarFormat) - - cl := newCL() - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - WithChunkSize(chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - tocDigest := rc.TOCDigest() - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - newStargz := buf.Bytes() - // NoPrefetchLandmark is added during `Bulid`, which is expected behaviour. - dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents}) - - for _, check := range tt.checks { - check(t, newStargz, tocDigest, dgstMap, cl, newCL) - } - }) - } - } - } - } - } - } -} - -// checkStargzTOC checks the TOC JSON of the passed stargz has the expected -// digest and contains valid chunks. It walks all entries in the stargz and -// checks all chunk digests stored to the TOC JSON match the actual contents. -func checkStargzTOC(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - digestMapTOC, err := listDigests(io.NewSectionReader( - bytes.NewReader(sgzData), 0, int64(len(sgzData))), - controller, - ) - if err != nil { - t.Fatalf("failed to list digest: %v", err) - } - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := controller.Reader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - // Check the digest of TOC JSON based on the actual contents - // It's sure that TOC JSON exists in this archive because - // Open succeeded. - dgstr := digest.Canonical.Digester() - if _, err := io.Copy(dgstr.Hash(), tr); err != nil { - t.Fatalf("failed to calculate digest of TOC JSON: %v", - err) - } - if dgstr.Digest() != tocDigest { - t.Errorf("invalid TOC JSON %q; want %q", tocDigest, dgstr.Digest()) - } - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - // Get the original digest to make sure the file contents are kept unchanged - // from the original tar, during the whole conversion steps. - id := chunkID(h.Name, n, ce.ChunkSize) - want, ok := dgstMap[id] - if !ok { - t.Errorf("Unexpected chunk %q(offset=%d,size=%d): %v", - h.Name, n, ce.ChunkSize, dgstMap) - return - } - found[id] = true - - // Check the file contents - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to calculate digest of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if want != dgstr.Digest() { - t.Errorf("Invalid contents in converted stargz %q: %q; want %q", - h.Name, dgstr.Digest(), want) - return - } - - // Check the digest stored in TOC JSON - dgstTOC, ok := digestMapTOC[ce.Offset] - if !ok { - t.Errorf("digest of %q(offset=%d,size=%d,chunkOffset=%d) isn't registered", - h.Name, ce.Offset, ce.ChunkSize, ce.ChunkOffset) - } - if want != dgstTOC { - t.Errorf("Invalid digest in TOCEntry %q: %q; want %q", - h.Name, dgstTOC, want) - return - } - - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyTOC checks the verification works for the TOC JSON of the passed -// stargz. It walks all entries in the stargz and checks the verifications for -// all chunks work. -func checkVerifyTOC(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Errorf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Errorf("failed to verify stargz: %v", err) - return - } - - found := make(map[string]bool) - for id := range dgstMap { - found[id] = false - } - zr, err := controller.Reader(bytes.NewReader(sgzData)) - if err != nil { - t.Fatalf("failed to decompress converted stargz: %v", err) - } - defer zr.Close() - tr := tar.NewReader(zr) - for { - h, err := tr.Next() - if err != nil { - if err != io.EOF { - t.Errorf("failed to read tar entry: %v", err) - return - } - break - } - if h.Name == TOCTarName { - continue - } - if _, ok := sgz.Lookup(h.Name); !ok { - t.Errorf("lost stargz entry %q in the converted TOC", h.Name) - return - } - var n int64 - for n < h.Size { - ce, ok := sgz.ChunkEntryForOffset(h.Name, n) - if !ok { - t.Errorf("lost chunk %q(offset=%d) in the converted TOC", - h.Name, n) - return - } - - v, err := ev.Verifier(ce) - if err != nil { - t.Errorf("failed to get verifier for %q(offset=%d)", h.Name, n) - } - - found[chunkID(h.Name, n, ce.ChunkSize)] = true - - // Check the file contents - if _, err := io.CopyN(v, tr, ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - h.Name, n, ce.ChunkSize) - } - if !v.Verified() { - t.Errorf("Invalid contents in converted stargz %q (should be succeeded)", - h.Name) - return - } - n += ce.ChunkSize - } - } - - for id, ok := range found { - if !ok { - t.Errorf("required chunk %q not found in the converted stargz: %v", id, found) - } - } -} - -// checkVerifyInvalidTOCEntryFail checks if misconfigured TOC JSON can be -// detected during the verification and the verification returns an error. -func checkVerifyInvalidTOCEntryFail(filename string) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - funcs := map[string]rewriteFunc{ - "lost digest in a entry": func(t TestingT, toc *JTOC, sgz *io.SectionReader) { - var found bool - for _, e := range toc.Entries { - if cleanEntryName(e.Name) == filename { - if e.Type != "reg" && e.Type != "chunk" { - t.Fatalf("entry %q to break must be regfile or chunk", filename) - } - if e.ChunkDigest == "" { - t.Fatalf("entry %q is already invalid", filename) - } - e.ChunkDigest = "" - found = true - } - } - if !found { - t.Fatalf("rewrite target not found") - } - }, - "duplicated entry offset": func(t TestingT, toc *JTOC, sgz *io.SectionReader) { - var ( - sampleEntry *TOCEntry - targetEntry *TOCEntry - ) - for _, e := range toc.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if cleanEntryName(e.Name) == filename { - targetEntry = e - } else { - sampleEntry = e - } - } - } - if sampleEntry == nil { - t.Fatalf("TOC must contain at least one regfile or chunk entry other than the rewrite target") - return - } - if targetEntry == nil { - t.Fatalf("rewrite target not found") - return - } - targetEntry.Offset = sampleEntry.Offset - }, - } - - for name, rFunc := range funcs { - t.Run(name, func(t *TestRunner) { - newSgz, newTocDigest := rewriteTOCJSON(t, io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), rFunc, controller) - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, newSgz); err != nil { - t.Fatalf("failed to get converted stargz") - } - isgz := buf.Bytes() - - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(isgz), 0, int64(len(isgz))), - WithDecompressors(controller), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(newTocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - }) - } - } -} - -// checkVerifyInvalidStargzFail checks if the verification detects that the -// given stargz file doesn't match to the expected digest and returns error. -func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - cl := newController() - rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - mStargz := buf.Bytes() - - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(mStargz), 0, int64(len(mStargz))), - WithDecompressors(cl), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - _, err = sgz.VerifyTOC(tocDigest) - if err == nil { - t.Errorf("must fail for invalid TOC") - return - } - } -} - -// checkVerifyBrokenContentFail checks if the verifier detects broken contents -// that doesn't match to the expected digest and returns error. -func checkVerifyBrokenContentFail(filename string) check { - return func(t *TestRunner, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) { - // Parse stargz file - sgz, err := Open( - io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))), - WithDecompressors(controller), - ) - if err != nil { - t.Fatalf("failed to parse converted stargz: %v", err) - return - } - ev, err := sgz.VerifyTOC(tocDigest) - if err != nil { - t.Fatalf("failed to verify stargz: %v", err) - return - } - - // Open the target file - sr, err := sgz.OpenFile(filename) - if err != nil { - t.Fatalf("failed to open file %q", filename) - } - ce, ok := sgz.ChunkEntryForOffset(filename, 0) - if !ok { - t.Fatalf("lost chunk %q(offset=%d) in the converted TOC", filename, 0) - return - } - if ce.ChunkSize == 0 { - t.Fatalf("file mustn't be empty") - return - } - data := make([]byte, ce.ChunkSize) - if _, err := sr.ReadAt(data, ce.ChunkOffset); err != nil { - t.Errorf("failed to get data of a chunk of %q(offset=%q)", - filename, ce.ChunkOffset) - } - - // Check the broken chunk (must fail) - v, err := ev.Verifier(ce) - if err != nil { - t.Fatalf("failed to get verifier for %q", filename) - } - broken := append([]byte{^data[0]}, data[1:]...) - if _, err := io.CopyN(v, bytes.NewReader(broken), ce.ChunkSize); err != nil { - t.Fatalf("failed to get chunk of %q (offset=%d,size=%d)", - filename, ce.ChunkOffset, ce.ChunkSize) - } - if v.Verified() { - t.Errorf("verification must fail for broken file chunk %q(org:%q,broken:%q)", - filename, data, broken) - } - } -} - -func chunkID(name string, offset, size int64) string { - return fmt.Sprintf("%s-%d-%d", cleanEntryName(name), offset, size) -} - -type rewriteFunc func(t TestingT, toc *JTOC, sgz *io.SectionReader) - -func rewriteTOCJSON(t TestingT, sgz *io.SectionReader, rewrite rewriteFunc, controller TestingController) (newSgz io.Reader, tocDigest digest.Digest) { - decodedJTOC, jtocOffset, err := parseStargz(sgz, controller) - if err != nil { - t.Fatalf("failed to extract TOC JSON: %v", err) - } - - rewrite(t, decodedJTOC, sgz) - - tocFooter, tocDigest, err := tocAndFooter(controller, decodedJTOC, jtocOffset) - if err != nil { - t.Fatalf("failed to create toc and footer: %v", err) - } - - // Reconstruct stargz file with the modified TOC JSON - if _, err := sgz.Seek(0, io.SeekStart); err != nil { - t.Fatalf("failed to reset the seek position of stargz: %v", err) - } - return io.MultiReader( - io.LimitReader(sgz, jtocOffset), // Original stargz (before TOC JSON) - tocFooter, // Rewritten TOC and footer - ), tocDigest -} - -func listDigests(sgz *io.SectionReader, controller TestingController) (map[int64]digest.Digest, error) { - decodedJTOC, _, err := parseStargz(sgz, controller) - if err != nil { - return nil, err - } - digestMap := make(map[int64]digest.Digest) - for _, e := range decodedJTOC.Entries { - if e.Type == "reg" || e.Type == "chunk" { - if e.Type == "reg" && e.Size == 0 { - continue // ignores empty file - } - if e.ChunkDigest == "" { - return nil, fmt.Errorf("ChunkDigest of %q(off=%d) not found in TOC JSON", - e.Name, e.Offset) - } - d, err := digest.Parse(e.ChunkDigest) - if err != nil { - return nil, err - } - digestMap[e.Offset] = d - } - } - return digestMap, nil -} - -func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJTOC *JTOC, jtocOffset int64, err error) { - fSize := controller.FooterSize() - footer := make([]byte, fSize) - if _, err := sgz.ReadAt(footer, sgz.Size()-fSize); err != nil { - return nil, 0, fmt.Errorf("error reading footer: %w", err) - } - _, tocOffset, _, err := controller.ParseFooter(footer[positive(int64(len(footer))-fSize):]) - if err != nil { - return nil, 0, fmt.Errorf("failed to parse footer: %w", err) - } - - // Decode the TOC JSON - var tocReader io.Reader - if tocOffset >= 0 { - tocReader = io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-fSize) - } - decodedJTOC, _, err = controller.ParseTOC(tocReader) - if err != nil { - return nil, 0, fmt.Errorf("failed to parse TOC: %w", err) - } - return decodedJTOC, tocOffset, nil -} - -func testWriteAndOpen(t *TestRunner, controllers ...TestingControllerFactory) { - const content = "Some contents" - invalidUtf8 := "\xff\xfe\xfd" - - xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8} - sampleOwner := owner{uid: 50, gid: 100} - - data64KB := randomContents(64000) - - tests := []struct { - name string - chunkSize int - minChunkSize int - in []tarEntry - want []stargzCheck - wantNumGz int // expected number of streams - - wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz - wantFailOnLossLess bool - wantTOCVersion int // default = 1 - }{ - { - name: "empty", - in: tarOf(), - wantNumGz: 2, // (empty tar) + TOC + footer - want: checks( - numTOCEntries(0), - ), - }, - { - name: "1dir_1empty_file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", ""), - ), - wantNumGz: 3, // dir, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", 0), - entryHasChildren("foo", "bar.txt"), - hasFileDigest("foo/bar.txt", digestFor("")), - ), - }, - { - name: "1dir_1file", - in: tarOf( - dir("foo/"), - file("foo/bar.txt", content, xAttrFile), - ), - wantNumGz: 4, // var dir, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(2), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - hasFileDigest("foo/bar.txt", digestFor(content)), - hasFileContentsRange("foo/bar.txt", 0, content), - hasFileContentsRange("foo/bar.txt", 1, content[1:]), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar.txt"), - hasFileXattrs("foo/bar.txt", "foo", "bar"), - hasFileXattrs("foo/bar.txt", "invalid-utf8", invalidUtf8), - ), - }, - { - name: "2meta_2file", - in: tarOf( - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(3), - hasDir("bar/"), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - entryHasChildren("", "bar", "foo"), - entryHasChildren("foo", "bar.txt"), - hasChunkEntries("foo/bar.txt", 1), - hasEntryOwner("bar/", sampleOwner), - hasEntryOwner("foo/", sampleOwner), - hasEntryOwner("foo/bar.txt", sampleOwner), - ), - }, - { - name: "3dir", - in: tarOf( - dir("bar/"), - dir("foo/"), - dir("foo/bar/"), - ), - wantNumGz: 3, // 3 dirs, TOC, footer - want: checks( - hasDirLinkCount("bar/", 2), - hasDirLinkCount("foo/", 3), - hasDirLinkCount("foo/bar/", 2), - ), - }, - { - name: "symlink", - in: tarOf( - dir("foo/"), - symlink("foo/bar", "../../x"), - ), - wantNumGz: 3, // metas + TOC + footer - want: checks( - numTOCEntries(2), - hasSymlink("foo/bar", "../../x"), - entryHasChildren("", "foo"), - entryHasChildren("foo", "bar"), - ), - }, - { - name: "chunked_file", - chunkSize: 4, - in: tarOf( - dir("foo/"), - file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"), - ), - wantNumGz: 9, // dir + big.txt(6 chunks) + TOC + footer - want: checks( - numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file - hasDir("foo/"), - hasFileLen("foo/big.txt", len("This is such a big file")), - hasFileDigest("foo/big.txt", digestFor("This is such a big file")), - hasFileContentsRange("foo/big.txt", 0, "This is such a big file"), - hasFileContentsRange("foo/big.txt", 1, "his is such a big file"), - hasFileContentsRange("foo/big.txt", 2, "is is such a big file"), - hasFileContentsRange("foo/big.txt", 3, "s is such a big file"), - hasFileContentsRange("foo/big.txt", 4, " is such a big file"), - hasFileContentsRange("foo/big.txt", 5, "is such a big file"), - hasFileContentsRange("foo/big.txt", 6, "s such a big file"), - hasFileContentsRange("foo/big.txt", 7, " such a big file"), - hasFileContentsRange("foo/big.txt", 8, "such a big file"), - hasFileContentsRange("foo/big.txt", 9, "uch a big file"), - hasFileContentsRange("foo/big.txt", 10, "ch a big file"), - hasFileContentsRange("foo/big.txt", 11, "h a big file"), - hasFileContentsRange("foo/big.txt", 12, " a big file"), - hasFileContentsRange("foo/big.txt", len("This is such a big file")-1, ""), - hasChunkEntries("foo/big.txt", 6), - ), - }, - { - name: "recursive", - in: tarOf( - dir("/", sampleOwner), - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - ), - wantNumGz: 4, // dirs, bar.txt alone, TOC, footer - want: checks( - maxDepth(2), // 0: root directory, 1: "foo/", 2: "bar.txt" - ), - }, - { - name: "block_char_fifo", - in: tarOf( - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "b", - Typeflag: tar.TypeBlock, - Devmajor: 123, - Devminor: 456, - Format: format, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "c", - Typeflag: tar.TypeChar, - Devmajor: 111, - Devminor: 222, - Format: format, - }) - }), - tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Name: prefix + "f", - Typeflag: tar.TypeFifo, - Format: format, - }) - }), - ), - wantNumGz: 3, - want: checks( - lookupMatch("b", &TOCEntry{Name: "b", Type: "block", DevMajor: 123, DevMinor: 456, NumLink: 1}), - lookupMatch("c", &TOCEntry{Name: "c", Type: "char", DevMajor: 111, DevMinor: 222, NumLink: 1}), - lookupMatch("f", &TOCEntry{Name: "f", Type: "fifo", NumLink: 1}), - ), - }, - { - name: "modes", - in: tarOf( - dir("foo1/", 0755|os.ModeDir|os.ModeSetgid), - file("foo1/bar1", content, 0700|os.ModeSetuid), - file("foo1/bar2", content, 0755|os.ModeSetgid), - dir("foo2/", 0755|os.ModeDir|os.ModeSticky), - file("foo2/bar3", content, 0755|os.ModeSticky), - dir("foo3/", 0755|os.ModeDir), - file("foo3/bar4", content, os.FileMode(0700)), - file("foo3/bar5", content, os.FileMode(0755)), - ), - wantNumGz: 8, // dir, bar1 alone, bar2 alone + dir, bar3 alone + dir, bar4 alone, bar5 alone, TOC, footer - want: checks( - hasMode("foo1/", 0755|os.ModeDir|os.ModeSetgid), - hasMode("foo1/bar1", 0700|os.ModeSetuid), - hasMode("foo1/bar2", 0755|os.ModeSetgid), - hasMode("foo2/", 0755|os.ModeDir|os.ModeSticky), - hasMode("foo2/bar3", 0755|os.ModeSticky), - hasMode("foo3/", 0755|os.ModeDir), - hasMode("foo3/bar4", os.FileMode(0700)), - hasMode("foo3/bar5", os.FileMode(0755)), - ), - }, - { - name: "lossy", - in: tarOf( - dir("bar/", sampleOwner), - dir("foo/", sampleOwner), - file("foo/bar.txt", content, sampleOwner), - file(TOCTarName, "dummy"), // ignored by the writer. (lossless write returns error) - ), - wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer - want: checks( - numTOCEntries(3), - hasDir("bar/"), - hasDir("foo/"), - hasFileLen("foo/bar.txt", len(content)), - entryHasChildren("", "bar", "foo"), - entryHasChildren("foo", "bar.txt"), - hasChunkEntries("foo/bar.txt", 1), - hasEntryOwner("bar/", sampleOwner), - hasEntryOwner("foo/", sampleOwner), - hasEntryOwner("foo/bar.txt", sampleOwner), - ), - wantFailOnLossLess: true, - }, - { - name: "hardlink should be replaced to the destination entry", - in: tarOf( - dir("foo/"), - file("foo/foo1", "test"), - link("foolink", "foo/foo1"), - ), - wantNumGz: 4, // dir, foo1 + link, TOC, footer - want: checks( - mustSameEntry("foo/foo1", "foolink"), - ), - }, - { - name: "several_files_in_chunk", - minChunkSize: 8000, - in: tarOf( - dir("foo/"), - file("foo/foo1", data64KB), - file("foo2", "bb"), - file("foo22", "ccc"), - dir("bar/"), - file("bar/bar.txt", "aaa"), - file("foo3", data64KB), - ), - // NOTE: we assume that the compressed "data64KB" is still larger than 8KB - wantNumGz: 4, // dir+foo1, foo2+foo22+dir+bar.txt+foo3, TOC, footer - want: checks( - numTOCEntries(7), // dir, foo1, foo2, foo22, dir, bar.txt, foo3 - hasDir("foo/"), - hasDir("bar/"), - hasFileLen("foo/foo1", len(data64KB)), - hasFileLen("foo2", len("bb")), - hasFileLen("foo22", len("ccc")), - hasFileLen("bar/bar.txt", len("aaa")), - hasFileLen("foo3", len(data64KB)), - hasFileDigest("foo/foo1", digestFor(data64KB)), - hasFileDigest("foo2", digestFor("bb")), - hasFileDigest("foo22", digestFor("ccc")), - hasFileDigest("bar/bar.txt", digestFor("aaa")), - hasFileDigest("foo3", digestFor(data64KB)), - hasFileContentsWithPreRead("foo22", 0, "ccc", chunkInfo{"foo2", "bb"}, chunkInfo{"bar/bar.txt", "aaa"}, chunkInfo{"foo3", data64KB}), - hasFileContentsRange("foo/foo1", 0, data64KB), - hasFileContentsRange("foo2", 0, "bb"), - hasFileContentsRange("foo2", 1, "b"), - hasFileContentsRange("foo22", 0, "ccc"), - hasFileContentsRange("foo22", 1, "cc"), - hasFileContentsRange("foo22", 2, "c"), - hasFileContentsRange("bar/bar.txt", 0, "aaa"), - hasFileContentsRange("bar/bar.txt", 1, "aa"), - hasFileContentsRange("bar/bar.txt", 2, "a"), - hasFileContentsRange("foo3", 0, data64KB), - hasFileContentsRange("foo3", 1, data64KB[1:]), - hasFileContentsRange("foo3", 2, data64KB[2:]), - hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]), - ), - }, - { - name: "several_files_in_chunk_chunked", - minChunkSize: 8000, - chunkSize: 32000, - in: tarOf( - dir("foo/"), - file("foo/foo1", data64KB), - file("foo2", "bb"), - dir("bar/"), - file("foo3", data64KB), - ), - // NOTE: we assume that the compressed chunk of "data64KB" is still larger than 8KB - wantNumGz: 6, // dir+foo1(1), foo1(2), foo2+dir+foo3(1), foo3(2), TOC, footer - want: checks( - numTOCEntries(7), // dir, foo1(2 chunks), foo2, dir, foo3(2 chunks) - hasDir("foo/"), - hasDir("bar/"), - hasFileLen("foo/foo1", len(data64KB)), - hasFileLen("foo2", len("bb")), - hasFileLen("foo3", len(data64KB)), - hasFileDigest("foo/foo1", digestFor(data64KB)), - hasFileDigest("foo2", digestFor("bb")), - hasFileDigest("foo3", digestFor(data64KB)), - hasFileContentsWithPreRead("foo2", 0, "bb", chunkInfo{"foo3", data64KB[:32000]}), - hasFileContentsRange("foo/foo1", 0, data64KB), - hasFileContentsRange("foo/foo1", 1, data64KB[1:]), - hasFileContentsRange("foo/foo1", 2, data64KB[2:]), - hasFileContentsRange("foo/foo1", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo/foo1", len(data64KB)-1, data64KB[len(data64KB)-1:]), - hasFileContentsRange("foo2", 0, "bb"), - hasFileContentsRange("foo2", 1, "b"), - hasFileContentsRange("foo3", 0, data64KB), - hasFileContentsRange("foo3", 1, data64KB[1:]), - hasFileContentsRange("foo3", 2, data64KB[2:]), - hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]), - hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]), - ), - }, - } - - for _, tt := range tests { - for _, newCL := range controllers { - newCL := newCL - for _, prefix := range allowedPrefix { - prefix := prefix - for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { - srcTarFormat := srcTarFormat - for _, lossless := range []bool{true, false} { - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *TestRunner) { - var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat) - origTarDgstr := digest.Canonical.Digester() - tr = io.TeeReader(tr, origTarDgstr.Hash()) - var stargzBuf bytes.Buffer - cl1 := newCL() - w := NewWriterWithCompressor(&stargzBuf, cl1) - w.ChunkSize = tt.chunkSize - w.MinChunkSize = tt.minChunkSize - if lossless { - err := w.AppendTarLossLess(tr) - if tt.wantFailOnLossLess { - if err != nil { - return // expected to fail - } - t.Fatalf("Append wanted to fail on lossless") - } - if err != nil { - t.Fatalf("Append(lossless): %v", err) - } - } else { - if err := w.AppendTar(tr); err != nil { - t.Fatalf("Append: %v", err) - } - } - if _, err := w.Close(); err != nil { - t.Fatalf("Writer.Close: %v", err) - } - b := stargzBuf.Bytes() - - if lossless { - // Check if the result blob reserves original tar metadata - rc, err := Unpack(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), cl1) - if err != nil { - t.Errorf("failed to decompress blob: %v", err) - return - } - defer rc.Close() - resultDgstr := digest.Canonical.Digester() - if _, err := io.Copy(resultDgstr.Hash(), rc); err != nil { - t.Errorf("failed to read result decompressed blob: %v", err) - return - } - if resultDgstr.Digest() != origTarDgstr.Digest() { - t.Errorf("lossy compression occurred: digest=%v; want %v", - resultDgstr.Digest(), origTarDgstr.Digest()) - return - } - } - - diffID := w.DiffID() - wantDiffID := cl1.DiffIDOf(t, b) - if diffID != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } - - telemetry, checkCalled := newCalledTelemetry() - sr := io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))) - r, err := Open( - sr, - WithDecompressors(cl1), - WithTelemetry(telemetry), - ) - if err != nil { - t.Fatalf("stargz.Open: %v", err) - } - if _, ok := r.Lookup(""); !ok { - t.Fatalf("failed to lookup rootdir: %v", err) - } - wantTOCVersion := 1 - if tt.wantTOCVersion > 0 { - wantTOCVersion = tt.wantTOCVersion - } - if r.toc.Version != wantTOCVersion { - t.Fatalf("invalid TOC Version %d; wanted %d", r.toc.Version, wantTOCVersion) - } - - footerSize := cl1.FooterSize() - footerOffset := sr.Size() - footerSize - footer := make([]byte, footerSize) - if _, err := sr.ReadAt(footer, footerOffset); err != nil { - t.Errorf("failed to read footer: %v", err) - } - _, tocOffset, _, err := cl1.ParseFooter(footer) - if err != nil { - t.Errorf("failed to parse footer: %v", err) - } - if err := checkCalled(tocOffset >= 0); err != nil { - t.Errorf("telemetry failure: %v", err) - } - - wantNumGz := tt.wantNumGz - if lossless && tt.wantNumGzLossLess > 0 { - wantNumGz = tt.wantNumGzLossLess - } - streamOffsets := []int64{0} - prevOffset := int64(-1) - streams := 0 - for _, e := range r.toc.Entries { - if e.Offset > prevOffset { - streamOffsets = append(streamOffsets, e.Offset) - prevOffset = e.Offset - streams++ - } - } - streams++ // TOC - if tocOffset >= 0 { - // toc is in the blob - streamOffsets = append(streamOffsets, tocOffset) - } - streams++ // footer - streamOffsets = append(streamOffsets, footerOffset) - if streams != wantNumGz { - t.Errorf("number of streams in TOC = %d; want %d", streams, wantNumGz) - } - - t.Logf("testing streams: %+v", streamOffsets) - cl1.TestStreams(t, b, streamOffsets) - - for _, want := range tt.want { - want.check(t, r) - } - }) - } - } - } - } - } -} - -type chunkInfo struct { - name string - data string -} - -func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) error) { - var getFooterLatencyCalled bool - var getTocLatencyCalled bool - var deserializeTocLatencyCalled bool - return &Telemetry{ - func(time.Time) { getFooterLatencyCalled = true }, - func(time.Time) { getTocLatencyCalled = true }, - func(time.Time) { deserializeTocLatencyCalled = true }, - }, func(needsGetTOC bool) error { - var allErr []error - if !getFooterLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called")) - } - if needsGetTOC { - if !getTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called")) - } - } - if !deserializeTocLatencyCalled { - allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called")) - } - return errorutil.Aggregate(allErr) - } -} - -func digestFor(content string) string { - sum := sha256.Sum256([]byte(content)) - return fmt.Sprintf("sha256:%x", sum) -} - -type numTOCEntries int - -func (n numTOCEntries) check(t TestingT, r *Reader) { - if r.toc == nil { - t.Fatal("nil TOC") - } - if got, want := len(r.toc.Entries), int(n); got != want { - t.Errorf("got %d TOC entries; want %d", got, want) - } - t.Logf("got TOC entries:") - for i, ent := range r.toc.Entries { - entj, _ := json.Marshal(ent) - t.Logf(" [%d]: %s\n", i, entj) - } - if t.Failed() { - t.FailNow() - } -} - -func checks(s ...stargzCheck) []stargzCheck { return s } - -type stargzCheck interface { - check(t TestingT, r *Reader) -} - -type stargzCheckFn func(TestingT, *Reader) - -func (f stargzCheckFn) check(t TestingT, r *Reader) { f(t, r) } - -func maxDepth(max int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - e, ok := r.Lookup("") - if !ok { - t.Fatal("root directory not found") - } - d, err := getMaxDepth(t, e, 0, 10*max) - if err != nil { - t.Errorf("failed to get max depth (wanted %d): %v", max, err) - return - } - if d != max { - t.Errorf("invalid depth %d; want %d", d, max) - return - } - }) -} - -func getMaxDepth(t TestingT, e *TOCEntry, current, limit int) (max int, rErr error) { - if current > limit { - return -1, fmt.Errorf("walkMaxDepth: exceeds limit: current:%d > limit:%d", - current, limit) - } - max = current - e.ForeachChild(func(baseName string, ent *TOCEntry) bool { - t.Logf("%q(basename:%q) is child of %q\n", ent.Name, baseName, e.Name) - d, err := getMaxDepth(t, ent, current+1, limit) - if err != nil { - rErr = err - return false - } - if d > max { - max = d - } - return true - }) - return -} - -func hasFileLen(file string, wantLen int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } else if ent.Size != int64(wantLen) { - t.Errorf("file size of %q = %d; want %d", file, ent.Size, wantLen) - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileXattrs(file, name, value string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "reg" { - t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type) - } - if ent.Xattrs == nil { - t.Errorf("file %q has no xattrs", file) - return - } - valueFound, found := ent.Xattrs[name] - if !found { - t.Errorf("file %q has no xattr %q", file, name) - return - } - if string(valueFound) != value { - t.Errorf("file %q has xattr %q with value %q instead of %q", file, name, valueFound, value) - } - - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasFileDigest(file string, digest string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("didn't find TOCEntry for file %q", file) - } - if ent.Digest != digest { - t.Fatalf("Digest(%q) = %q, want %q", file, ent.Digest, digest) - } - }) -} - -func hasFileContentsWithPreRead(file string, offset int, want string, extra ...chunkInfo) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - extraMap := make(map[string]chunkInfo) - for _, e := range extra { - extraMap[e.name] = e - } - var extraNames []string - for n := range extraMap { - extraNames = append(extraNames, n) - } - f, err := r.OpenFileWithPreReader(file, func(e *TOCEntry, cr io.Reader) error { - t.Logf("On %q: got preread of %q", file, e.Name) - ex, ok := extraMap[e.Name] - if !ok { - t.Fatalf("fail on %q: unexpected entry %q: %+v, %+v", file, e.Name, e, extraNames) - } - got, err := io.ReadAll(cr) - if err != nil { - t.Fatalf("fail on %q: failed to read %q: %v", file, e.Name, err) - } - if ex.data != string(got) { - t.Fatalf("fail on %q: unexpected contents of %q: len=%d; want=%d", file, e.Name, len(got), len(ex.data)) - } - delete(extraMap, e.Name) - return nil - }) - if err != nil { - t.Fatal(err) - } - got := make([]byte, len(want)) - n, err := f.ReadAt(got, int64(offset)) - if err != nil { - t.Fatalf("ReadAt(len %d, offset %d, size %d) = %v, %v", len(got), offset, f.Size(), n, err) - } - if string(got) != want { - t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want))) - } - if len(extraMap) != 0 { - var exNames []string - for _, ex := range extraMap { - exNames = append(exNames, ex.name) - } - t.Fatalf("fail on %q: some entries aren't read: %+v", file, exNames) - } - }) -} - -func hasFileContentsRange(file string, offset int, want string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - f, err := r.OpenFile(file) - if err != nil { - t.Fatal(err) - } - got := make([]byte, len(want)) - n, err := f.ReadAt(got, int64(offset)) - if err != nil { - t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err) - } - if string(got) != want { - t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want))) - } - }) -} - -func hasChunkEntries(file string, wantChunks int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(file) - if !ok { - t.Fatalf("no file for %q", file) - } - if ent.Type != "reg" { - t.Fatalf("file %q has unexpected type %q; want reg", file, ent.Type) - } - chunks := r.getChunks(ent) - if len(chunks) != wantChunks { - t.Errorf("len(r.getChunks(%q)) = %d; want %d", file, len(chunks), wantChunks) - return - } - f := chunks[0] - - var gotChunks []*TOCEntry - var last *TOCEntry - for off := int64(0); off < f.Size; off++ { - e, ok := r.ChunkEntryForOffset(file, off) - if !ok { - t.Errorf("no ChunkEntryForOffset at %d", off) - return - } - if last != e { - gotChunks = append(gotChunks, e) - last = e - } - } - if !reflect.DeepEqual(chunks, gotChunks) { - t.Errorf("gotChunks=%d, want=%d; contents mismatch", len(gotChunks), wantChunks) - } - - // And verify the NextOffset - for i := 0; i < len(gotChunks)-1; i++ { - ci := gotChunks[i] - cnext := gotChunks[i+1] - if ci.NextOffset() != cnext.Offset { - t.Errorf("chunk %d NextOffset %d != next chunk's Offset of %d", i, ci.NextOffset(), cnext.Offset) - } - } - }) -} - -func entryHasChildren(dir string, want ...string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - want := append([]string(nil), want...) - var got []string - ent, ok := r.Lookup(dir) - if !ok { - t.Fatalf("didn't find TOCEntry for dir node %q", dir) - } - for baseName := range ent.children { - got = append(got, baseName) - } - sort.Strings(got) - sort.Strings(want) - if !reflect.DeepEqual(got, want) { - t.Errorf("children of %q = %q; want %q", dir, got, want) - } - }) -} - -func hasDir(file string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasDirLinkCount(file string, count int) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Type != "dir" { - t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type) - return - } - if ent.NumLink != count { - t.Errorf("link count of %q = %d; want %d", file, ent.NumLink, count) - } - return - } - } - t.Errorf("directory %q not found", file) - }) -} - -func hasMode(file string, mode os.FileMode) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == cleanEntryName(file) { - if ent.Stat().Mode() != mode { - t.Errorf("invalid mode: got %v; want %v", ent.Stat().Mode(), mode) - return - } - return - } - } - t.Errorf("file %q not found", file) - }) -} - -func hasSymlink(file, target string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - for _, ent := range r.toc.Entries { - if ent.Name == file { - if ent.Type != "symlink" { - t.Errorf("file type of %q is %q; want \"symlink\"", file, ent.Type) - } else if ent.LinkName != target { - t.Errorf("link target of symlink %q is %q; want %q", file, ent.LinkName, target) - } - return - } - } - t.Errorf("symlink %q not found", file) - }) -} - -func lookupMatch(name string, want *TOCEntry) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - e, ok := r.Lookup(name) - if !ok { - t.Fatalf("failed to Lookup entry %q", name) - } - if !reflect.DeepEqual(e, want) { - t.Errorf("entry %q mismatch.\n got: %+v\nwant: %+v\n", name, e, want) - } - - }) -} - -func hasEntryOwner(entry string, owner owner) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - ent, ok := r.Lookup(strings.TrimSuffix(entry, "/")) - if !ok { - t.Errorf("entry %q not found", entry) - return - } - if ent.UID != owner.uid || ent.GID != owner.gid { - t.Errorf("entry %q has invalid owner (uid:%d, gid:%d) instead of (uid:%d, gid:%d)", entry, ent.UID, ent.GID, owner.uid, owner.gid) - return - } - }) -} - -func mustSameEntry(files ...string) stargzCheck { - return stargzCheckFn(func(t TestingT, r *Reader) { - var first *TOCEntry - for _, f := range files { - if first == nil { - var ok bool - first, ok = r.Lookup(f) - if !ok { - t.Errorf("unknown first file on Lookup: %q", f) - return - } - } - - // Test Lookup - e, ok := r.Lookup(f) - if !ok { - t.Errorf("unknown file on Lookup: %q", f) - return - } - if e != first { - t.Errorf("Lookup: %+v(%p) != %+v(%p)", e, e, first, first) - return - } - - // Test LookupChild - pe, ok := r.Lookup(filepath.Dir(filepath.Clean(f))) - if !ok { - t.Errorf("failed to get parent of %q", f) - return - } - e, ok = pe.LookupChild(filepath.Base(filepath.Clean(f))) - if !ok { - t.Errorf("failed to get %q as the child of %+v", f, pe) - return - } - if e != first { - t.Errorf("LookupChild: %+v(%p) != %+v(%p)", e, e, first, first) - return - } - - // Test ForeachChild - pe.ForeachChild(func(baseName string, e *TOCEntry) bool { - if baseName == filepath.Base(filepath.Clean(f)) { - if e != first { - t.Errorf("ForeachChild: %+v(%p) != %+v(%p)", e, e, first, first) - return false - } - } - return true - }) - } - }) -} - -func viewContent(c []byte) string { - if len(c) < 100 { - return string(c) - } - return string(c[:50]) + "...(omit)..." + string(c[50:100]) -} - -func tarOf(s ...tarEntry) []tarEntry { return s } - -type tarEntry interface { - appendTar(tw *tar.Writer, prefix string, format tar.Format) error -} - -type tarEntryFunc func(*tar.Writer, string, tar.Format) error - -func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format) error { - return f(tw, prefix, format) -} - -func buildTar(t TestingT, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader { - format := tar.FormatUnknown - for _, opt := range opts { - switch v := opt.(type) { - case tar.Format: - format = v - default: - panic(fmt.Errorf("unsupported opt for buildTar: %v", opt)) - } - } - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - for _, ent := range ents { - if err := ent.appendTar(tw, prefix, format); err != nil { - t.Fatalf("building input tar: %v", err) - } - } - if err := tw.Close(); err != nil { - t.Errorf("closing write of input tar: %v", err) - } - data := append(buf.Bytes(), make([]byte, 100)...) // append empty bytes at the tail to see lossless works - return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) -} - -func dir(name string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - var o owner - mode := os.FileMode(0755) - for _, opt := range opts { - switch v := opt.(type) { - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if !strings.HasSuffix(name, "/") { - panic(fmt.Sprintf("missing trailing slash in dir %q ", name)) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeDir, - Name: prefix + name, - Mode: tm, - Uid: o.uid, - Gid: o.gid, - Format: format, - }) - }) -} - -// xAttr are extended attributes to set on test files created with the file func. -type xAttr map[string]string - -// owner is owner ot set on test files and directories with the file and dir functions. -type owner struct { - uid int - gid int -} - -func file(name, contents string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - var xattrs xAttr - var o owner - mode := os.FileMode(0644) - for _, opt := range opts { - switch v := opt.(type) { - case xAttr: - xattrs = v - case owner: - o = v - case os.FileMode: - mode = v - default: - return errors.New("unsupported opt") - } - } - if strings.HasSuffix(name, "/") { - return fmt.Errorf("bogus trailing slash in file %q", name) - } - tm, err := fileModeToTarMode(mode) - if err != nil { - return err - } - if len(xattrs) > 0 { - format = tar.FormatPAX // only PAX supports xattrs - } - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Mode: tm, - Xattrs: xattrs, - Size: int64(len(contents)), - Uid: o.uid, - Gid: o.gid, - Format: format, - }); err != nil { - return err - } - _, err = io.WriteString(tw, contents) - return err - }) -} - -func symlink(name, target string) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { - return tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeSymlink, - Name: prefix + name, - Linkname: target, - Mode: 0644, - Format: format, - }) - }) -} - -func link(name string, linkname string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeLink, - Name: prefix + name, - Linkname: linkname, - ModTime: now, - Format: format, - }) - }) -} - -func chardev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeChar, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - Format: format, - }) - }) -} - -func blockdev(name string, major, minor int64) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeBlock, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - Format: format, - }) - }) -} -func fifo(name string) tarEntry { - now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeFifo, - Name: prefix + name, - ModTime: now, - Format: format, - }) - }) -} - -func prefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Name: PrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - Format: format, - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func noPrefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Name: NoPrefetchLandmark, - Typeflag: tar.TypeReg, - Size: int64(len([]byte{landmarkContents})), - Format: format, - }); err != nil { - return err - } - contents := []byte{landmarkContents} - if _, err := io.CopyN(w, bytes.NewReader(contents), int64(len(contents))); err != nil { - return err - } - return nil - }) -} - -func regDigest(t TestingT, name string, contentStr string, digestMap map[string]digest.Digest) tarEntry { - if digestMap == nil { - t.Fatalf("digest map mustn't be nil") - } - content := []byte(contentStr) - - var n int64 - for n < int64(len(content)) { - size := int64(chunkSize) - remain := int64(len(content)) - n - if remain < size { - size = remain - } - dgstr := digest.Canonical.Digester() - if _, err := io.CopyN(dgstr.Hash(), bytes.NewReader(content[n:n+size]), size); err != nil { - t.Fatalf("failed to calculate digest of %q (name=%q,offset=%d,size=%d)", - string(content[n:n+size]), name, n, size) - } - digestMap[chunkID(name, n, size)] = dgstr.Digest() - n += size - } - - return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { - if err := w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: prefix + name, - Size: int64(len(content)), - Format: format, - }); err != nil { - return err - } - if _, err := io.CopyN(w, bytes.NewReader(content), int64(len(content))); err != nil { - return err - } - return nil - }) -} - -var runes = []rune("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randomContents(n int) string { - b := make([]rune, n) - for i := range b { - bi, err := rand.Int(rand.Reader, big.NewInt(int64(len(runes)))) - if err != nil { - panic(err) - } - b[i] = runes[int(bi.Int64())] - } - return string(b) -} - -func fileModeToTarMode(mode os.FileMode) (int64, error) { - h, err := tar.FileInfoHeader(fileInfoOnlyMode(mode), "") - if err != nil { - return 0, err - } - return h.Mode, nil -} - -// fileInfoOnlyMode is os.FileMode that populates only file mode. -type fileInfoOnlyMode os.FileMode - -func (f fileInfoOnlyMode) Name() string { return "" } -func (f fileInfoOnlyMode) Size() int64 { return 0 } -func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) } -func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() } -func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() } -func (f fileInfoOnlyMode) Sys() interface{} { return nil } - -func CheckGzipHasStreams(t TestingT, b []byte, streams []int64) { - if len(streams) == 0 { - return // nop - } - - wants := map[int64]struct{}{} - for _, s := range streams { - wants[s] = struct{}{} - } - - len0 := len(b) - br := bytes.NewReader(b) - zr := new(gzip.Reader) - t.Logf("got gzip streams:") - numStreams := 0 - for { - zoff := len0 - br.Len() - if err := zr.Reset(br); err != nil { - if err == io.EOF { - return - } - t.Fatalf("countStreams(gzip), Reset: %v", err) - } - zr.Multistream(false) - n, err := io.Copy(io.Discard, zr) - if err != nil { - t.Fatalf("countStreams(gzip), Copy: %v", err) - } - var extra string - if len(zr.Extra) > 0 { - extra = fmt.Sprintf("; extra=%q", zr.Extra) - } - t.Logf(" [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra) - delete(wants, int64(zoff)) - numStreams++ - } -} - -func GzipDiffIDOf(t TestingT, b []byte) string { - h := sha256.New() - zr, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - t.Fatalf("diffIDOf(gzip): %v", err) - } - defer zr.Close() - if _, err := io.Copy(h, zr); err != nil { - t.Fatalf("diffIDOf(gzip).Copy: %v", err) - } - return fmt.Sprintf("sha256:%x", h.Sum(nil)) -} diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go deleted file mode 100644 index 57e0aa61..00000000 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go +++ /dev/null @@ -1,342 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Copyright 2019 The Go Authors. All rights reserved. - Use of this source code is governed by a BSD-style - license that can be found in the LICENSE file. -*/ - -package estargz - -import ( - "archive/tar" - "hash" - "io" - "os" - "path" - "time" - - digest "github.com/opencontainers/go-digest" -) - -const ( - // TOCTarName is the name of the JSON file in the tar archive in the - // table of contents gzip stream. - TOCTarName = "stargz.index.json" - - // FooterSize is the number of bytes in the footer - // - // The footer is an empty gzip stream with no compression and an Extra - // header of the form "%016xSTARGZ", where the 64 bit hex-encoded - // number is the offset to the gzip stream of JSON TOC. - // - // 51 comes from: - // - // 10 bytes gzip header - // 2 bytes XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ")) - // 2 bytes Extra: SI1 = 'S', SI2 = 'G' - // 2 bytes Extra: LEN = 22 (16 hex digits + len("STARGZ")) - // 22 bytes Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC) - // 5 bytes flate header - // 8 bytes gzip footer - // (End of the eStargz blob) - // - // NOTE: For Extra fields, subfield IDs SI1='S' SI2='G' is used for eStargz. - FooterSize = 51 - - // legacyFooterSize is the number of bytes in the legacy stargz footer. - // - // 47 comes from: - // - // 10 byte gzip header + - // 2 byte (LE16) length of extra, encoding 22 (16 hex digits + len("STARGZ")) == "\x16\x00" + - // 22 bytes of extra (fmt.Sprintf("%016xSTARGZ", tocGzipOffset)) - // 5 byte flate header - // 8 byte gzip footer (two little endian uint32s: digest, size) - legacyFooterSize = 47 - - // TOCJSONDigestAnnotation is an annotation for an image layer. This stores the - // digest of the TOC JSON. - // This annotation is valid only when it is specified in `.[]layers.annotations` - // of an image manifest. - TOCJSONDigestAnnotation = "containerd.io/snapshot/stargz/toc.digest" - - // StoreUncompressedSizeAnnotation is an additional annotation key for eStargz to enable lazy - // pulling on containers/storage. Stargz Store is required to expose the layer's uncompressed size - // to the runtime but current OCI image doesn't ship this information by default. So we store this - // to the special annotation. - StoreUncompressedSizeAnnotation = "io.containers.estargz.uncompressed-size" - - // PrefetchLandmark is a file entry which indicates the end position of - // prefetch in the stargz file. - PrefetchLandmark = ".prefetch.landmark" - - // NoPrefetchLandmark is a file entry which indicates that no prefetch should - // occur in the stargz file. - NoPrefetchLandmark = ".no.prefetch.landmark" - - landmarkContents = 0xf -) - -// JTOC is the JSON-serialized table of contents index of the files in the stargz file. -type JTOC struct { - Version int `json:"version"` - Entries []*TOCEntry `json:"entries"` -} - -// TOCEntry is an entry in the stargz file's TOC (Table of Contents). -type TOCEntry struct { - // Name is the tar entry's name. It is the complete path - // stored in the tar file, not just the base name. - Name string `json:"name"` - - // Type is one of "dir", "reg", "symlink", "hardlink", "char", - // "block", "fifo", or "chunk". - // The "chunk" type is used for regular file data chunks past the first - // TOCEntry; the 2nd chunk and on have only Type ("chunk"), Offset, - // ChunkOffset, and ChunkSize populated. - Type string `json:"type"` - - // Size, for regular files, is the logical size of the file. - Size int64 `json:"size,omitempty"` - - // ModTime3339 is the modification time of the tar entry. Empty - // means zero or unknown. Otherwise it's in UTC RFC3339 - // format. Use the ModTime method to access the time.Time value. - ModTime3339 string `json:"modtime,omitempty"` - modTime time.Time - - // LinkName, for symlinks and hardlinks, is the link target. - LinkName string `json:"linkName,omitempty"` - - // Mode is the permission and mode bits. - Mode int64 `json:"mode,omitempty"` - - // UID is the user ID of the owner. - UID int `json:"uid,omitempty"` - - // GID is the group ID of the owner. - GID int `json:"gid,omitempty"` - - // Uname is the username of the owner. - // - // In the serialized JSON, this field may only be present for - // the first entry with the same UID. - Uname string `json:"userName,omitempty"` - - // Gname is the group name of the owner. - // - // In the serialized JSON, this field may only be present for - // the first entry with the same GID. - Gname string `json:"groupName,omitempty"` - - // Offset, for regular files, provides the offset in the - // stargz file to the file's data bytes. See ChunkOffset and - // ChunkSize. - Offset int64 `json:"offset,omitempty"` - - // InnerOffset is an optional field indicates uncompressed offset - // of this "reg" or "chunk" payload in a stream starts from Offset. - // This field enables to put multiple "reg" or "chunk" payloads - // in one chunk with having the same Offset but different InnerOffset. - InnerOffset int64 `json:"innerOffset,omitempty"` - - nextOffset int64 // the Offset of the next entry with a non-zero Offset - - // DevMajor is the major device number for "char" and "block" types. - DevMajor int `json:"devMajor,omitempty"` - - // DevMinor is the major device number for "char" and "block" types. - DevMinor int `json:"devMinor,omitempty"` - - // NumLink is the number of entry names pointing to this entry. - // Zero means one name references this entry. - // This field is calculated during runtime and not recorded in TOC JSON. - NumLink int `json:"-"` - - // Xattrs are the extended attribute for the entry. - Xattrs map[string][]byte `json:"xattrs,omitempty"` - - // Digest stores the OCI checksum for regular files payload. - // It has the form "sha256:abcdef01234....". - Digest string `json:"digest,omitempty"` - - // ChunkOffset is non-zero if this is a chunk of a large, - // regular file. If so, the Offset is where the gzip header of - // ChunkSize bytes at ChunkOffset in Name begin. - // - // In serialized form, a "chunkSize" JSON field of zero means - // that the chunk goes to the end of the file. After reading - // from the stargz TOC, though, the ChunkSize is initialized - // to a non-zero file for when Type is either "reg" or - // "chunk". - ChunkOffset int64 `json:"chunkOffset,omitempty"` - ChunkSize int64 `json:"chunkSize,omitempty"` - - // ChunkDigest stores an OCI digest of the chunk. This must be formed - // as "sha256:0123abcd...". - ChunkDigest string `json:"chunkDigest,omitempty"` - - children map[string]*TOCEntry - - // chunkTopIndex is index of the entry where Offset starts in the blob. - chunkTopIndex int -} - -// ModTime returns the entry's modification time. -func (e *TOCEntry) ModTime() time.Time { return e.modTime } - -// NextOffset returns the position (relative to the start of the -// stargz file) of the next gzip boundary after e.Offset. -func (e *TOCEntry) NextOffset() int64 { return e.nextOffset } - -func (e *TOCEntry) addChild(baseName string, child *TOCEntry) { - if e.children == nil { - e.children = make(map[string]*TOCEntry) - } - if child.Type == "dir" { - e.NumLink++ // Entry ".." in the subdirectory links to this directory - } - e.children[baseName] = child -} - -// isDataType reports whether TOCEntry is a regular file or chunk (something that -// contains regular file data). -func (e *TOCEntry) isDataType() bool { return e.Type == "reg" || e.Type == "chunk" } - -// Stat returns a FileInfo value representing e. -func (e *TOCEntry) Stat() os.FileInfo { return fileInfo{e} } - -// ForeachChild calls f for each child item. If f returns false, iteration ends. -// If e is not a directory, f is not called. -func (e *TOCEntry) ForeachChild(f func(baseName string, ent *TOCEntry) bool) { - for name, ent := range e.children { - if !f(name, ent) { - return - } - } -} - -// LookupChild returns the directory e's child by its base name. -func (e *TOCEntry) LookupChild(baseName string) (child *TOCEntry, ok bool) { - child, ok = e.children[baseName] - return -} - -// fileInfo implements os.FileInfo using the wrapped *TOCEntry. -type fileInfo struct{ e *TOCEntry } - -var _ os.FileInfo = fileInfo{} - -func (fi fileInfo) Name() string { return path.Base(fi.e.Name) } -func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" } -func (fi fileInfo) Size() int64 { return fi.e.Size } -func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() } -func (fi fileInfo) Sys() interface{} { return fi.e } -func (fi fileInfo) Mode() (m os.FileMode) { - // TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg. - m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() & - (os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky) - switch fi.e.Type { - case "dir": - m |= os.ModeDir - case "symlink": - m |= os.ModeSymlink - case "char": - m |= os.ModeDevice | os.ModeCharDevice - case "block": - m |= os.ModeDevice - case "fifo": - m |= os.ModeNamedPipe - } - return m -} - -// TOCEntryVerifier holds verifiers that are usable for verifying chunks contained -// in a eStargz blob. -type TOCEntryVerifier interface { - - // Verifier provides a content verifier that can be used for verifying the - // contents of the specified TOCEntry. - Verifier(ce *TOCEntry) (digest.Verifier, error) -} - -// Compression provides the compression helper to be used creating and parsing eStargz. -// This package provides gzip-based Compression by default, but any compression -// algorithm (e.g. zstd) can be used as long as it implements Compression. -type Compression interface { - Compressor - Decompressor -} - -// Compressor represents the helper mothods to be used for creating eStargz. -type Compressor interface { - // Writer returns WriteCloser to be used for writing a chunk to eStargz. - // Everytime a chunk is written, the WriteCloser is closed and Writer is - // called again for writing the next chunk. - // - // The returned writer should implement "Flush() error" function that flushes - // any pending compressed data to the underlying writer. - Writer(w io.Writer) (WriteFlushCloser, error) - - // WriteTOCAndFooter is called to write JTOC to the passed Writer. - // diffHash calculates the DiffID (uncompressed sha256 hash) of the blob - // WriteTOCAndFooter can optionally write anything that affects DiffID calculation - // (e.g. uncompressed TOC JSON). - // - // This function returns tocDgst that represents the digest of TOC that will be used - // to verify this blob when it's parsed. - WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (tocDgst digest.Digest, err error) -} - -// Decompressor represents the helper mothods to be used for parsing eStargz. -type Decompressor interface { - // Reader returns ReadCloser to be used for decompressing file payload. - Reader(r io.Reader) (io.ReadCloser, error) - - // FooterSize returns the size of the footer of this blob. - FooterSize() int64 - - // ParseFooter parses the footer and returns the offset and (compressed) size of TOC. - // payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between - // the top until the TOC JSON). - // - // If tocOffset < 0, we assume that TOC isn't contained in the blob and pass nil reader - // to ParseTOC. We expect that ParseTOC acquire TOC from the external location and return it. - // - // tocSize is optional. If tocSize <= 0, it's by default the size of the range from tocOffset until the beginning of the - // footer (blob size - tocOff - FooterSize). - // If blobPayloadSize < 0, blobPayloadSize become the blob size. - ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) - - // ParseTOC parses TOC from the passed reader. The reader provides the partial contents - // of the underlying blob that has the range specified by ParseFooter method. - // - // This function returns tocDgst that represents the digest of TOC that will be used - // to verify this blob. This must match to the value returned from - // Compressor.WriteTOCAndFooter that is used when creating this blob. - // - // If tocOffset returned by ParseFooter is < 0, we assume that TOC isn't contained in the blob. - // Pass nil reader to ParseTOC then we expect that ParseTOC acquire TOC from the external location - // and return it. - ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) -} - -type WriteFlushCloser interface { - io.WriteCloser - Flush() error -} diff --git a/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go b/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go deleted file mode 100644 index 69021bce..00000000 --- a/vendor/github.com/google/go-containerregistry/internal/estargz/estargz.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package estargz adapts the containerd estargz package to our abstractions. -package estargz - -import ( - "bytes" - "io" - - "github.com/containerd/stargz-snapshotter/estargz" - v1 "github.com/google/go-containerregistry/pkg/v1" -) - -// Assert that what we're returning is an io.ReadCloser -var _ io.ReadCloser = (*estargz.Blob)(nil) - -// ReadCloser reads uncompressed tarball input from the io.ReadCloser and -// returns: -// - An io.ReadCloser from which compressed data may be read, and -// - A v1.Hash with the hash of the estargz table of contents, or -// - An error if the estargz processing encountered a problem. -// -// Refer to estargz for the options: -// https://pkg.go.dev/github.com/containerd/stargz-snapshotter/estargz@v0.4.1#Option -func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) { - defer r.Close() - - // TODO(#876): Avoid buffering into memory. - bs, err := io.ReadAll(r) - if err != nil { - return nil, v1.Hash{}, err - } - br := bytes.NewReader(bs) - - rc, err := estargz.Build(io.NewSectionReader(br, 0, int64(len(bs))), opts...) - if err != nil { - return nil, v1.Hash{}, err - } - - h, err := v1.NewHash(rc.TOCDigest().String()) - return rc, h, err -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go index 6f08460b..ade2c63c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go @@ -25,7 +25,6 @@ import ( "github.com/docker/cli/cli/config/configfile" "github.com/docker/cli/cli/config/types" "github.com/google/go-containerregistry/pkg/name" - "github.com/mitchellh/go-homedir" ) // Resource represents a registry or repository that can be authenticated against. @@ -95,7 +94,7 @@ func (dk *defaultKeychain) ResolveContext(_ context.Context, target Resource) (A // First, check $HOME/.docker/config.json foundDockerConfig := false - home, err := homedir.Dir() + home, err := os.UserHomeDir() if err == nil { foundDockerConfig = fileExists(filepath.Join(home, ".docker/config.json")) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go index 5e6b6e62..7531d242 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/registry.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/registry.go @@ -24,8 +24,8 @@ import ( "strings" ) -// Detect more complex forms of local references. -var reLocal = regexp.MustCompile(`.*\.local(?:host)?(?::\d{1,5})?$`) +// Detect more complex forms of localhost references. +var reLocal = regexp.MustCompile(`.*\.localhost(?::\d{1,5})?$`) // Detect the loopback IP (127.0.0.1) var reLoopback = regexp.MustCompile(regexp.QuoteMeta("127.0.0.1")) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go index 22d483f3..783eecc6 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/manifest.go @@ -29,6 +29,7 @@ type Manifest struct { Layers []Descriptor `json:"layers"` Annotations map[string]string `json:"annotations,omitempty"` Subject *Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` } // IndexManifest represents an OCI image index in a structured way. @@ -38,6 +39,7 @@ type IndexManifest struct { Manifests []Descriptor `json:"manifests"` Annotations map[string]string `json:"annotations,omitempty"` Subject *Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` } // Descriptor holds a reference from the manifest to one of its constituent elements. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go index 3ea27fe4..0c58e0c1 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go @@ -52,6 +52,16 @@ func (i *image) MediaType() (types.MediaType, error) { return i.base.MediaType() } +// isImageConfig reports whether the media type is a Docker or OCI image config. +func isImageConfig(mt types.MediaType) bool { + switch mt { + case types.DockerConfigJSON, types.OCIConfigJSON: + return true + default: + return false + } +} + func (i *image) compute() error { i.Lock() defer i.Unlock() @@ -60,6 +70,24 @@ func (i *image) compute() error { if i.computed { return nil } + + m, err := i.base.Manifest() + if err != nil { + return err + } + manifest := m.DeepCopy() + + diffIDMap := make(map[v1.Hash]v1.Layer) + digestMap := make(map[v1.Hash]v1.Layer) + + // Determine effective config media type (user override takes precedence). + cfgMediaType := manifest.Config.MediaType + if i.configMediaType != nil { + cfgMediaType = *i.configMediaType + } + + imageConfig := isImageConfig(cfgMediaType) + var configFile *v1.ConfigFile if i.configFile != nil { configFile = i.configFile @@ -70,33 +98,32 @@ func (i *image) compute() error { } configFile = cf.DeepCopy() } - diffIDs := configFile.RootFS.DiffIDs - history := configFile.History - diffIDMap := make(map[v1.Hash]v1.Layer) - digestMap := make(map[v1.Hash]v1.Layer) + // For image configs, update RootFS.DiffIDs and History from added layers. + // For artifacts, skip this: the config has no rootfs or history fields. + if imageConfig { + diffIDs := configFile.RootFS.DiffIDs + history := configFile.History - for _, add := range i.adds { - history = append(history, add.History) - if add.Layer != nil { - diffID, err := add.Layer.DiffID() - if err != nil { - return err + for _, add := range i.adds { + history = append(history, add.History) + if add.Layer != nil { + diffID, err := add.Layer.DiffID() + if err != nil { + return err + } + diffIDs = append(diffIDs, diffID) + diffIDMap[diffID] = add.Layer } - diffIDs = append(diffIDs, diffID) - diffIDMap[diffID] = add.Layer } - } - m, err := i.base.Manifest() - if err != nil { - return err + configFile.RootFS.DiffIDs = diffIDs + configFile.History = history } - manifest := m.DeepCopy() + manifestLayers := manifest.Layers for _, add := range i.adds { if add.Layer == nil { - // Empty layers include only history in manifest. continue } @@ -105,14 +132,12 @@ func (i *image) compute() error { return err } - // Fields in the addendum override the original descriptor. if len(add.Annotations) != 0 { desc.Annotations = add.Annotations } if len(add.URLs) != 0 { desc.URLs = add.URLs } - if add.MediaType != "" { desc.MediaType = add.MediaType } @@ -120,42 +145,38 @@ func (i *image) compute() error { manifestLayers = append(manifestLayers, *desc) digestMap[desc.Digest] = add.Layer } - - configFile.RootFS.DiffIDs = diffIDs - configFile.History = history - manifest.Layers = manifestLayers - rcfg, err := json.Marshal(configFile) - if err != nil { - return err - } - d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) - if err != nil { - return err - } - manifest.Config.Digest = d - manifest.Config.Size = sz + // For image configs, re-marshal the config and update the manifest digest. + // For artifacts, preserve the original config blob as-is to avoid + // corrupting the digest via re-marshaling. + if imageConfig { + rcfg, err := json.Marshal(configFile) + if err != nil { + return err + } + d, sz, err := v1.SHA256(bytes.NewBuffer(rcfg)) + if err != nil { + return err + } + manifest.Config.Digest = d + manifest.Config.Size = sz - // If Data was set in the base image, we need to update it in the mutated image. - if m.Config.Data != nil { - manifest.Config.Data = rcfg + if m.Config.Data != nil { + manifest.Config.Data = rcfg + } } - // If the user wants to mutate the media type of the config if i.configMediaType != nil { manifest.Config.MediaType = *i.configMediaType } - if i.mediaType != nil { manifest.MediaType = *i.mediaType } - if i.annotations != nil { if manifest.Annotations == nil { manifest.Annotations = map[string]string{} } - for k, v := range i.annotations { manifest.Annotations[k] = v } @@ -173,29 +194,34 @@ func (i *image) compute() error { // Layers returns the ordered collection of filesystem layers that comprise this image. // The order of the list is oldest/base layer first, and most-recent/top layer last. func (i *image) Layers() ([]v1.Layer, error) { - if err := i.compute(); errors.Is(err, stream.ErrNotComputed) { - // Image contains a streamable layer which has not yet been - // consumed. Just return the layers we have in case the caller - // is going to consume the layers. + if err := i.compute(); errors.Is(err, stream.ErrNotComputed) || (i.manifest != nil && !isImageConfig(i.manifest.Config.MediaType)) { + // Stream not yet consumed, or non-image OCI artifact (RootFS.DiffIDs + // is empty so partial.DiffIDs returns nothing). Fall back to the base + // layers plus any added layers. layers, err := i.base.Layers() if err != nil { return nil, err } for _, add := range i.adds { - layers = append(layers, add.Layer) + if add.Layer != nil { + layers = append(layers, add.Layer) + } } return layers, nil } else if err != nil { return nil, err } - diffIDs, err := partial.DiffIDs(i) - if err != nil { - return nil, err - } - ls := make([]v1.Layer, 0, len(diffIDs)) - for _, h := range diffIDs { - l, err := i.LayerByDiffID(h) + // Walk manifest layer descriptors by digest rather than rootfs diff + // IDs. Two layers can legitimately share a diff ID — same uncompressed + // content, different compression — and produce distinct digests. The + // manifest preserves the per-occurrence digest; LayerByDiffID does not, + // which previously caused duplicate-diff-ID layers to collapse to a + // single entry in the returned slice and break blob upload for + // downstream pushers (see #2034). + ls := make([]v1.Layer, 0, len(i.manifest.Layers)) + for _, desc := range i.manifest.Layers { + l, err := i.LayerByDigest(desc.Digest) if err != nil { return nil, err } @@ -220,11 +246,18 @@ func (i *image) ConfigFile() (*v1.ConfigFile, error) { return i.configFile.DeepCopy(), nil } -// RawConfigFile returns the serialized bytes of ConfigFile() +// RawConfigFile returns the serialized bytes of ConfigFile(). +// For non-image OCI artifacts, returns the original raw config to preserve +// the config blob digest. func (i *image) RawConfigFile() ([]byte, error) { if err := i.compute(); err != nil { return nil, err } + // If the manifest config is not a standard image config, return the + // original raw bytes to avoid corrupting the digest via re-marshaling. + if i.manifest != nil && !isImageConfig(i.manifest.Config.MediaType) { + return i.base.RawConfigFile() + } return json.Marshal(i.configFile) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go index 2e35f225..a6186b37 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/mutate.go @@ -277,68 +277,91 @@ func extract(img v1.Image, w io.Writer) error { // whiteout layers more efficient, since we can just keep track of the removed // files as we see .wh. layers and ignore those in previous layers. for i := len(layers) - 1; i >= 0; i-- { - layer := layers[i] - layerReader, err := layer.Uncompressed() + if err := extractLayer(tarWriter, fileMap, layers[i]); err != nil { + return err + } + } + return nil +} + +func extractLayer(tarWriter *tar.Writer, fileMap map[string]bool, layer v1.Layer) error { + layerReader, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("reading layer contents: %w", err) + } + defer layerReader.Close() + + tarReader := tar.NewReader(layerReader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } if err != nil { - return fmt.Errorf("reading layer contents: %w", err) + return fmt.Errorf("reading tar: %w", err) } - defer layerReader.Close() - tarReader := tar.NewReader(layerReader) - for { - header, err := tarReader.Next() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("reading tar: %w", err) + + // Some tools prepend everything with "./", so if we don't Clean the + // name, we may have duplicate entries, which angers tar-split. + header.Name = filepath.Clean(header.Name) + + // Reject relative symlinks and hardlinks whose targets escape the + // image rootfs. Relative targets are resolved against the symlink's + // own directory: if the clean result starts with ".." the link would + // leave the rootfs. Relative symlinks that stay within the rootfs + // (common for glibc, C toolchains, etc.) are preserved unchanged. + // Absolute targets are left as-is; see #2238 for ongoing discussion + // on whether they should be pruned. + if header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeLink { + if !filepath.IsAbs(header.Linkname) { + resolved := filepath.Clean(filepath.Join(filepath.Dir(header.Name), header.Linkname)) //nolint:gosec // G305: path is only used for validation, not file I/O + if strings.HasPrefix(resolved, "..") { + continue + } } + } - // Some tools prepend everything with "./", so if we don't Clean the - // name, we may have duplicate entries, which angers tar-split. - header.Name = filepath.Clean(header.Name) + // force PAX format to remove Name/Linkname length limit of 100 characters + // required by USTAR and to not depend on internal tar package guess which + // prefers USTAR over PAX + header.Format = tar.FormatPAX - // force PAX format to remove Name/Linkname length limit of 100 characters - // required by USTAR and to not depend on internal tar package guess which - // prefers USTAR over PAX - header.Format = tar.FormatPAX + basename := filepath.Base(header.Name) + dirname := filepath.Dir(header.Name) + tombstone := strings.HasPrefix(basename, whiteoutPrefix) + if tombstone { + basename = basename[len(whiteoutPrefix):] + } - basename := filepath.Base(header.Name) - dirname := filepath.Dir(header.Name) - tombstone := strings.HasPrefix(basename, whiteoutPrefix) - if tombstone { - basename = basename[len(whiteoutPrefix):] - } + // check if we have seen value before + // if we're checking a directory, don't filepath.Join names + var name string + if header.Typeflag == tar.TypeDir { + name = header.Name + } else { + name = filepath.Join(dirname, basename) + } - // check if we have seen value before - // if we're checking a directory, don't filepath.Join names - var name string - if header.Typeflag == tar.TypeDir { - name = header.Name - } else { - name = filepath.Join(dirname, basename) - } + if _, ok := fileMap[name]; ok && !tombstone { + continue + } - if _, ok := fileMap[name]; ok && !tombstone { - continue - } + // check for a whited out parent directory + if inWhiteoutDir(fileMap, name) { + continue + } - // check for a whited out parent directory - if inWhiteoutDir(fileMap, name) { - continue + // mark file as handled. non-directory implicitly tombstones + // any entries with a matching (or child) name + fileMap[name] = tombstone || (header.Typeflag != tar.TypeDir) + if !tombstone { + if err := tarWriter.WriteHeader(header); err != nil { + return err } - - // mark file as handled. non-directory implicitly tombstones - // any entries with a matching (or child) name - fileMap[name] = tombstone || (header.Typeflag != tar.TypeDir) - if !tombstone { - if err := tarWriter.WriteHeader(header); err != nil { + if header.Size > 0 { + if _, err := io.CopyN(tarWriter, tarReader, header.Size); err != nil { return err } - if header.Size > 0 { - if _, err := io.CopyN(tarWriter, tarReader, header.Size); err != nil { - return err - } - } } } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index c8b22b3f..e83b4a88 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -337,8 +337,12 @@ func Descriptor(d Describable) (*v1.Descriptor, error) { mf, _ := Manifest(wrm) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - desc.ArtifactType = string(mf.Config.MediaType) + if mf != nil { + if mf.ArtifactType != "" { + desc.ArtifactType = mf.ArtifactType + } else { + desc.ArtifactType = string(mf.Config.MediaType) + } } } } @@ -429,7 +433,10 @@ func ArtifactType(w WithManifest) (string, error) { mf, _ := w.Manifest() // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { + if mf != nil { + if mf.ArtifactType != "" { + return mf.ArtifactType, nil + } return string(mf.Config.MediaType), nil } return "", nil diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go index d77b37c0..3bda1d09 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/fetcher.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "io" + "net" "net/http" "net/url" "strings" @@ -40,8 +41,9 @@ const ( // fetcher implements methods for reading from a registry. type fetcher struct { - target resource - client *http.Client + target resource + client *http.Client + limiter *pullLimiter } func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, error) { @@ -69,10 +71,40 @@ func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, er } return &fetcher{ target: target, - client: &http.Client{Transport: tr}, + client: &http.Client{ + Transport: tr, + CheckRedirect: checkRedirectSSRF, + }, + limiter: o.limiter, }, nil } +// checkRedirectSSRF rejects HTTP redirects that cross from a public host to a +// private or link-local IP literal. This prevents a malicious registry from +// issuing a 302 to a cloud instance metadata service (e.g. 169.254.169.254) +// or another internal network address during blob or manifest downloads. +// +// Same-host redirects and redirects to non-IP hostnames (including DNS names +// that may resolve to private addresses) are allowed. The first redirect in +// the chain uses the original request URL as the "origin host" via +// req.Response.Request, falling back to req.URL when no prior response exists. +func checkRedirectSSRF(req *http.Request, via []*http.Request) error { + if len(via) == 0 || req.Response == nil { + return nil + } + origHost := via[0].URL.Hostname() + destHost := req.URL.Hostname() + if destHost == origHost { + return nil // same-host redirect is always allowed + } + if ip := net.ParseIP(destHost); ip != nil { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { + return fmt.Errorf("SSRF protection: redirect from %q to private/link-local host %q denied", origHost, destHost) + } + } + return nil +} + func (f *fetcher) Do(req *http.Request) (*http.Response, error) { return f.client.Do(req) } @@ -162,11 +194,20 @@ func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, accepta } var artifactType string + var annotations map[string]string mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - artifactType = string(mf.Config.MediaType) + if mf != nil { + // Per the OCI distribution spec, artifactType on the descriptor is + // set to the manifest's artifactType if present, otherwise it falls + // back to the config descriptor's mediaType. + if mf.ArtifactType != "" { + artifactType = mf.ArtifactType + } else { + artifactType = string(mf.Config.MediaType) + } + annotations = mf.Annotations } // Do nothing for tags; I give up. @@ -183,6 +224,7 @@ func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, accepta Size: size, MediaType: mediaType, ArtifactType: artifactType, + Annotations: annotations, } return manifest, &desc, nil @@ -247,18 +289,30 @@ func (f *fetcher) headManifest(ctx context.Context, ref name.Reference, acceptab func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.ReadCloser, error) { u := f.url("blobs", h.String()) + return f.fetchBlobURL(ctx, u, size, h) +} + +func (f *fetcher) fetchBlobURL(ctx context.Context, u url.URL, size int64, h v1.Hash) (io.ReadCloser, error) { + release, err := f.limiter.acquire(ctx) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { + release() return nil, err } resp, err := f.client.Do(req.WithContext(ctx)) if err != nil { + release() return nil, redact.Error(err) } if err := transport.CheckError(resp, http.StatusOK); err != nil { resp.Body.Close() + release() return nil, err } @@ -269,11 +323,22 @@ func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.Read if size == verify.SizeUnknown { size = hsize } else if hsize != size { + resp.Body.Close() + release() return nil, fmt.Errorf("GET %s: Content-Length header %d does not match expected size %d", u.String(), hsize, size) } } - return verify.ReadCloser(resp.Body, size, h) + rc, err := verify.ReadCloser(resp.Body, size, h) + if err != nil { + resp.Body.Close() + release() + return nil, err + } + return &limitedReadCloser{ + ReadCloser: rc, + release: release, + }, nil } func (f *fetcher) headBlob(ctx context.Context, h v1.Hash) (*http.Response, error) { @@ -296,6 +361,57 @@ func (f *fetcher) headBlob(ctx context.Context, h v1.Hash) (*http.Response, erro return resp, nil } +// validateForeignURL rejects foreign layer URLs that use a disallowed scheme +// or resolve to a private / link-local IP address (SSRF protection). DNS-based +// SSRF is out of scope, matching transport.validateRealmURL. +func validateForeignURL(rawURL string, insecure bool) error { + u, err := url.Parse(rawURL) + if err != nil { + return fmt.Errorf("parsing foreign layer URL %q: %w", rawURL, err) + } + switch u.Scheme { + case "https": + case "http": + if !insecure { + return fmt.Errorf("foreign layer URL scheme %q not allowed for a secure registry; use https", u.Scheme) + } + default: + return fmt.Errorf("foreign layer URL scheme %q not allowed; must be https (or http for insecure registries)", u.Scheme) + } + host := u.Hostname() + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { + return fmt.Errorf("foreign layer URL host %q is a private or link-local address", host) + } + } + return nil +} + +// fetchForeignBlobURL fetches a foreign-layer blob, validating every redirect +// destination through validateForeignURL (SSRF protection). +func (f *fetcher) fetchForeignBlobURL(ctx context.Context, u url.URL, size int64, h v1.Hash, insecure bool) (io.ReadCloser, error) { + safeClient := &http.Client{ + Transport: f.client.Transport, + CheckRedirect: func(req *http.Request, _ []*http.Request) error { + return validateForeignURL(req.URL.String(), insecure) + }, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + resp, err := safeClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("GET %s: unexpected status %s", u.String(), resp.Status) + } + return verify.ReadCloser(resp.Body, size, h) +} + func (f *fetcher) blobExists(ctx context.Context, h v1.Hash) (bool, error) { u := f.url("blobs", h.String()) req, err := http.NewRequest(http.MethodHead, u.String(), nil) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index f085967e..d39a0e71 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "io" - "net/http" "net/url" "sync" @@ -27,7 +26,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -172,7 +170,7 @@ func (rl *remoteImageLayer) Digest() (v1.Hash, error) { // Compressed implements partial.CompressedLayer func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) { - urls := []url.URL{rl.ri.fetcher.url("blobs", rl.digest.String())} + u := rl.ri.fetcher.url("blobs", rl.digest.String()) // Add alternative layer sources from URLs (usually none). d, err := partial.BlobDescriptor(rl, rl.digest) @@ -187,38 +185,38 @@ func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) { // We don't want to log binary layers -- this can break terminals. ctx := redact.NewContext(rl.ctx, "omitting binary blobs from logs") - for _, s := range d.URLs { - u, err := url.Parse(s) - if err != nil { - return nil, err - } - urls = append(urls, *u) - } + insecure := rl.ri.fetcher.target.Scheme() == "http" // The lastErr for most pulls will be the same (the first error), but for // foreign layers we'll want to surface the last one, since we try to pull // from the registry first, which would often fail. // TODO: Maybe we don't want to try pulling from the registry first? var lastErr error - for _, u := range urls { - req, err := http.NewRequest(http.MethodGet, u.String(), nil) - if err != nil { + rc, err := rl.ri.fetcher.fetchBlobURL(ctx, u, d.Size, rl.digest) + if err == nil { + return rc, nil + } + lastErr = err + + foreignURLs := make([]url.URL, 0, len(d.URLs)) + for _, s := range d.URLs { + if err := validateForeignURL(s, insecure); err != nil { return nil, err } - - resp, err := rl.ri.fetcher.Do(req.WithContext(ctx)) + fu, err := url.Parse(s) if err != nil { - lastErr = err - continue + return nil, err } + foreignURLs = append(foreignURLs, *fu) + } - if err := transport.CheckError(resp, http.StatusOK); err != nil { - resp.Body.Close() + for _, fu := range foreignURLs { + rc, err := rl.ri.fetcher.fetchForeignBlobURL(ctx, fu, d.Size, rl.digest, insecure) + if err != nil { lastErr = err continue } - - return verify.ReadCloser(resp.Body, d.Size, rl.digest) + return rc, nil } return nil, lastErr diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index b80972c8..3956a38e 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -225,8 +225,12 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) mf, _ := v1.ParseManifest(bytes.NewReader(manifest)) // Failing to parse as a manifest should just be ignored. // The manifest might not be valid, and that's okay. - if mf != nil && !mf.Config.MediaType.IsConfig() { - child.ArtifactType = string(mf.Config.MediaType) + if mf != nil { + if mf.ArtifactType != "" { + child.ArtifactType = mf.ArtifactType + } else { + child.ArtifactType = string(mf.Config.MediaType) + } } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go new file mode 100644 index 00000000..54ec3b28 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/limiter.go @@ -0,0 +1,55 @@ +// Copyright 2026 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "context" + "io" + "sync" +) + +type pullLimiter struct { + tokens chan struct{} +} + +func newPullLimiter(jobs int) *pullLimiter { + return &pullLimiter{ + tokens: make(chan struct{}, jobs), + } +} + +func (l *pullLimiter) acquire(ctx context.Context) (func(), error) { + if l == nil { + return func() {}, nil + } + select { + case l.tokens <- struct{}{}: + return func() { <-l.tokens }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +type limitedReadCloser struct { + io.ReadCloser + release func() + once sync.Once +} + +func (l *limitedReadCloser) Close() error { + err := l.ReadCloser.Close() + l.once.Do(l.release) + return err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 15b7da1e..3d8bb8dd 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -45,6 +45,7 @@ type options struct { retryBackoff Backoff retryPredicate retry.Predicate retryStatusCodes []int + limiter *pullLimiter // Only these options can overwrite Reuse()d options. platform v1.Platform @@ -92,6 +93,7 @@ var fastBackoff = Backoff{ var defaultRetryStatusCodes = []int{ http.StatusRequestTimeout, + http.StatusTooManyRequests, // 429: OCI distribution-spec rate limit; TooManyRequestsErrorCode is already classified temporary in transport/error.go http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, @@ -142,6 +144,7 @@ func makeOptions(opts ...Option) (*options, error) { return nil, err } } + o.limiter = newPullLimiter(o.jobs) switch { case o.auth != nil && o.keychain != nil: diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go index 47e3b806..5675c97c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/pusher.go @@ -156,7 +156,11 @@ func (p *Pusher) Upload(ctx context.Context, repo name.Repository, l v1.Layer) e } func (p *Pusher) Delete(ctx context.Context, ref name.Reference) error { - w, err := p.writer(ctx, ref.Context(), p.o) + // Use a transport scoped for delete. Requesting DeleteScope (which + // includes the "delete" action) allows registries that require an + // explicit delete permission—such as IBM Cloud Container Registry—to + // grant access. + client, err := makeDeleteClient(ctx, ref.Context(), p.o) if err != nil { return err } @@ -172,7 +176,7 @@ func (p *Pusher) Delete(ctx context.Context, ref name.Reference) error { return err } - resp, err := w.w.client.Do(req.WithContext(ctx)) + resp, err := client.Do(req.WithContext(ctx)) if err != nil { return err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go index 4bc6f70a..17e9c26e 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/referrers.go @@ -67,7 +67,7 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string, var b []byte if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Type") == string(types.OCIImageIndex) { - b, err = io.ReadAll(resp.Body) + b, err = io.ReadAll(io.LimitReader(resp.Body, manifestLimit)) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index 44a323ee..f845dd1f 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -33,6 +33,10 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote/internal/authchallenge" ) +// maxTokenBodySize limits bearer token response body reads to prevent OOM +// when a token endpoint returns an unexpectedly large body. +const maxTokenBodySize = 64 * 1024 // 64 KiB + type Token struct { Token string `json:"token"` AccessToken string `json:"access_token,omitempty"` @@ -86,7 +90,7 @@ func fromChallenge(reg name.Registry, auth authn.Authenticator, t http.RoundTrip // registry can supply a realm pointing at an internal service or cloud // metadata endpoint (e.g. 169.254.169.254), causing SSRF when the client // subsequently fetches a token. - if err := validateRealmURL(realm, pr.Insecure); err != nil { + if err := validateRealmURL(realm, reg.RegistryStr(), pr.Insecure); err != nil { return nil, fmt.Errorf("invalid realm in www-authenticate: %w", err) } service := pr.Parameters["service"] @@ -105,10 +109,24 @@ func fromChallenge(reg name.Registry, auth authn.Authenticator, t http.RoundTrip }, nil } +// realmRedirectCheck mimics the default http.Client redirect policy but also +// validates each redirect URL with validateRealmURL. +func realmRedirectCheck(registryHost string, insecure bool) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("stopped after 10 redirects") + } + if err := validateRealmURL(req.URL.String(), registryHost, insecure); err != nil { + return fmt.Errorf("refusing token-server redirect to %q: %w", req.URL, err) + } + return nil + } +} + // validateRealmURL returns an error if the realm URL uses a disallowed scheme -// or resolves to a private / link-local IP address. This prevents a crafted -// WWW-Authenticate header from redirecting token fetches to internal services. -func validateRealmURL(realm string, insecure bool) error { +// or resolves to a private / link-local IP address. Realm URLs matching the +// registry host:port are always allowed. See #2258. +func validateRealmURL(realm, registryHost string, insecure bool) error { u, err := url.Parse(realm) if err != nil { return fmt.Errorf("parsing realm %q: %w", realm, err) @@ -123,6 +141,10 @@ func validateRealmURL(realm string, insecure bool) error { default: return fmt.Errorf("realm scheme %q not allowed; must be https (or http for insecure registries)", u.Scheme) } + // Always allow realms matching the registry host:port. + if registryHost != "" && u.Host == registryHost { + return nil + } // Reject IP literals that resolve to private or link-local ranges. // This blocks direct references to RFC 1918 addresses, loopback, and // link-local ranges including the cloud instance metadata service @@ -130,7 +152,7 @@ func validateRealmURL(realm string, insecure bool) error { // here; callers should apply network-level controls if needed. host := u.Hostname() if ip := net.ParseIP(host); ip != nil { - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { return fmt.Errorf("realm host %q is a private or link-local address", host) } } @@ -374,7 +396,8 @@ func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { v.Set("access_type", "offline") } - client := http.Client{Transport: bt.inner} + allowInsecure := bt.scheme == "http" + client := http.Client{Transport: bt.inner, CheckRedirect: realmRedirectCheck(bt.registry.RegistryStr(), allowInsecure)} req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return nil, err @@ -397,7 +420,7 @@ func (bt *bearerTransport) refreshOauth(ctx context.Context) ([]byte, error) { return nil, err } - return io.ReadAll(resp.Body) + return io.ReadAll(io.LimitReader(resp.Body, maxTokenBodySize)) } // https://docs.docker.com/registry/spec/auth/token/ @@ -411,7 +434,8 @@ func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { auth: bt.basic, target: u.Host, } - client := http.Client{Transport: b} + allowInsecure := bt.scheme == "http" + client := http.Client{Transport: b, CheckRedirect: realmRedirectCheck(bt.registry.RegistryStr(), allowInsecure)} v := u.Query() bt.mx.RLock() @@ -441,5 +465,5 @@ func (bt *bearerTransport) refreshBasic(ctx context.Context) ([]byte, error) { return nil, err } - return io.ReadAll(resp.Body) + return io.ReadAll(io.LimitReader(resp.Body, maxTokenBodySize)) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index d38e6762..934484db 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -15,6 +15,7 @@ package transport import ( + "bytes" "encoding/json" "fmt" "io" @@ -112,6 +113,10 @@ type ErrorCode string // The set of error conditions a registry may return: // https://github.com/distribution/distribution/blob/aac2f6c8b7c5a6c60190848bab5cbeed2b5ba0a9/docs/spec/api.md#errors-2 +// maxErrorBodySize limits HTTP error response body reads to prevent OOM when +// a registry returns an unexpectedly large error body. +const maxErrorBodySize = 64 * 1024 // 64 KiB + const ( BlobUnknownErrorCode ErrorCode = "BLOB_UNKNOWN" BlobUploadInvalidErrorCode ErrorCode = "BLOB_UPLOAD_INVALID" @@ -146,6 +151,7 @@ var temporaryErrorCodes = map[ErrorCode]struct{}{ var temporaryStatusCodes = map[int]struct{}{ http.StatusRequestTimeout: {}, + http.StatusTooManyRequests: {}, // matches TooManyRequestsErrorCode in temporaryErrorCodes http.StatusInternalServerError: {}, http.StatusBadGateway: {}, http.StatusServiceUnavailable: {}, @@ -161,7 +167,7 @@ func CheckError(resp *http.Response, codes ...int) error { } } - b, err := io.ReadAll(resp.Body) + b, err := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodySize)) if err != nil { return err } @@ -185,11 +191,16 @@ func makeError(resp *http.Response, body []byte) *Error { } func retryError(resp *http.Response) error { - b, err := io.ReadAll(resp.Body) + b, err := io.ReadAll(io.LimitReader(resp.Body, maxErrorBodySize)) if err != nil { return err } + // Restore the body so that a subsequent CheckError call (after the + // retry loop exhausts its retries) can still read and parse the + // structured registry error from the response. + resp.Body = io.NopCloser(bytes.NewReader(b)) + rerr := makeError(resp, b) rerr.temporary = true return rerr diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go index c3b56f7a..6ee7782b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/scope.go @@ -18,7 +18,9 @@ package transport const ( PullScope string = "pull" PushScope string = "push,pull" - // For now DELETE is PUSH, which is the read/write ACL. - DeleteScope string = PushScope + // DeleteScope requests "delete" in addition to push/pull so that + // registries requiring an explicit delete action (e.g. IBM Cloud + // Container Registry) grant the necessary access. + DeleteScope string = "push,pull,delete" CatalogScope string = "catalog" ) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 58adfd8d..4ff30a8c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "sort" @@ -66,6 +67,25 @@ type writer struct { scopes []string } +// makeDeleteClient returns an HTTP client whose token includes the "delete" +// action so that registries requiring an explicit delete permission grant +// access for manifest deletion. +func makeDeleteClient(ctx context.Context, repo name.Repository, o *options) (*http.Client, error) { + auth := o.auth + if o.keychain != nil { + kauth, err := authn.Resolve(ctx, o.keychain, repo) + if err != nil { + return nil, err + } + auth = kauth + } + tr, err := transport.NewWithContext(ctx, repo.Registry, auth, o.transport, []string{repo.Scope(transport.DeleteScope)}) + if err != nil { + return nil, err + } + return &http.Client{Transport: tr}, nil +} + func makeWriter(ctx context.Context, repo name.Repository, ls []v1.Layer, o *options) (*writer, error) { auth := o.auth if o.keychain != nil { @@ -148,7 +168,29 @@ func (w *writer) nextLocation(resp *http.Response) (string, error) { // If the location header returned is just a url path, then fully qualify it. // We cannot simply call w.url, since there might be an embedded query string. - return resp.Request.URL.ResolveReference(u).String(), nil + resolved := resp.Request.URL.ResolveReference(u) + + // Reject Location headers that redirect to a DIFFERENT host that resolves to + // a private or link-local IP literal. A malicious or compromised registry can + // respond to a blob upload initiation (POST /v2/.../blobs/uploads/) with a + // crafted Location header pointing at an internal service, causing the client + // to send subsequent PATCH/PUT requests (including the layer data as the body) + // to that internal address. Pre-signed blob URLs from cloud storage providers + // (GCS, S3, Azure Blob) use public hostnames, so legitimate cross-host + // redirects are unaffected. + // + // Same-host redirects (e.g. a different path on the registry itself) are + // always allowed regardless of whether the registry IP is private. + origHost := resp.Request.URL.Hostname() + if destHost := resolved.Hostname(); destHost != origHost { + if ip := net.ParseIP(destHost); ip != nil { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() || ip.IsUnspecified() { + return "", fmt.Errorf("SSRF protection: Location header redirects to private/link-local host %q", destHost) + } + } + } + + return resolved.String(), nil } // checkExistingBlob checks if a blob exists already in the repository by making a @@ -555,9 +597,10 @@ func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Refere return err } var mf struct { - MediaType types.MediaType `json:"mediaType"` - Subject *v1.Descriptor `json:"subject,omitempty"` - Config struct { + MediaType types.MediaType `json:"mediaType"` + Subject *v1.Descriptor `json:"subject,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Config struct { MediaType types.MediaType `json:"mediaType"` } `json:"config"` } @@ -599,10 +642,14 @@ func (w *writer) commitManifest(ctx context.Context, t Taggable, ref name.Refere return err } desc := v1.Descriptor{ - ArtifactType: string(mf.Config.MediaType), - MediaType: mf.MediaType, - Digest: h, - Size: size, + MediaType: mf.MediaType, + Digest: h, + Size: size, + } + if mf.ArtifactType != "" { + desc.ArtifactType = mf.ArtifactType + } else { + desc.ArtifactType = string(mf.Config.MediaType) } if err := w.commitSubjectReferrers(ctx, ref.Context().Digest(mf.Subject.Digest.String()), diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 8a263096..5e63d52d 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -22,10 +22,7 @@ import ( "os" "sync" - "github.com/containerd/stargz-snapshotter/estargz" - "github.com/google/go-containerregistry/internal/and" comp "github.com/google/go-containerregistry/internal/compression" - gestargz "github.com/google/go-containerregistry/internal/estargz" ggzip "github.com/google/go-containerregistry/internal/gzip" "github.com/google/go-containerregistry/internal/zstd" "github.com/google/go-containerregistry/pkg/compression" @@ -43,7 +40,6 @@ type layer struct { compression compression.Compression compressionLevel int annotations map[string]string - estgzopts []estargz.Option mediaType types.MediaType } @@ -161,53 +157,15 @@ func WithCompressedCaching(l *layer) { // through estargz.Options to the underlying compression layer. This is // only meaningful when estargz is enabled. // -// Deprecated: WithEstargz is deprecated, and will be removed in a future release. -func WithEstargzOptions(opts ...estargz.Option) LayerOption { - return func(l *layer) { - l.estgzopts = opts - } +// Deprecated: WithEstargz is deprecated; it is a no-op. +func WithEstargzOptions(...any) LayerOption { + return func(*layer) {} } // WithEstargz is a functional option that explicitly enables estargz support. // -// Deprecated: WithEstargz is deprecated, and will be removed in a future release. -func WithEstargz(l *layer) { - oguncompressed := l.uncompressedopener - estargz := func() (io.ReadCloser, error) { - crc, err := oguncompressed() - if err != nil { - return nil, err - } - eopts := append(l.estgzopts, estargz.WithCompressionLevel(l.compressionLevel)) - rc, h, err := gestargz.ReadCloser(crc, eopts...) - if err != nil { - return nil, err - } - l.annotations[estargz.TOCJSONDigestAnnotation] = h.String() - return &and.ReadCloser{ - Reader: rc, - CloseFunc: func() error { - err := rc.Close() - if err != nil { - return err - } - // As an optimization, leverage the DiffID exposed by the estargz ReadCloser - l.diffID, err = v1.NewHash(rc.DiffID().String()) - return err - }, - }, nil - } - uncompressed := func() (io.ReadCloser, error) { - urc, err := estargz() - if err != nil { - return nil, err - } - return ggzip.UnzipReadCloser(urc) - } - - l.compressedopener = estargz - l.uncompressedopener = uncompressed -} +// Deprecated: WithEstargz is deprecated; it is a no-op. +func WithEstargz(*layer) {} // LayerFromFile returns a v1.Layer given a tarball func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) { @@ -241,11 +199,6 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { mediaType: types.DockerLayer, } - if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" { - logs.Warn.Println("GGCR_EXPERIMENT_ESTARGZ is deprecated, and will be removed in a future release.") - opts = append([]LayerOption{WithEstargz}, opts...) - } - switch comp { case compression.GZip: layer.compressedopener = opener diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go index 94fb767b..c3c56ff0 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/validate/image.go @@ -119,6 +119,12 @@ func validateLayers(img v1.Image, opt ...Option) error { udiffids := []v1.Hash{} sizes := []int64{} for i, layer := range layers { + if mt, err := layer.MediaType(); err != nil { + return fmt.Errorf("getting mediaType[%d]: %w", i, err) + } else if !mt.IsLayer() { + continue + } + cl, err := computeLayer(layer) if errors.Is(err, io.ErrUnexpectedEOF) { // Errored while reading tar content of layer because a header or @@ -153,19 +159,23 @@ func validateLayers(img v1.Image, opt ...Option) error { errs := []string{} for i, layer := range layers { - digest, err := layer.Digest() + mediaType, err := layer.MediaType() if err != nil { return err } - diffid, err := layer.DiffID() + if !mediaType.IsLayer() { + continue + } + + digest, err := layer.Digest() if err != nil { return err } - size, err := layer.Size() + diffid, err := layer.DiffID() if err != nil { return err } - mediaType, err := layer.MediaType() + size, err := layer.Size() if err != nil { return err } diff --git a/vendor/github.com/vbatts/tar-split/LICENSE b/vendor/github.com/vbatts/tar-split/LICENSE deleted file mode 100644 index ca03685b..00000000 --- a/vendor/github.com/vbatts/tar-split/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/common.go b/vendor/github.com/vbatts/tar-split/archive/tar/common.go deleted file mode 100644 index e687a08c..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/common.go +++ /dev/null @@ -1,724 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package tar implements access to tar archives. -// -// Tape archives (tar) are a file format for storing a sequence of files that -// can be read and written in a streaming manner. -// This package aims to cover most variations of the format, -// including those produced by GNU and BSD tar tools. -package tar - -import ( - "errors" - "fmt" - "math" - "os" - "path" - "reflect" - "strconv" - "strings" - "time" -) - -// BUG: Use of the Uid and Gid fields in Header could overflow on 32-bit -// architectures. If a large value is encountered when decoding, the result -// stored in Header will be the truncated version. - -var ( - ErrHeader = errors.New("archive/tar: invalid tar header") - ErrWriteTooLong = errors.New("archive/tar: write too long") - ErrFieldTooLong = errors.New("archive/tar: header field too long") - ErrWriteAfterClose = errors.New("archive/tar: write after close") - errMissData = errors.New("archive/tar: sparse file references non-existent data") - errUnrefData = errors.New("archive/tar: sparse file contains unreferenced data") - errWriteHole = errors.New("archive/tar: write non-NUL byte in sparse hole") - errSparseTooLong = errors.New("archive/tar: sparse map too long") -) - -type headerError []string - -func (he headerError) Error() string { - const prefix = "archive/tar: cannot encode header" - var ss []string - for _, s := range he { - if s != "" { - ss = append(ss, s) - } - } - if len(ss) == 0 { - return prefix - } - return fmt.Sprintf("%s: %v", prefix, strings.Join(ss, "; and ")) -} - -// Type flags for Header.Typeflag. -const ( - // Type '0' indicates a regular file. - TypeReg = '0' - TypeRegA = '\x00' // Deprecated: Use TypeReg instead. - - // Type '1' to '6' are header-only flags and may not have a data body. - TypeLink = '1' // Hard link - TypeSymlink = '2' // Symbolic link - TypeChar = '3' // Character device node - TypeBlock = '4' // Block device node - TypeDir = '5' // Directory - TypeFifo = '6' // FIFO node - - // Type '7' is reserved. - TypeCont = '7' - - // Type 'x' is used by the PAX format to store key-value records that - // are only relevant to the next file. - // This package transparently handles these types. - TypeXHeader = 'x' - - // Type 'g' is used by the PAX format to store key-value records that - // are relevant to all subsequent files. - // This package only supports parsing and composing such headers, - // but does not currently support persisting the global state across files. - TypeXGlobalHeader = 'g' - - // Type 'S' indicates a sparse file in the GNU format. - TypeGNUSparse = 'S' - - // Types 'L' and 'K' are used by the GNU format for a meta file - // used to store the path or link name for the next file. - // This package transparently handles these types. - TypeGNULongName = 'L' - TypeGNULongLink = 'K' -) - -// Keywords for PAX extended header records. -const ( - paxNone = "" // Indicates that no PAX key is suitable - paxPath = "path" - paxLinkpath = "linkpath" - paxSize = "size" - paxUid = "uid" - paxGid = "gid" - paxUname = "uname" - paxGname = "gname" - paxMtime = "mtime" - paxAtime = "atime" - paxCtime = "ctime" // Removed from later revision of PAX spec, but was valid - paxCharset = "charset" // Currently unused - paxComment = "comment" // Currently unused - - paxSchilyXattr = "SCHILY.xattr." - - // Keywords for GNU sparse files in a PAX extended header. - paxGNUSparse = "GNU.sparse." - paxGNUSparseNumBlocks = "GNU.sparse.numblocks" - paxGNUSparseOffset = "GNU.sparse.offset" - paxGNUSparseNumBytes = "GNU.sparse.numbytes" - paxGNUSparseMap = "GNU.sparse.map" - paxGNUSparseName = "GNU.sparse.name" - paxGNUSparseMajor = "GNU.sparse.major" - paxGNUSparseMinor = "GNU.sparse.minor" - paxGNUSparseSize = "GNU.sparse.size" - paxGNUSparseRealSize = "GNU.sparse.realsize" -) - -// basicKeys is a set of the PAX keys for which we have built-in support. -// This does not contain "charset" or "comment", which are both PAX-specific, -// so adding them as first-class features of Header is unlikely. -// Users can use the PAXRecords field to set it themselves. -var basicKeys = map[string]bool{ - paxPath: true, paxLinkpath: true, paxSize: true, paxUid: true, paxGid: true, - paxUname: true, paxGname: true, paxMtime: true, paxAtime: true, paxCtime: true, -} - -// A Header represents a single header in a tar archive. -// Some fields may not be populated. -// -// For forward compatibility, users that retrieve a Header from Reader.Next, -// mutate it in some ways, and then pass it back to Writer.WriteHeader -// should do so by creating a new Header and copying the fields -// that they are interested in preserving. -type Header struct { - // Typeflag is the type of header entry. - // The zero value is automatically promoted to either TypeReg or TypeDir - // depending on the presence of a trailing slash in Name. - Typeflag byte - - Name string // Name of file entry - Linkname string // Target name of link (valid for TypeLink or TypeSymlink) - - Size int64 // Logical file size in bytes - Mode int64 // Permission and mode bits - Uid int // User ID of owner - Gid int // Group ID of owner - Uname string // User name of owner - Gname string // Group name of owner - - // If the Format is unspecified, then Writer.WriteHeader rounds ModTime - // to the nearest second and ignores the AccessTime and ChangeTime fields. - // - // To use AccessTime or ChangeTime, specify the Format as PAX or GNU. - // To use sub-second resolution, specify the Format as PAX. - ModTime time.Time // Modification time - AccessTime time.Time // Access time (requires either PAX or GNU support) - ChangeTime time.Time // Change time (requires either PAX or GNU support) - - Devmajor int64 // Major device number (valid for TypeChar or TypeBlock) - Devminor int64 // Minor device number (valid for TypeChar or TypeBlock) - - // Xattrs stores extended attributes as PAX records under the - // "SCHILY.xattr." namespace. - // - // The following are semantically equivalent: - // h.Xattrs[key] = value - // h.PAXRecords["SCHILY.xattr."+key] = value - // - // When Writer.WriteHeader is called, the contents of Xattrs will take - // precedence over those in PAXRecords. - // - // Deprecated: Use PAXRecords instead. - Xattrs map[string]string - - // PAXRecords is a map of PAX extended header records. - // - // User-defined records should have keys of the following form: - // VENDOR.keyword - // Where VENDOR is some namespace in all uppercase, and keyword may - // not contain the '=' character (e.g., "GOLANG.pkg.version"). - // The key and value should be non-empty UTF-8 strings. - // - // When Writer.WriteHeader is called, PAX records derived from the - // other fields in Header take precedence over PAXRecords. - PAXRecords map[string]string - - // Format specifies the format of the tar header. - // - // This is set by Reader.Next as a best-effort guess at the format. - // Since the Reader liberally reads some non-compliant files, - // it is possible for this to be FormatUnknown. - // - // If the format is unspecified when Writer.WriteHeader is called, - // then it uses the first format (in the order of USTAR, PAX, GNU) - // capable of encoding this Header (see Format). - Format Format -} - -// sparseEntry represents a Length-sized fragment at Offset in the file. -type sparseEntry struct{ Offset, Length int64 } - -func (s sparseEntry) endOffset() int64 { return s.Offset + s.Length } - -// A sparse file can be represented as either a sparseDatas or a sparseHoles. -// As long as the total size is known, they are equivalent and one can be -// converted to the other form and back. The various tar formats with sparse -// file support represent sparse files in the sparseDatas form. That is, they -// specify the fragments in the file that has data, and treat everything else as -// having zero bytes. As such, the encoding and decoding logic in this package -// deals with sparseDatas. -// -// However, the external API uses sparseHoles instead of sparseDatas because the -// zero value of sparseHoles logically represents a normal file (i.e., there are -// no holes in it). On the other hand, the zero value of sparseDatas implies -// that the file has no data in it, which is rather odd. -// -// As an example, if the underlying raw file contains the 10-byte data: -// var compactFile = "abcdefgh" -// -// And the sparse map has the following entries: -// var spd sparseDatas = []sparseEntry{ -// {Offset: 2, Length: 5}, // Data fragment for 2..6 -// {Offset: 18, Length: 3}, // Data fragment for 18..20 -// } -// var sph sparseHoles = []sparseEntry{ -// {Offset: 0, Length: 2}, // Hole fragment for 0..1 -// {Offset: 7, Length: 11}, // Hole fragment for 7..17 -// {Offset: 21, Length: 4}, // Hole fragment for 21..24 -// } -// -// Then the content of the resulting sparse file with a Header.Size of 25 is: -// var sparseFile = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4 -type ( - sparseDatas []sparseEntry - sparseHoles []sparseEntry -) - -// validateSparseEntries reports whether sp is a valid sparse map. -// It does not matter whether sp represents data fragments or hole fragments. -func validateSparseEntries(sp []sparseEntry, size int64) bool { - // Validate all sparse entries. These are the same checks as performed by - // the BSD tar utility. - if size < 0 { - return false - } - var pre sparseEntry - for _, cur := range sp { - switch { - case cur.Offset < 0 || cur.Length < 0: - return false // Negative values are never okay - case cur.Offset > math.MaxInt64-cur.Length: - return false // Integer overflow with large length - case cur.endOffset() > size: - return false // Region extends beyond the actual size - case pre.endOffset() > cur.Offset: - return false // Regions cannot overlap and must be in order - } - pre = cur - } - return true -} - -// alignSparseEntries mutates src and returns dst where each fragment's -// starting offset is aligned up to the nearest block edge, and each -// ending offset is aligned down to the nearest block edge. -// -// Even though the Go tar Reader and the BSD tar utility can handle entries -// with arbitrary offsets and lengths, the GNU tar utility can only handle -// offsets and lengths that are multiples of blockSize. -func alignSparseEntries(src []sparseEntry, size int64) []sparseEntry { - dst := src[:0] - for _, s := range src { - pos, end := s.Offset, s.endOffset() - pos += blockPadding(+pos) // Round-up to nearest blockSize - if end != size { - end -= blockPadding(-end) // Round-down to nearest blockSize - } - if pos < end { - dst = append(dst, sparseEntry{Offset: pos, Length: end - pos}) - } - } - return dst -} - -// invertSparseEntries converts a sparse map from one form to the other. -// If the input is sparseHoles, then it will output sparseDatas and vice-versa. -// The input must have been already validated. -// -// This function mutates src and returns a normalized map where: -// * adjacent fragments are coalesced together -// * only the last fragment may be empty -// * the endOffset of the last fragment is the total size -func invertSparseEntries(src []sparseEntry, size int64) []sparseEntry { - dst := src[:0] - var pre sparseEntry - for _, cur := range src { - if cur.Length == 0 { - continue // Skip empty fragments - } - pre.Length = cur.Offset - pre.Offset - if pre.Length > 0 { - dst = append(dst, pre) // Only add non-empty fragments - } - pre.Offset = cur.endOffset() - } - pre.Length = size - pre.Offset // Possibly the only empty fragment - return append(dst, pre) -} - -// fileState tracks the number of logical (includes sparse holes) and physical -// (actual in tar archive) bytes remaining for the current file. -// -// Invariant: LogicalRemaining >= PhysicalRemaining -type fileState interface { - LogicalRemaining() int64 - PhysicalRemaining() int64 -} - -// allowedFormats determines which formats can be used. -// The value returned is the logical OR of multiple possible formats. -// If the value is FormatUnknown, then the input Header cannot be encoded -// and an error is returned explaining why. -// -// As a by-product of checking the fields, this function returns paxHdrs, which -// contain all fields that could not be directly encoded. -// A value receiver ensures that this method does not mutate the source Header. -func (h Header) allowedFormats() (format Format, paxHdrs map[string]string, err error) { - format = FormatUSTAR | FormatPAX | FormatGNU - paxHdrs = make(map[string]string) - - var whyNoUSTAR, whyNoPAX, whyNoGNU string - var preferPAX bool // Prefer PAX over USTAR - verifyString := func(s string, size int, name, paxKey string) { - // NUL-terminator is optional for path and linkpath. - // Technically, it is required for uname and gname, - // but neither GNU nor BSD tar checks for it. - tooLong := len(s) > size - allowLongGNU := paxKey == paxPath || paxKey == paxLinkpath - if hasNUL(s) || (tooLong && !allowLongGNU) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%q", name, s) - format.mustNotBe(FormatGNU) - } - if !isASCII(s) || tooLong { - canSplitUSTAR := paxKey == paxPath - if _, _, ok := splitUSTARPath(s); !canSplitUSTAR || !ok { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%q", name, s) - format.mustNotBe(FormatUSTAR) - } - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%q", name, s) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = s - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == s { - paxHdrs[paxKey] = v - } - } - verifyNumeric := func(n int64, size int, name, paxKey string) { - if !fitsInBase256(size, n) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%d", name, n) - format.mustNotBe(FormatGNU) - } - if !fitsInOctal(size, n) { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%d", name, n) - format.mustNotBe(FormatUSTAR) - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%d", name, n) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = strconv.FormatInt(n, 10) - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == strconv.FormatInt(n, 10) { - paxHdrs[paxKey] = v - } - } - verifyTime := func(ts time.Time, size int, name, paxKey string) { - if ts.IsZero() { - return // Always okay - } - if !fitsInBase256(size, ts.Unix()) { - whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%v", name, ts) - format.mustNotBe(FormatGNU) - } - isMtime := paxKey == paxMtime - fitsOctal := fitsInOctal(size, ts.Unix()) - if (isMtime && !fitsOctal) || !isMtime { - whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%v", name, ts) - format.mustNotBe(FormatUSTAR) - } - needsNano := ts.Nanosecond() != 0 - if !isMtime || !fitsOctal || needsNano { - preferPAX = true // USTAR may truncate sub-second measurements - if paxKey == paxNone { - whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%v", name, ts) - format.mustNotBe(FormatPAX) - } else { - paxHdrs[paxKey] = formatPAXTime(ts) - } - } - if v, ok := h.PAXRecords[paxKey]; ok && v == formatPAXTime(ts) { - paxHdrs[paxKey] = v - } - } - - // Check basic fields. - var blk block - v7 := blk.V7() - ustar := blk.USTAR() - gnu := blk.GNU() - verifyString(h.Name, len(v7.Name()), "Name", paxPath) - verifyString(h.Linkname, len(v7.LinkName()), "Linkname", paxLinkpath) - verifyString(h.Uname, len(ustar.UserName()), "Uname", paxUname) - verifyString(h.Gname, len(ustar.GroupName()), "Gname", paxGname) - verifyNumeric(h.Mode, len(v7.Mode()), "Mode", paxNone) - verifyNumeric(int64(h.Uid), len(v7.UID()), "Uid", paxUid) - verifyNumeric(int64(h.Gid), len(v7.GID()), "Gid", paxGid) - verifyNumeric(h.Size, len(v7.Size()), "Size", paxSize) - verifyNumeric(h.Devmajor, len(ustar.DevMajor()), "Devmajor", paxNone) - verifyNumeric(h.Devminor, len(ustar.DevMinor()), "Devminor", paxNone) - verifyTime(h.ModTime, len(v7.ModTime()), "ModTime", paxMtime) - verifyTime(h.AccessTime, len(gnu.AccessTime()), "AccessTime", paxAtime) - verifyTime(h.ChangeTime, len(gnu.ChangeTime()), "ChangeTime", paxCtime) - - // Check for header-only types. - var whyOnlyPAX, whyOnlyGNU string - switch h.Typeflag { - case TypeReg, TypeChar, TypeBlock, TypeFifo, TypeGNUSparse: - // Exclude TypeLink and TypeSymlink, since they may reference directories. - if strings.HasSuffix(h.Name, "/") { - return FormatUnknown, nil, headerError{"filename may not have trailing slash"} - } - case TypeXHeader, TypeGNULongName, TypeGNULongLink: - return FormatUnknown, nil, headerError{"cannot manually encode TypeXHeader, TypeGNULongName, or TypeGNULongLink headers"} - case TypeXGlobalHeader: - h2 := Header{Name: h.Name, Typeflag: h.Typeflag, Xattrs: h.Xattrs, PAXRecords: h.PAXRecords, Format: h.Format} - if !reflect.DeepEqual(h, h2) { - return FormatUnknown, nil, headerError{"only PAXRecords should be set for TypeXGlobalHeader"} - } - whyOnlyPAX = "only PAX supports TypeXGlobalHeader" - format.mayOnlyBe(FormatPAX) - } - if !isHeaderOnlyType(h.Typeflag) && h.Size < 0 { - return FormatUnknown, nil, headerError{"negative size on header-only type"} - } - - // Check PAX records. - if len(h.Xattrs) > 0 { - for k, v := range h.Xattrs { - paxHdrs[paxSchilyXattr+k] = v - } - whyOnlyPAX = "only PAX supports Xattrs" - format.mayOnlyBe(FormatPAX) - } - if len(h.PAXRecords) > 0 { - for k, v := range h.PAXRecords { - switch _, exists := paxHdrs[k]; { - case exists: - continue // Do not overwrite existing records - case h.Typeflag == TypeXGlobalHeader: - paxHdrs[k] = v // Copy all records - case !basicKeys[k] && !strings.HasPrefix(k, paxGNUSparse): - paxHdrs[k] = v // Ignore local records that may conflict - } - } - whyOnlyPAX = "only PAX supports PAXRecords" - format.mayOnlyBe(FormatPAX) - } - for k, v := range paxHdrs { - if !validPAXRecord(k, v) { - return FormatUnknown, nil, headerError{fmt.Sprintf("invalid PAX record: %q", k+" = "+v)} - } - } - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Check sparse files. - if len(h.SparseHoles) > 0 || h.Typeflag == TypeGNUSparse { - if isHeaderOnlyType(h.Typeflag) { - return FormatUnknown, nil, headerError{"header-only type cannot be sparse"} - } - if !validateSparseEntries(h.SparseHoles, h.Size) { - return FormatUnknown, nil, headerError{"invalid sparse holes"} - } - if h.Typeflag == TypeGNUSparse { - whyOnlyGNU = "only GNU supports TypeGNUSparse" - format.mayOnlyBe(FormatGNU) - } else { - whyNoGNU = "GNU supports sparse files only with TypeGNUSparse" - format.mustNotBe(FormatGNU) - } - whyNoUSTAR = "USTAR does not support sparse files" - format.mustNotBe(FormatUSTAR) - } - */ - - // Check desired format. - if wantFormat := h.Format; wantFormat != FormatUnknown { - if wantFormat.has(FormatPAX) && !preferPAX { - wantFormat.mayBe(FormatUSTAR) // PAX implies USTAR allowed too - } - format.mayOnlyBe(wantFormat) // Set union of formats allowed and format wanted - } - if format == FormatUnknown { - switch h.Format { - case FormatUSTAR: - err = headerError{"Format specifies USTAR", whyNoUSTAR, whyOnlyPAX, whyOnlyGNU} - case FormatPAX: - err = headerError{"Format specifies PAX", whyNoPAX, whyOnlyGNU} - case FormatGNU: - err = headerError{"Format specifies GNU", whyNoGNU, whyOnlyPAX} - default: - err = headerError{whyNoUSTAR, whyNoPAX, whyNoGNU, whyOnlyPAX, whyOnlyGNU} - } - } - return format, paxHdrs, err -} - -// FileInfo returns an os.FileInfo for the Header. -func (h *Header) FileInfo() os.FileInfo { - return headerFileInfo{h} -} - -// headerFileInfo implements os.FileInfo. -type headerFileInfo struct { - h *Header -} - -func (fi headerFileInfo) Size() int64 { return fi.h.Size } -func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } -func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime } -func (fi headerFileInfo) Sys() interface{} { return fi.h } - -// Name returns the base name of the file. -func (fi headerFileInfo) Name() string { - if fi.IsDir() { - return path.Base(path.Clean(fi.h.Name)) - } - return path.Base(fi.h.Name) -} - -// Mode returns the permission and mode bits for the headerFileInfo. -func (fi headerFileInfo) Mode() (mode os.FileMode) { - // Set file permission bits. - mode = os.FileMode(fi.h.Mode).Perm() - - // Set setuid, setgid and sticky bits. - if fi.h.Mode&c_ISUID != 0 { - mode |= os.ModeSetuid - } - if fi.h.Mode&c_ISGID != 0 { - mode |= os.ModeSetgid - } - if fi.h.Mode&c_ISVTX != 0 { - mode |= os.ModeSticky - } - - // Set file mode bits; clear perm, setuid, setgid, and sticky bits. - switch m := os.FileMode(fi.h.Mode) &^ 07777; m { - case c_ISDIR: - mode |= os.ModeDir - case c_ISFIFO: - mode |= os.ModeNamedPipe - case c_ISLNK: - mode |= os.ModeSymlink - case c_ISBLK: - mode |= os.ModeDevice - case c_ISCHR: - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case c_ISSOCK: - mode |= os.ModeSocket - } - - switch fi.h.Typeflag { - case TypeSymlink: - mode |= os.ModeSymlink - case TypeChar: - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case TypeBlock: - mode |= os.ModeDevice - case TypeDir: - mode |= os.ModeDir - case TypeFifo: - mode |= os.ModeNamedPipe - } - - return mode -} - -// sysStat, if non-nil, populates h from system-dependent fields of fi. -var sysStat func(fi os.FileInfo, h *Header) error - -const ( - // Mode constants from the USTAR spec: - // See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 - c_ISUID = 04000 // Set uid - c_ISGID = 02000 // Set gid - c_ISVTX = 01000 // Save text (sticky bit) - - // Common Unix mode constants; these are not defined in any common tar standard. - // Header.FileInfo understands these, but FileInfoHeader will never produce these. - c_ISDIR = 040000 // Directory - c_ISFIFO = 010000 // FIFO - c_ISREG = 0100000 // Regular file - c_ISLNK = 0120000 // Symbolic link - c_ISBLK = 060000 // Block special file - c_ISCHR = 020000 // Character special file - c_ISSOCK = 0140000 // Socket -) - -// FileInfoHeader creates a partially-populated Header from fi. -// If fi describes a symlink, FileInfoHeader records link as the link target. -// If fi describes a directory, a slash is appended to the name. -// -// Since os.FileInfo's Name method only returns the base name of -// the file it describes, it may be necessary to modify Header.Name -// to provide the full path name of the file. -func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { - if fi == nil { - return nil, errors.New("archive/tar: FileInfo is nil") - } - fm := fi.Mode() - h := &Header{ - Name: fi.Name(), - ModTime: fi.ModTime(), - Mode: int64(fm.Perm()), // or'd with c_IS* constants later - } - switch { - case fm.IsRegular(): - h.Typeflag = TypeReg - h.Size = fi.Size() - case fi.IsDir(): - h.Typeflag = TypeDir - h.Name += "/" - case fm&os.ModeSymlink != 0: - h.Typeflag = TypeSymlink - h.Linkname = link - case fm&os.ModeDevice != 0: - if fm&os.ModeCharDevice != 0 { - h.Typeflag = TypeChar - } else { - h.Typeflag = TypeBlock - } - case fm&os.ModeNamedPipe != 0: - h.Typeflag = TypeFifo - case fm&os.ModeSocket != 0: - return nil, fmt.Errorf("archive/tar: sockets not supported") - default: - return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) - } - if fm&os.ModeSetuid != 0 { - h.Mode |= c_ISUID - } - if fm&os.ModeSetgid != 0 { - h.Mode |= c_ISGID - } - if fm&os.ModeSticky != 0 { - h.Mode |= c_ISVTX - } - // If possible, populate additional fields from OS-specific - // FileInfo fields. - if sys, ok := fi.Sys().(*Header); ok { - // This FileInfo came from a Header (not the OS). Use the - // original Header to populate all remaining fields. - h.Uid = sys.Uid - h.Gid = sys.Gid - h.Uname = sys.Uname - h.Gname = sys.Gname - h.AccessTime = sys.AccessTime - h.ChangeTime = sys.ChangeTime - if sys.Xattrs != nil { - h.Xattrs = make(map[string]string) - for k, v := range sys.Xattrs { - h.Xattrs[k] = v - } - } - if sys.Typeflag == TypeLink { - // hard link - h.Typeflag = TypeLink - h.Size = 0 - h.Linkname = sys.Linkname - } - if sys.PAXRecords != nil { - h.PAXRecords = make(map[string]string) - for k, v := range sys.PAXRecords { - h.PAXRecords[k] = v - } - } - } - if sysStat != nil { - return h, sysStat(fi, h) - } - return h, nil -} - -// isHeaderOnlyType checks if the given type flag is of the type that has no -// data section even if a size is specified. -func isHeaderOnlyType(flag byte) bool { - switch flag { - case TypeLink, TypeSymlink, TypeChar, TypeBlock, TypeDir, TypeFifo: - return true - default: - return false - } -} - -func min(a, b int64) int64 { - if a < b { - return a - } - return b -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/format.go b/vendor/github.com/vbatts/tar-split/archive/tar/format.go deleted file mode 100644 index 60977980..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/format.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import "strings" - -// Format represents the tar archive format. -// -// The original tar format was introduced in Unix V7. -// Since then, there have been multiple competing formats attempting to -// standardize or extend the V7 format to overcome its limitations. -// The most common formats are the USTAR, PAX, and GNU formats, -// each with their own advantages and limitations. -// -// The following table captures the capabilities of each format: -// -// | USTAR | PAX | GNU -// ------------------+--------+-----------+---------- -// Name | 256B | unlimited | unlimited -// Linkname | 100B | unlimited | unlimited -// Size | uint33 | unlimited | uint89 -// Mode | uint21 | uint21 | uint57 -// Uid/Gid | uint21 | unlimited | uint57 -// Uname/Gname | 32B | unlimited | 32B -// ModTime | uint33 | unlimited | int89 -// AccessTime | n/a | unlimited | int89 -// ChangeTime | n/a | unlimited | int89 -// Devmajor/Devminor | uint21 | uint21 | uint57 -// ------------------+--------+-----------+---------- -// string encoding | ASCII | UTF-8 | binary -// sub-second times | no | yes | no -// sparse files | no | yes | yes -// -// The table's upper portion shows the Header fields, where each format reports -// the maximum number of bytes allowed for each string field and -// the integer type used to store each numeric field -// (where timestamps are stored as the number of seconds since the Unix epoch). -// -// The table's lower portion shows specialized features of each format, -// such as supported string encodings, support for sub-second timestamps, -// or support for sparse files. -// -// The Writer currently provides no support for sparse files. -type Format int - -// Constants to identify various tar formats. -const ( - // Deliberately hide the meaning of constants from public API. - _ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc... - - // FormatUnknown indicates that the format is unknown. - FormatUnknown - - // The format of the original Unix V7 tar tool prior to standardization. - formatV7 - - // FormatUSTAR represents the USTAR header format defined in POSIX.1-1988. - // - // While this format is compatible with most tar readers, - // the format has several limitations making it unsuitable for some usages. - // Most notably, it cannot support sparse files, files larger than 8GiB, - // filenames larger than 256 characters, and non-ASCII filenames. - // - // Reference: - // http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 - FormatUSTAR - - // FormatPAX represents the PAX header format defined in POSIX.1-2001. - // - // PAX extends USTAR by writing a special file with Typeflag TypeXHeader - // preceding the original header. This file contains a set of key-value - // records, which are used to overcome USTAR's shortcomings, in addition to - // providing the ability to have sub-second resolution for timestamps. - // - // Some newer formats add their own extensions to PAX by defining their - // own keys and assigning certain semantic meaning to the associated values. - // For example, sparse file support in PAX is implemented using keys - // defined by the GNU manual (e.g., "GNU.sparse.map"). - // - // Reference: - // http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html - FormatPAX - - // FormatGNU represents the GNU header format. - // - // The GNU header format is older than the USTAR and PAX standards and - // is not compatible with them. The GNU format supports - // arbitrary file sizes, filenames of arbitrary encoding and length, - // sparse files, and other features. - // - // It is recommended that PAX be chosen over GNU unless the target - // application can only parse GNU formatted archives. - // - // Reference: - // https://www.gnu.org/software/tar/manual/html_node/Standard.html - FormatGNU - - // Schily's tar format, which is incompatible with USTAR. - // This does not cover STAR extensions to the PAX format; these fall under - // the PAX format. - formatSTAR - - formatMax -) - -func (f Format) has(f2 Format) bool { return f&f2 != 0 } -func (f *Format) mayBe(f2 Format) { *f |= f2 } -func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 } -func (f *Format) mustNotBe(f2 Format) { *f &^= f2 } - -var formatNames = map[Format]string{ - formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR", -} - -func (f Format) String() string { - var ss []string - for f2 := Format(1); f2 < formatMax; f2 <<= 1 { - if f.has(f2) { - ss = append(ss, formatNames[f2]) - } - } - switch len(ss) { - case 0: - return "" - case 1: - return ss[0] - default: - return "(" + strings.Join(ss, " | ") + ")" - } -} - -// Magics used to identify various formats. -const ( - magicGNU, versionGNU = "ustar ", " \x00" - magicUSTAR, versionUSTAR = "ustar\x00", "00" - trailerSTAR = "tar\x00" -) - -// Size constants from various tar specifications. -const ( - blockSize = 512 // Size of each block in a tar stream - nameSize = 100 // Max length of the name field in USTAR format - prefixSize = 155 // Max length of the prefix field in USTAR format - - // Max length of a special file (PAX header, GNU long name or link). - // This matches the limit used by libarchive. - maxSpecialFileSize = 1 << 20 -) - -// blockPadding computes the number of bytes needed to pad offset up to the -// nearest block edge where 0 <= n < blockSize. -func blockPadding(offset int64) (n int64) { - return -offset & (blockSize - 1) -} - -var zeroBlock block - -type block [blockSize]byte - -// Convert block to any number of formats. -func (b *block) V7() *headerV7 { return (*headerV7)(b) } -func (b *block) GNU() *headerGNU { return (*headerGNU)(b) } -func (b *block) STAR() *headerSTAR { return (*headerSTAR)(b) } -func (b *block) USTAR() *headerUSTAR { return (*headerUSTAR)(b) } -func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) } - -// GetFormat checks that the block is a valid tar header based on the checksum. -// It then attempts to guess the specific format based on magic values. -// If the checksum fails, then FormatUnknown is returned. -func (b *block) GetFormat() Format { - // Verify checksum. - var p parser - value := p.parseOctal(b.V7().Chksum()) - chksum1, chksum2 := b.ComputeChecksum() - if p.err != nil || (value != chksum1 && value != chksum2) { - return FormatUnknown - } - - // Guess the magic values. - magic := string(b.USTAR().Magic()) - version := string(b.USTAR().Version()) - trailer := string(b.STAR().Trailer()) - switch { - case magic == magicUSTAR && trailer == trailerSTAR: - return formatSTAR - case magic == magicUSTAR: - return FormatUSTAR | FormatPAX - case magic == magicGNU && version == versionGNU: - return FormatGNU - default: - return formatV7 - } -} - -// SetFormat writes the magic values necessary for specified format -// and then updates the checksum accordingly. -func (b *block) SetFormat(format Format) { - // Set the magic values. - switch { - case format.has(formatV7): - // Do nothing. - case format.has(FormatGNU): - copy(b.GNU().Magic(), magicGNU) - copy(b.GNU().Version(), versionGNU) - case format.has(formatSTAR): - copy(b.STAR().Magic(), magicUSTAR) - copy(b.STAR().Version(), versionUSTAR) - copy(b.STAR().Trailer(), trailerSTAR) - case format.has(FormatUSTAR | FormatPAX): - copy(b.USTAR().Magic(), magicUSTAR) - copy(b.USTAR().Version(), versionUSTAR) - default: - panic("invalid format") - } - - // Update checksum. - // This field is special in that it is terminated by a NULL then space. - var f formatter - field := b.V7().Chksum() - chksum, _ := b.ComputeChecksum() // Possible values are 256..128776 - f.formatOctal(field[:7], chksum) // Never fails since 128776 < 262143 - field[7] = ' ' -} - -// ComputeChecksum computes the checksum for the header block. -// POSIX specifies a sum of the unsigned byte values, but the Sun tar used -// signed byte values. -// We compute and return both. -func (b *block) ComputeChecksum() (unsigned, signed int64) { - for i, c := range b { - if 148 <= i && i < 156 { - c = ' ' // Treat the checksum field itself as all spaces. - } - unsigned += int64(c) - signed += int64(int8(c)) - } - return unsigned, signed -} - -// Reset clears the block with all zeros. -func (b *block) Reset() { - *b = block{} -} - -type headerV7 [blockSize]byte - -func (h *headerV7) Name() []byte { return h[000:][:100] } -func (h *headerV7) Mode() []byte { return h[100:][:8] } -func (h *headerV7) UID() []byte { return h[108:][:8] } -func (h *headerV7) GID() []byte { return h[116:][:8] } -func (h *headerV7) Size() []byte { return h[124:][:12] } -func (h *headerV7) ModTime() []byte { return h[136:][:12] } -func (h *headerV7) Chksum() []byte { return h[148:][:8] } -func (h *headerV7) TypeFlag() []byte { return h[156:][:1] } -func (h *headerV7) LinkName() []byte { return h[157:][:100] } - -type headerGNU [blockSize]byte - -func (h *headerGNU) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerGNU) Magic() []byte { return h[257:][:6] } -func (h *headerGNU) Version() []byte { return h[263:][:2] } -func (h *headerGNU) UserName() []byte { return h[265:][:32] } -func (h *headerGNU) GroupName() []byte { return h[297:][:32] } -func (h *headerGNU) DevMajor() []byte { return h[329:][:8] } -func (h *headerGNU) DevMinor() []byte { return h[337:][:8] } -func (h *headerGNU) AccessTime() []byte { return h[345:][:12] } -func (h *headerGNU) ChangeTime() []byte { return h[357:][:12] } -func (h *headerGNU) Sparse() sparseArray { return (sparseArray)(h[386:][:24*4+1]) } -func (h *headerGNU) RealSize() []byte { return h[483:][:12] } - -type headerSTAR [blockSize]byte - -func (h *headerSTAR) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerSTAR) Magic() []byte { return h[257:][:6] } -func (h *headerSTAR) Version() []byte { return h[263:][:2] } -func (h *headerSTAR) UserName() []byte { return h[265:][:32] } -func (h *headerSTAR) GroupName() []byte { return h[297:][:32] } -func (h *headerSTAR) DevMajor() []byte { return h[329:][:8] } -func (h *headerSTAR) DevMinor() []byte { return h[337:][:8] } -func (h *headerSTAR) Prefix() []byte { return h[345:][:131] } -func (h *headerSTAR) AccessTime() []byte { return h[476:][:12] } -func (h *headerSTAR) ChangeTime() []byte { return h[488:][:12] } -func (h *headerSTAR) Trailer() []byte { return h[508:][:4] } - -type headerUSTAR [blockSize]byte - -func (h *headerUSTAR) V7() *headerV7 { return (*headerV7)(h) } -func (h *headerUSTAR) Magic() []byte { return h[257:][:6] } -func (h *headerUSTAR) Version() []byte { return h[263:][:2] } -func (h *headerUSTAR) UserName() []byte { return h[265:][:32] } -func (h *headerUSTAR) GroupName() []byte { return h[297:][:32] } -func (h *headerUSTAR) DevMajor() []byte { return h[329:][:8] } -func (h *headerUSTAR) DevMinor() []byte { return h[337:][:8] } -func (h *headerUSTAR) Prefix() []byte { return h[345:][:155] } - -type sparseArray []byte - -func (s sparseArray) Entry(i int) sparseElem { return (sparseElem)(s[i*24:]) } -func (s sparseArray) IsExtended() []byte { return s[24*s.MaxEntries():][:1] } -func (s sparseArray) MaxEntries() int { return len(s) / 24 } - -type sparseElem []byte - -func (s sparseElem) Offset() []byte { return s[00:][:12] } -func (s sparseElem) Length() []byte { return s[12:][:12] } diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go b/vendor/github.com/vbatts/tar-split/archive/tar/reader.go deleted file mode 100644 index a645c416..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go +++ /dev/null @@ -1,944 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "io" - "strconv" - "strings" - "time" -) - -// Reader provides sequential access to the contents of a tar archive. -// Reader.Next advances to the next file in the archive (including the first), -// and then Reader can be treated as an io.Reader to access the file's data. -type Reader struct { - r io.Reader - pad int64 // Amount of padding (ignored) after current file entry - curr fileReader // Reader for current file entry - blk block // Buffer to use as temporary local storage - - // err is a persistent error. - // It is only the responsibility of every exported method of Reader to - // ensure that this error is sticky. - err error - - RawAccounting bool // Whether to enable the access needed to reassemble the tar from raw bytes. Some performance/memory hit for this. - rawBytes *bytes.Buffer // last raw bits -} - -type fileReader interface { - io.Reader - fileState - - WriteTo(io.Writer) (int64, error) -} - -// RawBytes accesses the raw bytes of the archive, apart from the file payload itself. -// This includes the header and padding. -// -// # This call resets the current rawbytes buffer -// -// Only when RawAccounting is enabled, otherwise this returns nil -func (tr *Reader) RawBytes() []byte { - if !tr.RawAccounting { - return nil - } - if tr.rawBytes == nil { - tr.rawBytes = bytes.NewBuffer(nil) - } - defer tr.rawBytes.Reset() // if we've read them, then flush them. - - return tr.rawBytes.Bytes() - -} - -// ExpectedPadding returns the number of bytes of padding expected after the last header returned by Next() -func (tr *Reader) ExpectedPadding() int64 { - return tr.pad -} - -// NewReader creates a new Reader reading from r. -func NewReader(r io.Reader) *Reader { - return &Reader{r: r, curr: ®FileReader{r, 0}} -} - -// Next advances to the next entry in the tar archive. -// The Header.Size determines how many bytes can be read for the next file. -// Any remaining data in the current file is automatically discarded. -// -// io.EOF is returned at the end of the input. -func (tr *Reader) Next() (*Header, error) { - if tr.err != nil { - return nil, tr.err - } - hdr, err := tr.next() - tr.err = err - return hdr, err -} - -func (tr *Reader) next() (*Header, error) { - var paxHdrs map[string]string - var gnuLongName, gnuLongLink string - - if tr.RawAccounting { - if tr.rawBytes == nil { - tr.rawBytes = bytes.NewBuffer(nil) - } else { - tr.rawBytes.Reset() - } - } - - // Externally, Next iterates through the tar archive as if it is a series of - // files. Internally, the tar format often uses fake "files" to add meta - // data that describes the next file. These meta data "files" should not - // normally be visible to the outside. As such, this loop iterates through - // one or more "header files" until it finds a "normal file". - format := FormatUSTAR | FormatPAX | FormatGNU - for { - // Discard the remainder of the file and any padding. - if err := discard(tr, tr.curr.PhysicalRemaining()); err != nil { - return nil, err - } - n, err := tryReadFull(tr.r, tr.blk[:tr.pad]) - if err != nil { - return nil, err - } - if tr.RawAccounting { - tr.rawBytes.Write(tr.blk[:n]) - } - tr.pad = 0 - - hdr, rawHdr, err := tr.readHeader() - if err != nil { - return nil, err - } - if err := tr.handleRegularFile(hdr); err != nil { - return nil, err - } - format.mayOnlyBe(hdr.Format) - - // Check for PAX/GNU special headers and files. - switch hdr.Typeflag { - case TypeXHeader, TypeXGlobalHeader: - format.mayOnlyBe(FormatPAX) - paxHdrs, err = parsePAX(tr) - if err != nil { - return nil, err - } - if hdr.Typeflag == TypeXGlobalHeader { - if err = mergePAX(hdr, paxHdrs); err != nil { - return nil, err - } - return &Header{ - Name: hdr.Name, - Typeflag: hdr.Typeflag, - Xattrs: hdr.Xattrs, - PAXRecords: hdr.PAXRecords, - Format: format, - }, nil - } - continue // This is a meta header affecting the next header - case TypeGNULongName, TypeGNULongLink: - format.mayOnlyBe(FormatGNU) - realname, err := readSpecialFile(tr) - if err != nil { - return nil, err - } - - if tr.RawAccounting { - tr.rawBytes.Write(realname) - } - - var p parser - switch hdr.Typeflag { - case TypeGNULongName: - gnuLongName = p.parseString(realname) - case TypeGNULongLink: - gnuLongLink = p.parseString(realname) - } - continue // This is a meta header affecting the next header - default: - // The old GNU sparse format is handled here since it is technically - // just a regular file with additional attributes. - - if err := mergePAX(hdr, paxHdrs); err != nil { - return nil, err - } - if gnuLongName != "" { - hdr.Name = gnuLongName - } - if gnuLongLink != "" { - hdr.Linkname = gnuLongLink - } - if hdr.Typeflag == TypeRegA { - if strings.HasSuffix(hdr.Name, "/") { - hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories - } else { - hdr.Typeflag = TypeReg - } - } - - // The extended headers may have updated the size. - // Thus, setup the regFileReader again after merging PAX headers. - if err := tr.handleRegularFile(hdr); err != nil { - return nil, err - } - - // Sparse formats rely on being able to read from the logical data - // section; there must be a preceding call to handleRegularFile. - if err := tr.handleSparseFile(hdr, rawHdr); err != nil { - return nil, err - } - - // Set the final guess at the format. - if format.has(FormatUSTAR) && format.has(FormatPAX) { - format.mayOnlyBe(FormatUSTAR) - } - hdr.Format = format - return hdr, nil // This is a file, so stop - } - } -} - -// handleRegularFile sets up the current file reader and padding such that it -// can only read the following logical data section. It will properly handle -// special headers that contain no data section. -func (tr *Reader) handleRegularFile(hdr *Header) error { - nb := hdr.Size - if isHeaderOnlyType(hdr.Typeflag) { - nb = 0 - } - if nb < 0 { - return ErrHeader - } - - tr.pad = blockPadding(nb) - tr.curr = ®FileReader{r: tr.r, nb: nb} - return nil -} - -// handleSparseFile checks if the current file is a sparse format of any type -// and sets the curr reader appropriately. -func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block) error { - var spd sparseDatas - var err error - if hdr.Typeflag == TypeGNUSparse { - spd, err = tr.readOldGNUSparseMap(hdr, rawHdr) - } else { - spd, err = tr.readGNUSparsePAXHeaders(hdr) - } - - // If sp is non-nil, then this is a sparse file. - // Note that it is possible for len(sp) == 0. - if err == nil && spd != nil { - if isHeaderOnlyType(hdr.Typeflag) || !validateSparseEntries(spd, hdr.Size) { - return ErrHeader - } - sph := invertSparseEntries(spd, hdr.Size) - tr.curr = &sparseFileReader{tr.curr, sph, 0} - } - return err -} - -// readGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. -// If they are found, then this function reads the sparse map and returns it. -// This assumes that 0.0 headers have already been converted to 0.1 headers -// by the PAX header parsing logic. -func (tr *Reader) readGNUSparsePAXHeaders(hdr *Header) (sparseDatas, error) { - // Identify the version of GNU headers. - var is1x0 bool - major, minor := hdr.PAXRecords[paxGNUSparseMajor], hdr.PAXRecords[paxGNUSparseMinor] - switch { - case major == "0" && (minor == "0" || minor == "1"): - is1x0 = false - case major == "1" && minor == "0": - is1x0 = true - case major != "" || minor != "": - return nil, nil // Unknown GNU sparse PAX version - case hdr.PAXRecords[paxGNUSparseMap] != "": - is1x0 = false // 0.0 and 0.1 did not have explicit version records, so guess - default: - return nil, nil // Not a PAX format GNU sparse file. - } - hdr.Format.mayOnlyBe(FormatPAX) - - // Update hdr from GNU sparse PAX headers. - if name := hdr.PAXRecords[paxGNUSparseName]; name != "" { - hdr.Name = name - } - size := hdr.PAXRecords[paxGNUSparseSize] - if size == "" { - size = hdr.PAXRecords[paxGNUSparseRealSize] - } - if size != "" { - n, err := strconv.ParseInt(size, 10, 64) - if err != nil { - return nil, ErrHeader - } - hdr.Size = n - } - - // Read the sparse map according to the appropriate format. - if is1x0 { - return readGNUSparseMap1x0(tr.curr) - } - return readGNUSparseMap0x1(hdr.PAXRecords) -} - -// mergePAX merges paxHdrs into hdr for all relevant fields of Header. -func mergePAX(hdr *Header, paxHdrs map[string]string) (err error) { - for k, v := range paxHdrs { - if v == "" { - continue // Keep the original USTAR value - } - var id64 int64 - switch k { - case paxPath: - hdr.Name = v - case paxLinkpath: - hdr.Linkname = v - case paxUname: - hdr.Uname = v - case paxGname: - hdr.Gname = v - case paxUid: - id64, err = strconv.ParseInt(v, 10, 64) - hdr.Uid = int(id64) // Integer overflow possible - case paxGid: - id64, err = strconv.ParseInt(v, 10, 64) - hdr.Gid = int(id64) // Integer overflow possible - case paxAtime: - hdr.AccessTime, err = parsePAXTime(v) - case paxMtime: - hdr.ModTime, err = parsePAXTime(v) - case paxCtime: - hdr.ChangeTime, err = parsePAXTime(v) - case paxSize: - hdr.Size, err = strconv.ParseInt(v, 10, 64) - default: - if strings.HasPrefix(k, paxSchilyXattr) { - if hdr.Xattrs == nil { - hdr.Xattrs = make(map[string]string) - } - hdr.Xattrs[k[len(paxSchilyXattr):]] = v - } - } - if err != nil { - return ErrHeader - } - } - hdr.PAXRecords = paxHdrs - return nil -} - -// parsePAX parses PAX headers. -// If an extended header (type 'x') is invalid, ErrHeader is returned -func parsePAX(r io.Reader) (map[string]string, error) { - buf, err := readSpecialFile(r) - if err != nil { - return nil, err - } - // leaving this function for io.Reader makes it more testable - if tr, ok := r.(*Reader); ok && tr.RawAccounting { - if _, err = tr.rawBytes.Write(buf); err != nil { - return nil, err - } - } - sbuf := string(buf) - - // For GNU PAX sparse format 0.0 support. - // This function transforms the sparse format 0.0 headers into format 0.1 - // headers since 0.0 headers were not PAX compliant. - var sparseMap []string - - paxHdrs := make(map[string]string) - for len(sbuf) > 0 { - key, value, residual, err := parsePAXRecord(sbuf) - if err != nil { - return nil, ErrHeader - } - sbuf = residual - - switch key { - case paxGNUSparseOffset, paxGNUSparseNumBytes: - // Validate sparse header order and value. - if (len(sparseMap)%2 == 0 && key != paxGNUSparseOffset) || - (len(sparseMap)%2 == 1 && key != paxGNUSparseNumBytes) || - strings.Contains(value, ",") { - return nil, ErrHeader - } - sparseMap = append(sparseMap, value) - default: - paxHdrs[key] = value - } - } - if len(sparseMap) > 0 { - paxHdrs[paxGNUSparseMap] = strings.Join(sparseMap, ",") - } - return paxHdrs, nil -} - -// readHeader reads the next block header and assumes that the underlying reader -// is already aligned to a block boundary. It returns the raw block of the -// header in case further processing is required. -// -// The err will be set to io.EOF only when one of the following occurs: -// - Exactly 0 bytes are read and EOF is hit. -// - Exactly 1 block of zeros is read and EOF is hit. -// - At least 2 blocks of zeros are read. -func (tr *Reader) readHeader() (*Header, *block, error) { - // Two blocks of zero bytes marks the end of the archive. - n, err := io.ReadFull(tr.r, tr.blk[:]) - if tr.RawAccounting && (err == nil || err == io.EOF) { - tr.rawBytes.Write(tr.blk[:n]) - } - if err != nil { - return nil, nil, err // EOF is okay here; exactly 0 bytes read - } - - if bytes.Equal(tr.blk[:], zeroBlock[:]) { - n, err = io.ReadFull(tr.r, tr.blk[:]) - if tr.RawAccounting && (err == nil || err == io.EOF) { - tr.rawBytes.Write(tr.blk[:n]) - } - if err != nil { - return nil, nil, err // EOF is okay here; exactly 1 block of zeros read - } - if bytes.Equal(tr.blk[:], zeroBlock[:]) { - return nil, nil, io.EOF // normal EOF; exactly 2 block of zeros read - } - return nil, nil, ErrHeader // Zero block and then non-zero block - } - - // Verify the header matches a known format. - format := tr.blk.GetFormat() - if format == FormatUnknown { - return nil, nil, ErrHeader - } - - var p parser - hdr := new(Header) - - // Unpack the V7 header. - v7 := tr.blk.V7() - hdr.Typeflag = v7.TypeFlag()[0] - hdr.Name = p.parseString(v7.Name()) - hdr.Linkname = p.parseString(v7.LinkName()) - hdr.Size = p.parseNumeric(v7.Size()) - hdr.Mode = p.parseNumeric(v7.Mode()) - hdr.Uid = int(p.parseNumeric(v7.UID())) - hdr.Gid = int(p.parseNumeric(v7.GID())) - hdr.ModTime = time.Unix(p.parseNumeric(v7.ModTime()), 0) - - // Unpack format specific fields. - if format > formatV7 { - ustar := tr.blk.USTAR() - hdr.Uname = p.parseString(ustar.UserName()) - hdr.Gname = p.parseString(ustar.GroupName()) - hdr.Devmajor = p.parseNumeric(ustar.DevMajor()) - hdr.Devminor = p.parseNumeric(ustar.DevMinor()) - - var prefix string - switch { - case format.has(FormatUSTAR | FormatPAX): - hdr.Format = format - ustar := tr.blk.USTAR() - prefix = p.parseString(ustar.Prefix()) - - // For Format detection, check if block is properly formatted since - // the parser is more liberal than what USTAR actually permits. - notASCII := func(r rune) bool { return r >= 0x80 } - if bytes.IndexFunc(tr.blk[:], notASCII) >= 0 { - hdr.Format = FormatUnknown // Non-ASCII characters in block. - } - nul := func(b []byte) bool { return int(b[len(b)-1]) == 0 } - if !(nul(v7.Size()) && nul(v7.Mode()) && nul(v7.UID()) && nul(v7.GID()) && - nul(v7.ModTime()) && nul(ustar.DevMajor()) && nul(ustar.DevMinor())) { - hdr.Format = FormatUnknown // Numeric fields must end in NUL - } - case format.has(formatSTAR): - star := tr.blk.STAR() - prefix = p.parseString(star.Prefix()) - hdr.AccessTime = time.Unix(p.parseNumeric(star.AccessTime()), 0) - hdr.ChangeTime = time.Unix(p.parseNumeric(star.ChangeTime()), 0) - case format.has(FormatGNU): - hdr.Format = format - var p2 parser - gnu := tr.blk.GNU() - if b := gnu.AccessTime(); b[0] != 0 { - hdr.AccessTime = time.Unix(p2.parseNumeric(b), 0) - } - if b := gnu.ChangeTime(); b[0] != 0 { - hdr.ChangeTime = time.Unix(p2.parseNumeric(b), 0) - } - - // Prior to Go1.8, the Writer had a bug where it would output - // an invalid tar file in certain rare situations because the logic - // incorrectly believed that the old GNU format had a prefix field. - // This is wrong and leads to an output file that mangles the - // atime and ctime fields, which are often left unused. - // - // In order to continue reading tar files created by former, buggy - // versions of Go, we skeptically parse the atime and ctime fields. - // If we are unable to parse them and the prefix field looks like - // an ASCII string, then we fallback on the pre-Go1.8 behavior - // of treating these fields as the USTAR prefix field. - // - // Note that this will not use the fallback logic for all possible - // files generated by a pre-Go1.8 toolchain. If the generated file - // happened to have a prefix field that parses as valid - // atime and ctime fields (e.g., when they are valid octal strings), - // then it is impossible to distinguish between an valid GNU file - // and an invalid pre-Go1.8 file. - // - // See https://golang.org/issues/12594 - // See https://golang.org/issues/21005 - if p2.err != nil { - hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} - ustar := tr.blk.USTAR() - if s := p.parseString(ustar.Prefix()); isASCII(s) { - prefix = s - } - hdr.Format = FormatUnknown // Buggy file is not GNU - } - } - if len(prefix) > 0 { - hdr.Name = prefix + "/" + hdr.Name - } - } - return hdr, &tr.blk, p.err -} - -// readOldGNUSparseMap reads the sparse map from the old GNU sparse format. -// The sparse map is stored in the tar header if it's small enough. -// If it's larger than four entries, then one or more extension headers are used -// to store the rest of the sparse map. -// -// The Header.Size does not reflect the size of any extended headers used. -// Thus, this function will read from the raw io.Reader to fetch extra headers. -// This method mutates blk in the process. -func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, error) { - // Make sure that the input format is GNU. - // Unfortunately, the STAR format also has a sparse header format that uses - // the same type flag but has a completely different layout. - if blk.GetFormat() != FormatGNU { - return nil, ErrHeader - } - hdr.Format.mayOnlyBe(FormatGNU) - - var p parser - hdr.Size = p.parseNumeric(blk.GNU().RealSize()) - if p.err != nil { - return nil, p.err - } - s := blk.GNU().Sparse() - spd := make(sparseDatas, 0, s.MaxEntries()) - for { - for i := 0; i < s.MaxEntries(); i++ { - // This termination condition is identical to GNU and BSD tar. - if s.Entry(i).Offset()[0] == 0x00 { - break // Don't return, need to process extended headers (even if empty) - } - offset := p.parseNumeric(s.Entry(i).Offset()) - length := p.parseNumeric(s.Entry(i).Length()) - if p.err != nil { - return nil, p.err - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - } - - if s.IsExtended()[0] > 0 { - // There are more entries. Read an extension header and parse its entries. - if _, err := mustReadFull(tr.r, blk[:]); err != nil { - return nil, err - } - if tr.RawAccounting { - tr.rawBytes.Write(blk[:]) - } - s = blk.Sparse() - continue - } - return spd, nil // Done - } -} - -// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format -// version 1.0. The format of the sparse map consists of a series of -// newline-terminated numeric fields. The first field is the number of entries -// and is always present. Following this are the entries, consisting of two -// fields (offset, length). This function must stop reading at the end -// boundary of the block containing the last newline. -// -// Note that the GNU manual says that numeric values should be encoded in octal -// format. However, the GNU tar utility itself outputs these values in decimal. -// As such, this library treats values as being encoded in decimal. -func readGNUSparseMap1x0(r io.Reader) (sparseDatas, error) { - var ( - cntNewline int64 - buf bytes.Buffer - blk block - totalSize int - ) - - // feedTokens copies data in blocks from r into buf until there are - // at least cnt newlines in buf. It will not read more blocks than needed. - feedTokens := func(n int64) error { - for cntNewline < n { - totalSize += len(blk) - if totalSize > maxSpecialFileSize { - return errSparseTooLong - } - if _, err := mustReadFull(r, blk[:]); err != nil { - return err - } - buf.Write(blk[:]) - for _, c := range blk { - if c == '\n' { - cntNewline++ - } - } - } - return nil - } - - // nextToken gets the next token delimited by a newline. This assumes that - // at least one newline exists in the buffer. - nextToken := func() string { - cntNewline-- - tok, _ := buf.ReadString('\n') - return strings.TrimRight(tok, "\n") - } - - // Parse for the number of entries. - // Use integer overflow resistant math to check this. - if err := feedTokens(1); err != nil { - return nil, err - } - numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int - if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { - return nil, ErrHeader - } - - // Parse for all member entries. - // numEntries is trusted after this since feedTokens limits the number of - // tokens based on maxSpecialFileSize. - if err := feedTokens(2 * numEntries); err != nil { - return nil, err - } - spd := make(sparseDatas, 0, numEntries) - for i := int64(0); i < numEntries; i++ { - offset, err1 := strconv.ParseInt(nextToken(), 10, 64) - length, err2 := strconv.ParseInt(nextToken(), 10, 64) - if err1 != nil || err2 != nil { - return nil, ErrHeader - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - } - return spd, nil -} - -// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format -// version 0.1. The sparse map is stored in the PAX headers. -func readGNUSparseMap0x1(paxHdrs map[string]string) (sparseDatas, error) { - // Get number of entries. - // Use integer overflow resistant math to check this. - numEntriesStr := paxHdrs[paxGNUSparseNumBlocks] - numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int - if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { - return nil, ErrHeader - } - - // There should be two numbers in sparseMap for each entry. - sparseMap := strings.Split(paxHdrs[paxGNUSparseMap], ",") - if len(sparseMap) == 1 && sparseMap[0] == "" { - sparseMap = sparseMap[:0] - } - if int64(len(sparseMap)) != 2*numEntries { - return nil, ErrHeader - } - - // Loop through the entries in the sparse map. - // numEntries is trusted now. - spd := make(sparseDatas, 0, numEntries) - for len(sparseMap) >= 2 { - offset, err1 := strconv.ParseInt(sparseMap[0], 10, 64) - length, err2 := strconv.ParseInt(sparseMap[1], 10, 64) - if err1 != nil || err2 != nil { - return nil, ErrHeader - } - spd = append(spd, sparseEntry{Offset: offset, Length: length}) - sparseMap = sparseMap[2:] - } - return spd, nil -} - -// Read reads from the current file in the tar archive. -// It returns (0, io.EOF) when it reaches the end of that file, -// until Next is called to advance to the next file. -// -// If the current file is sparse, then the regions marked as a hole -// are read back as NUL-bytes. -// -// Calling Read on special types like TypeLink, TypeSymlink, TypeChar, -// TypeBlock, TypeDir, and TypeFifo returns (0, io.EOF) regardless of what -// the Header.Size claims. -func (tr *Reader) Read(b []byte) (int, error) { - if tr.err != nil { - return 0, tr.err - } - n, err := tr.curr.Read(b) - if err != nil && err != io.EOF { - tr.err = err - } - return n, err -} - -// writeTo writes the content of the current file to w. -// The bytes written matches the number of remaining bytes in the current file. -// -// If the current file is sparse and w is an io.WriteSeeker, -// then writeTo uses Seek to skip past holes defined in Header.SparseHoles, -// assuming that skipped regions are filled with NULs. -// This always writes the last byte to ensure w is the right size. -// -// TODO(dsnet): Re-export this when adding sparse file support. -// See https://golang.org/issue/22735 -func (tr *Reader) writeTo(w io.Writer) (int64, error) { - if tr.err != nil { - return 0, tr.err - } - n, err := tr.curr.WriteTo(w) - if err != nil { - tr.err = err - } - return n, err -} - -// regFileReader is a fileReader for reading data from a regular file entry. -type regFileReader struct { - r io.Reader // Underlying Reader - nb int64 // Number of remaining bytes to read -} - -func (fr *regFileReader) Read(b []byte) (n int, err error) { - if int64(len(b)) > fr.nb { - b = b[:fr.nb] - } - if len(b) > 0 { - n, err = fr.r.Read(b) - fr.nb -= int64(n) - } - switch { - case err == io.EOF && fr.nb > 0: - return n, io.ErrUnexpectedEOF - case err == nil && fr.nb == 0: - return n, io.EOF - default: - return n, err - } -} - -func (fr *regFileReader) WriteTo(w io.Writer) (int64, error) { - return io.Copy(w, struct{ io.Reader }{fr}) -} - -func (fr regFileReader) LogicalRemaining() int64 { - return fr.nb -} - -func (fr regFileReader) PhysicalRemaining() int64 { - return fr.nb -} - -// sparseFileReader is a fileReader for reading data from a sparse file entry. -type sparseFileReader struct { - fr fileReader // Underlying fileReader - sp sparseHoles // Normalized list of sparse holes - pos int64 // Current position in sparse file -} - -func (sr *sparseFileReader) Read(b []byte) (n int, err error) { - finished := int64(len(b)) >= sr.LogicalRemaining() - if finished { - b = b[:sr.LogicalRemaining()] - } - - b0 := b - endPos := sr.pos + int64(len(b)) - for endPos > sr.pos && err == nil { - var nf int // Bytes read in fragment - holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() - if sr.pos < holeStart { // In a data fragment - bf := b[:min(int64(len(b)), holeStart-sr.pos)] - nf, err = tryReadFull(sr.fr, bf) - } else { // In a hole fragment - bf := b[:min(int64(len(b)), holeEnd-sr.pos)] - nf, err = tryReadFull(zeroReader{}, bf) - } - b = b[nf:] - sr.pos += int64(nf) - if sr.pos >= holeEnd && len(sr.sp) > 1 { - sr.sp = sr.sp[1:] // Ensure last fragment always remains - } - } - - n = len(b0) - len(b) - switch { - case err == io.EOF: - return n, errMissData // Less data in dense file than sparse file - case err != nil: - return n, err - case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: - return n, errUnrefData // More data in dense file than sparse file - case finished: - return n, io.EOF - default: - return n, nil - } -} - -func (sr *sparseFileReader) WriteTo(w io.Writer) (n int64, err error) { - ws, ok := w.(io.WriteSeeker) - if ok { - if _, err := ws.Seek(0, io.SeekCurrent); err != nil { - ok = false // Not all io.Seeker can really seek - } - } - if !ok { - return io.Copy(w, struct{ io.Reader }{sr}) - } - - var writeLastByte bool - pos0 := sr.pos - for sr.LogicalRemaining() > 0 && !writeLastByte && err == nil { - var nf int64 // Size of fragment - holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() - if sr.pos < holeStart { // In a data fragment - nf = holeStart - sr.pos - nf, err = io.CopyN(ws, sr.fr, nf) - } else { // In a hole fragment - nf = holeEnd - sr.pos - if sr.PhysicalRemaining() == 0 { - writeLastByte = true - nf-- - } - _, err = ws.Seek(nf, io.SeekCurrent) - } - sr.pos += nf - if sr.pos >= holeEnd && len(sr.sp) > 1 { - sr.sp = sr.sp[1:] // Ensure last fragment always remains - } - } - - // If the last fragment is a hole, then seek to 1-byte before EOF, and - // write a single byte to ensure the file is the right size. - if writeLastByte && err == nil { - _, err = ws.Write([]byte{0}) - sr.pos++ - } - - n = sr.pos - pos0 - switch { - case err == io.EOF: - return n, errMissData // Less data in dense file than sparse file - case err != nil: - return n, err - case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: - return n, errUnrefData // More data in dense file than sparse file - default: - return n, nil - } -} - -func (sr sparseFileReader) LogicalRemaining() int64 { - return sr.sp[len(sr.sp)-1].endOffset() - sr.pos -} -func (sr sparseFileReader) PhysicalRemaining() int64 { - return sr.fr.PhysicalRemaining() -} - -type zeroReader struct{} - -func (zeroReader) Read(b []byte) (int, error) { - for i := range b { - b[i] = 0 - } - return len(b), nil -} - -// mustReadFull is like io.ReadFull except it returns -// io.ErrUnexpectedEOF when io.EOF is hit before len(b) bytes are read. -func mustReadFull(r io.Reader, b []byte) (int, error) { - n, err := tryReadFull(r, b) - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return n, err -} - -// tryReadFull is like io.ReadFull except it returns -// io.EOF when it is hit before len(b) bytes are read. -func tryReadFull(r io.Reader, b []byte) (n int, err error) { - for len(b) > n && err == nil { - var nn int - nn, err = r.Read(b[n:]) - n += nn - } - if len(b) == n && err == io.EOF { - err = nil - } - return n, err -} - -// readSpecialFile is like io.ReadAll except it returns -// ErrFieldTooLong if more than maxSpecialFileSize is read. -func readSpecialFile(r io.Reader) ([]byte, error) { - buf, err := io.ReadAll(io.LimitReader(r, maxSpecialFileSize+1)) - if len(buf) > maxSpecialFileSize { - return nil, ErrFieldTooLong - } - return buf, err -} - -// discard skips n bytes in r, reporting an error if unable to do so. -func discard(tr *Reader, n int64) error { - var seekSkipped, copySkipped int64 - var err error - r := tr.r - if tr.RawAccounting { - - copySkipped, err = io.CopyN(tr.rawBytes, tr.r, n) - goto out - } - - // If possible, Seek to the last byte before the end of the data section. - // Do this because Seek is often lazy about reporting errors; this will mask - // the fact that the stream may be truncated. We can rely on the - // io.CopyN done shortly afterwards to trigger any IO errors. - if sr, ok := r.(io.Seeker); ok && n > 1 { - // Not all io.Seeker can actually Seek. For example, os.Stdin implements - // io.Seeker, but calling Seek always returns an error and performs - // no action. Thus, we try an innocent seek to the current position - // to see if Seek is really supported. - pos1, err := sr.Seek(0, io.SeekCurrent) - if pos1 >= 0 && err == nil { - // Seek seems supported, so perform the real Seek. - pos2, err := sr.Seek(n-1, io.SeekCurrent) - if pos2 < 0 || err != nil { - return err - } - seekSkipped = pos2 - pos1 - } - } - - copySkipped, err = io.CopyN(io.Discard, r, n-seekSkipped) -out: - if err == io.EOF && seekSkipped+copySkipped < n { - err = io.ErrUnexpectedEOF - } - return err -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go deleted file mode 100644 index cf9cc79c..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime1.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux dragonfly openbsd solaris - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atim.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctim.Unix()) -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go deleted file mode 100644 index 6f17dbe3..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_actime2.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin freebsd netbsd - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atimespec.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctimespec.Unix()) -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go b/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go deleted file mode 100644 index 868105f3..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux darwin dragonfly freebsd openbsd netbsd solaris - -package tar - -import ( - "os" - "os/user" - "runtime" - "strconv" - "sync" - "syscall" -) - -func init() { - sysStat = statUnix -} - -// userMap and groupMap caches UID and GID lookups for performance reasons. -// The downside is that renaming uname or gname by the OS never takes effect. -var userMap, groupMap sync.Map // map[int]string - -func statUnix(fi os.FileInfo, h *Header) error { - sys, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return nil - } - h.Uid = int(sys.Uid) - h.Gid = int(sys.Gid) - - // Best effort at populating Uname and Gname. - // The os/user functions may fail for any number of reasons - // (not implemented on that platform, cgo not enabled, etc). - if u, ok := userMap.Load(h.Uid); ok { - h.Uname = u.(string) - } else if u, err := user.LookupId(strconv.Itoa(h.Uid)); err == nil { - h.Uname = u.Username - userMap.Store(h.Uid, h.Uname) - } - if g, ok := groupMap.Load(h.Gid); ok { - h.Gname = g.(string) - } else if g, err := user.LookupGroupId(strconv.Itoa(h.Gid)); err == nil { - h.Gname = g.Name - groupMap.Store(h.Gid, h.Gname) - } - - h.AccessTime = statAtime(sys) - h.ChangeTime = statCtime(sys) - - // Best effort at populating Devmajor and Devminor. - if h.Typeflag == TypeChar || h.Typeflag == TypeBlock { - dev := uint64(sys.Rdev) // May be int32 or uint32 - switch runtime.GOOS { - case "linux": - // Copied from golang.org/x/sys/unix/dev_linux.go. - major := uint32((dev & 0x00000000000fff00) >> 8) - major |= uint32((dev & 0xfffff00000000000) >> 32) - minor := uint32((dev & 0x00000000000000ff) >> 0) - minor |= uint32((dev & 0x00000ffffff00000) >> 12) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "darwin": - // Copied from golang.org/x/sys/unix/dev_darwin.go. - major := uint32((dev >> 24) & 0xff) - minor := uint32(dev & 0xffffff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "dragonfly": - // Copied from golang.org/x/sys/unix/dev_dragonfly.go. - major := uint32((dev >> 8) & 0xff) - minor := uint32(dev & 0xffff00ff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "freebsd": - // Copied from golang.org/x/sys/unix/dev_freebsd.go. - major := uint32((dev >> 8) & 0xff) - minor := uint32(dev & 0xffff00ff) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "netbsd": - // Copied from golang.org/x/sys/unix/dev_netbsd.go. - major := uint32((dev & 0x000fff00) >> 8) - minor := uint32((dev & 0x000000ff) >> 0) - minor |= uint32((dev & 0xfff00000) >> 12) - h.Devmajor, h.Devminor = int64(major), int64(minor) - case "openbsd": - // Copied from golang.org/x/sys/unix/dev_openbsd.go. - major := uint32((dev & 0x0000ff00) >> 8) - minor := uint32((dev & 0x000000ff) >> 0) - minor |= uint32((dev & 0xffff0000) >> 8) - h.Devmajor, h.Devminor = int64(major), int64(minor) - default: - // TODO: Implement solaris (see https://golang.org/issue/8106) - } - } - return nil -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go b/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go deleted file mode 100644 index d144485a..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/strconv.go +++ /dev/null @@ -1,326 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "fmt" - "strconv" - "strings" - "time" -) - -// hasNUL reports whether the NUL character exists within s. -func hasNUL(s string) bool { - return strings.IndexByte(s, 0) >= 0 -} - -// isASCII reports whether the input is an ASCII C-style string. -func isASCII(s string) bool { - for _, c := range s { - if c >= 0x80 || c == 0x00 { - return false - } - } - return true -} - -// toASCII converts the input to an ASCII C-style string. -// This a best effort conversion, so invalid characters are dropped. -func toASCII(s string) string { - if isASCII(s) { - return s - } - b := make([]byte, 0, len(s)) - for _, c := range s { - if c < 0x80 && c != 0x00 { - b = append(b, byte(c)) - } - } - return string(b) -} - -type parser struct { - err error // Last error seen -} - -type formatter struct { - err error // Last error seen -} - -// parseString parses bytes as a NUL-terminated C-style string. -// If a NUL byte is not found then the whole slice is returned as a string. -func (*parser) parseString(b []byte) string { - if i := bytes.IndexByte(b, 0); i >= 0 { - return string(b[:i]) - } - return string(b) -} - -// formatString copies s into b, NUL-terminating if possible. -func (f *formatter) formatString(b []byte, s string) { - if len(s) > len(b) { - f.err = ErrFieldTooLong - } - copy(b, s) - if len(s) < len(b) { - b[len(s)] = 0 - } - - // Some buggy readers treat regular files with a trailing slash - // in the V7 path field as a directory even though the full path - // recorded elsewhere (e.g., via PAX record) contains no trailing slash. - if len(s) > len(b) && b[len(b)-1] == '/' { - n := len(strings.TrimRight(s[:len(b)], "/")) - b[n] = 0 // Replace trailing slash with NUL terminator - } -} - -// fitsInBase256 reports whether x can be encoded into n bytes using base-256 -// encoding. Unlike octal encoding, base-256 encoding does not require that the -// string ends with a NUL character. Thus, all n bytes are available for output. -// -// If operating in binary mode, this assumes strict GNU binary mode; which means -// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is -// equivalent to the sign bit in two's complement form. -func fitsInBase256(n int, x int64) bool { - binBits := uint(n-1) * 8 - return n >= 9 || (x >= -1< 0 && b[0]&0x80 != 0 { - // Handling negative numbers relies on the following identity: - // -a-1 == ^a - // - // If the number is negative, we use an inversion mask to invert the - // data bytes and treat the value as an unsigned number. - var inv byte // 0x00 if positive or zero, 0xff if negative - if b[0]&0x40 != 0 { - inv = 0xff - } - - var x uint64 - for i, c := range b { - c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing - if i == 0 { - c &= 0x7f // Ignore signal bit in first byte - } - if (x >> 56) > 0 { - p.err = ErrHeader // Integer overflow - return 0 - } - x = x<<8 | uint64(c) - } - if (x >> 63) > 0 { - p.err = ErrHeader // Integer overflow - return 0 - } - if inv == 0xff { - return ^int64(x) - } - return int64(x) - } - - // Normal case is base-8 (octal) format. - return p.parseOctal(b) -} - -// formatNumeric encodes x into b using base-8 (octal) encoding if possible. -// Otherwise it will attempt to use base-256 (binary) encoding. -func (f *formatter) formatNumeric(b []byte, x int64) { - if fitsInOctal(len(b), x) { - f.formatOctal(b, x) - return - } - - if fitsInBase256(len(b), x) { - for i := len(b) - 1; i >= 0; i-- { - b[i] = byte(x) - x >>= 8 - } - b[0] |= 0x80 // Highest bit indicates binary format - return - } - - f.formatOctal(b, 0) // Last resort, just write zero - f.err = ErrFieldTooLong -} - -func (p *parser) parseOctal(b []byte) int64 { - // Because unused fields are filled with NULs, we need - // to skip leading NULs. Fields may also be padded with - // spaces or NULs. - // So we remove leading and trailing NULs and spaces to - // be sure. - b = bytes.Trim(b, " \x00") - - if len(b) == 0 { - return 0 - } - x, perr := strconv.ParseUint(p.parseString(b), 8, 64) - if perr != nil { - p.err = ErrHeader - } - return int64(x) -} - -func (f *formatter) formatOctal(b []byte, x int64) { - if !fitsInOctal(len(b), x) { - x = 0 // Last resort, just write zero - f.err = ErrFieldTooLong - } - - s := strconv.FormatInt(x, 8) - // Add leading zeros, but leave room for a NUL. - if n := len(b) - len(s) - 1; n > 0 { - s = strings.Repeat("0", n) + s - } - f.formatString(b, s) -} - -// fitsInOctal reports whether the integer x fits in a field n-bytes long -// using octal encoding with the appropriate NUL terminator. -func fitsInOctal(n int, x int64) bool { - octBits := uint(n-1) * 3 - return x >= 0 && (n >= 22 || x < 1<= 0 { - ss, sn = s[:pos], s[pos+1:] - } - - // Parse the seconds. - secs, err := strconv.ParseInt(ss, 10, 64) - if err != nil { - return time.Time{}, ErrHeader - } - if len(sn) == 0 { - return time.Unix(secs, 0), nil // No sub-second values - } - - // Parse the nanoseconds. - if strings.Trim(sn, "0123456789") != "" { - return time.Time{}, ErrHeader - } - if len(sn) < maxNanoSecondDigits { - sn += strings.Repeat("0", maxNanoSecondDigits-len(sn)) // Right pad - } else { - sn = sn[:maxNanoSecondDigits] // Right truncate - } - nsecs, _ := strconv.ParseInt(sn, 10, 64) // Must succeed - if len(ss) > 0 && ss[0] == '-' { - return time.Unix(secs, -1*nsecs), nil // Negative correction - } - return time.Unix(secs, nsecs), nil -} - -// formatPAXTime converts ts into a time of the form %d.%d as described in the -// PAX specification. This function is capable of negative timestamps. -func formatPAXTime(ts time.Time) (s string) { - secs, nsecs := ts.Unix(), ts.Nanosecond() - if nsecs == 0 { - return strconv.FormatInt(secs, 10) - } - - // If seconds is negative, then perform correction. - sign := "" - if secs < 0 { - sign = "-" // Remember sign - secs = -(secs + 1) // Add a second to secs - nsecs = -(nsecs - 1E9) // Take that second away from nsecs - } - return strings.TrimRight(fmt.Sprintf("%s%d.%09d", sign, secs, nsecs), "0") -} - -// parsePAXRecord parses the input PAX record string into a key-value pair. -// If parsing is successful, it will slice off the currently read record and -// return the remainder as r. -func parsePAXRecord(s string) (k, v, r string, err error) { - // The size field ends at the first space. - sp := strings.IndexByte(s, ' ') - if sp == -1 { - return "", "", s, ErrHeader - } - - // Parse the first token as a decimal integer. - n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int - if perr != nil || n < 5 || int64(len(s)) < n { - return "", "", s, ErrHeader - } - - // Extract everything between the space and the final newline. - rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:] - if nl != "\n" { - return "", "", s, ErrHeader - } - - // The first equals separates the key from the value. - eq := strings.IndexByte(rec, '=') - if eq == -1 { - return "", "", s, ErrHeader - } - k, v = rec[:eq], rec[eq+1:] - - if !validPAXRecord(k, v) { - return "", "", s, ErrHeader - } - return k, v, rem, nil -} - -// formatPAXRecord formats a single PAX record, prefixing it with the -// appropriate length. -func formatPAXRecord(k, v string) (string, error) { - if !validPAXRecord(k, v) { - return "", ErrHeader - } - - const padding = 3 // Extra padding for ' ', '=', and '\n' - size := len(k) + len(v) + padding - size += len(strconv.Itoa(size)) - record := strconv.Itoa(size) + " " + k + "=" + v + "\n" - - // Final adjustment if adding size field increased the record size. - if len(record) != size { - size = len(record) - record = strconv.Itoa(size) + " " + k + "=" + v + "\n" - } - return record, nil -} - -// validPAXRecord reports whether the key-value pair is valid where each -// record is formatted as: -// "%d %s=%s\n" % (size, key, value) -// -// Keys and values should be UTF-8, but the number of bad writers out there -// forces us to be a more liberal. -// Thus, we only reject all keys with NUL, and only reject NULs in values -// for the PAX version of the USTAR string fields. -// The key must not contain an '=' character. -func validPAXRecord(k, v string) bool { - if k == "" || strings.IndexByte(k, '=') >= 0 { - return false - } - switch k { - case paxPath, paxLinkpath, paxUname, paxGname: - return !hasNUL(v) - default: - return !hasNUL(k) - } -} diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go b/vendor/github.com/vbatts/tar-split/archive/tar/writer.go deleted file mode 100644 index 893eac00..00000000 --- a/vendor/github.com/vbatts/tar-split/archive/tar/writer.go +++ /dev/null @@ -1,656 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "fmt" - "io" - "path" - "sort" - "strings" - "time" -) - -// Writer provides sequential writing of a tar archive. -// Write.WriteHeader begins a new file with the provided Header, -// and then Writer can be treated as an io.Writer to supply that file's data. -type Writer struct { - w io.Writer - pad int64 // Amount of padding to write after current file entry - curr fileWriter // Writer for current file entry - hdr Header // Shallow copy of Header that is safe for mutations - blk block // Buffer to use as temporary local storage - - // err is a persistent error. - // It is only the responsibility of every exported method of Writer to - // ensure that this error is sticky. - err error -} - -// NewWriter creates a new Writer writing to w. -func NewWriter(w io.Writer) *Writer { - return &Writer{w: w, curr: ®FileWriter{w, 0}} -} - -type fileWriter interface { - io.Writer - fileState - - ReadFrom(io.Reader) (int64, error) -} - -// Flush finishes writing the current file's block padding. -// The current file must be fully written before Flush can be called. -// -// This is unnecessary as the next call to WriteHeader or Close -// will implicitly flush out the file's padding. -func (tw *Writer) Flush() error { - if tw.err != nil { - return tw.err - } - if nb := tw.curr.LogicalRemaining(); nb > 0 { - return fmt.Errorf("archive/tar: missed writing %d bytes", nb) - } - if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil { - return tw.err - } - tw.pad = 0 - return nil -} - -// WriteHeader writes hdr and prepares to accept the file's contents. -// The Header.Size determines how many bytes can be written for the next file. -// If the current file is not fully written, then this returns an error. -// This implicitly flushes any padding necessary before writing the header. -func (tw *Writer) WriteHeader(hdr *Header) error { - if err := tw.Flush(); err != nil { - return err - } - tw.hdr = *hdr // Shallow copy of Header - - // Avoid usage of the legacy TypeRegA flag, and automatically promote - // it to use TypeReg or TypeDir. - if tw.hdr.Typeflag == TypeRegA { - if strings.HasSuffix(tw.hdr.Name, "/") { - tw.hdr.Typeflag = TypeDir - } else { - tw.hdr.Typeflag = TypeReg - } - } - - // Round ModTime and ignore AccessTime and ChangeTime unless - // the format is explicitly chosen. - // This ensures nominal usage of WriteHeader (without specifying the format) - // does not always result in the PAX format being chosen, which - // causes a 1KiB increase to every header. - if tw.hdr.Format == FormatUnknown { - tw.hdr.ModTime = tw.hdr.ModTime.Round(time.Second) - tw.hdr.AccessTime = time.Time{} - tw.hdr.ChangeTime = time.Time{} - } - - allowedFormats, paxHdrs, err := tw.hdr.allowedFormats() - switch { - case allowedFormats.has(FormatUSTAR): - tw.err = tw.writeUSTARHeader(&tw.hdr) - return tw.err - case allowedFormats.has(FormatPAX): - tw.err = tw.writePAXHeader(&tw.hdr, paxHdrs) - return tw.err - case allowedFormats.has(FormatGNU): - tw.err = tw.writeGNUHeader(&tw.hdr) - return tw.err - default: - return err // Non-fatal error - } -} - -func (tw *Writer) writeUSTARHeader(hdr *Header) error { - // Check if we can use USTAR prefix/suffix splitting. - var namePrefix string - if prefix, suffix, ok := splitUSTARPath(hdr.Name); ok { - namePrefix, hdr.Name = prefix, suffix - } - - // Pack the main header. - var f formatter - blk := tw.templateV7Plus(hdr, f.formatString, f.formatOctal) - f.formatString(blk.USTAR().Prefix(), namePrefix) - blk.SetFormat(FormatUSTAR) - if f.err != nil { - return f.err // Should never happen since header is validated - } - return tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag) -} - -func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error { - realName, realSize := hdr.Name, hdr.Size - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Handle sparse files. - var spd sparseDatas - var spb []byte - if len(hdr.SparseHoles) > 0 { - sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map - sph = alignSparseEntries(sph, hdr.Size) - spd = invertSparseEntries(sph, hdr.Size) - - // Format the sparse map. - hdr.Size = 0 // Replace with encoded size - spb = append(strconv.AppendInt(spb, int64(len(spd)), 10), '\n') - for _, s := range spd { - hdr.Size += s.Length - spb = append(strconv.AppendInt(spb, s.Offset, 10), '\n') - spb = append(strconv.AppendInt(spb, s.Length, 10), '\n') - } - pad := blockPadding(int64(len(spb))) - spb = append(spb, zeroBlock[:pad]...) - hdr.Size += int64(len(spb)) // Accounts for encoded sparse map - - // Add and modify appropriate PAX records. - dir, file := path.Split(realName) - hdr.Name = path.Join(dir, "GNUSparseFile.0", file) - paxHdrs[paxGNUSparseMajor] = "1" - paxHdrs[paxGNUSparseMinor] = "0" - paxHdrs[paxGNUSparseName] = realName - paxHdrs[paxGNUSparseRealSize] = strconv.FormatInt(realSize, 10) - paxHdrs[paxSize] = strconv.FormatInt(hdr.Size, 10) - delete(paxHdrs, paxPath) // Recorded by paxGNUSparseName - } - */ - _ = realSize - - // Write PAX records to the output. - isGlobal := hdr.Typeflag == TypeXGlobalHeader - if len(paxHdrs) > 0 || isGlobal { - // Sort keys for deterministic ordering. - var keys []string - for k := range paxHdrs { - keys = append(keys, k) - } - sort.Strings(keys) - - // Write each record to a buffer. - var buf strings.Builder - for _, k := range keys { - rec, err := formatPAXRecord(k, paxHdrs[k]) - if err != nil { - return err - } - buf.WriteString(rec) - } - - // Write the extended header file. - var name string - var flag byte - if isGlobal { - name = realName - if name == "" { - name = "GlobalHead.0.0" - } - flag = TypeXGlobalHeader - } else { - dir, file := path.Split(realName) - name = path.Join(dir, "PaxHeaders.0", file) - flag = TypeXHeader - } - data := buf.String() - if len(data) > maxSpecialFileSize { - return ErrFieldTooLong - } - if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal { - return err // Global headers return here - } - } - - // Pack the main header. - var f formatter // Ignore errors since they are expected - fmtStr := func(b []byte, s string) { f.formatString(b, toASCII(s)) } - blk := tw.templateV7Plus(hdr, fmtStr, f.formatOctal) - blk.SetFormat(FormatPAX) - if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { - return err - } - - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - // Write the sparse map and setup the sparse writer if necessary. - if len(spd) > 0 { - // Use tw.curr since the sparse map is accounted for in hdr.Size. - if _, err := tw.curr.Write(spb); err != nil { - return err - } - tw.curr = &sparseFileWriter{tw.curr, spd, 0} - } - */ - return nil -} - -func (tw *Writer) writeGNUHeader(hdr *Header) error { - // Use long-link files if Name or Linkname exceeds the field size. - const longName = "././@LongLink" - if len(hdr.Name) > nameSize { - data := hdr.Name + "\x00" - if err := tw.writeRawFile(longName, data, TypeGNULongName, FormatGNU); err != nil { - return err - } - } - if len(hdr.Linkname) > nameSize { - data := hdr.Linkname + "\x00" - if err := tw.writeRawFile(longName, data, TypeGNULongLink, FormatGNU); err != nil { - return err - } - } - - // Pack the main header. - var f formatter // Ignore errors since they are expected - var spd sparseDatas - var spb []byte - blk := tw.templateV7Plus(hdr, f.formatString, f.formatNumeric) - if !hdr.AccessTime.IsZero() { - f.formatNumeric(blk.GNU().AccessTime(), hdr.AccessTime.Unix()) - } - if !hdr.ChangeTime.IsZero() { - f.formatNumeric(blk.GNU().ChangeTime(), hdr.ChangeTime.Unix()) - } - // TODO(dsnet): Re-enable this when adding sparse support. - // See https://golang.org/issue/22735 - /* - if hdr.Typeflag == TypeGNUSparse { - sph := append([]sparseEntry{}, hdr.SparseHoles...) // Copy sparse map - sph = alignSparseEntries(sph, hdr.Size) - spd = invertSparseEntries(sph, hdr.Size) - - // Format the sparse map. - formatSPD := func(sp sparseDatas, sa sparseArray) sparseDatas { - for i := 0; len(sp) > 0 && i < sa.MaxEntries(); i++ { - f.formatNumeric(sa.Entry(i).Offset(), sp[0].Offset) - f.formatNumeric(sa.Entry(i).Length(), sp[0].Length) - sp = sp[1:] - } - if len(sp) > 0 { - sa.IsExtended()[0] = 1 - } - return sp - } - sp2 := formatSPD(spd, blk.GNU().Sparse()) - for len(sp2) > 0 { - var spHdr block - sp2 = formatSPD(sp2, spHdr.Sparse()) - spb = append(spb, spHdr[:]...) - } - - // Update size fields in the header block. - realSize := hdr.Size - hdr.Size = 0 // Encoded size; does not account for encoded sparse map - for _, s := range spd { - hdr.Size += s.Length - } - copy(blk.V7().Size(), zeroBlock[:]) // Reset field - f.formatNumeric(blk.V7().Size(), hdr.Size) - f.formatNumeric(blk.GNU().RealSize(), realSize) - } - */ - blk.SetFormat(FormatGNU) - if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { - return err - } - - // Write the extended sparse map and setup the sparse writer if necessary. - if len(spd) > 0 { - // Use tw.w since the sparse map is not accounted for in hdr.Size. - if _, err := tw.w.Write(spb); err != nil { - return err - } - tw.curr = &sparseFileWriter{tw.curr, spd, 0} - } - return nil -} - -type ( - stringFormatter func([]byte, string) - numberFormatter func([]byte, int64) -) - -// templateV7Plus fills out the V7 fields of a block using values from hdr. -// It also fills out fields (uname, gname, devmajor, devminor) that are -// shared in the USTAR, PAX, and GNU formats using the provided formatters. -// -// The block returned is only valid until the next call to -// templateV7Plus or writeRawFile. -func (tw *Writer) templateV7Plus(hdr *Header, fmtStr stringFormatter, fmtNum numberFormatter) *block { - tw.blk.Reset() - - modTime := hdr.ModTime - if modTime.IsZero() { - modTime = time.Unix(0, 0) - } - - v7 := tw.blk.V7() - v7.TypeFlag()[0] = hdr.Typeflag - fmtStr(v7.Name(), hdr.Name) - fmtStr(v7.LinkName(), hdr.Linkname) - fmtNum(v7.Mode(), hdr.Mode) - fmtNum(v7.UID(), int64(hdr.Uid)) - fmtNum(v7.GID(), int64(hdr.Gid)) - fmtNum(v7.Size(), hdr.Size) - fmtNum(v7.ModTime(), modTime.Unix()) - - ustar := tw.blk.USTAR() - fmtStr(ustar.UserName(), hdr.Uname) - fmtStr(ustar.GroupName(), hdr.Gname) - fmtNum(ustar.DevMajor(), hdr.Devmajor) - fmtNum(ustar.DevMinor(), hdr.Devminor) - - return &tw.blk -} - -// writeRawFile writes a minimal file with the given name and flag type. -// It uses format to encode the header format and will write data as the body. -// It uses default values for all of the other fields (as BSD and GNU tar does). -func (tw *Writer) writeRawFile(name, data string, flag byte, format Format) error { - tw.blk.Reset() - - // Best effort for the filename. - name = toASCII(name) - if len(name) > nameSize { - name = name[:nameSize] - } - name = strings.TrimRight(name, "/") - - var f formatter - v7 := tw.blk.V7() - v7.TypeFlag()[0] = flag - f.formatString(v7.Name(), name) - f.formatOctal(v7.Mode(), 0) - f.formatOctal(v7.UID(), 0) - f.formatOctal(v7.GID(), 0) - f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB - f.formatOctal(v7.ModTime(), 0) - tw.blk.SetFormat(format) - if f.err != nil { - return f.err // Only occurs if size condition is violated - } - - // Write the header and data. - if err := tw.writeRawHeader(&tw.blk, int64(len(data)), flag); err != nil { - return err - } - _, err := io.WriteString(tw, data) - return err -} - -// writeRawHeader writes the value of blk, regardless of its value. -// It sets up the Writer such that it can accept a file of the given size. -// If the flag is a special header-only flag, then the size is treated as zero. -func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error { - if err := tw.Flush(); err != nil { - return err - } - if _, err := tw.w.Write(blk[:]); err != nil { - return err - } - if isHeaderOnlyType(flag) { - size = 0 - } - tw.curr = ®FileWriter{tw.w, size} - tw.pad = blockPadding(size) - return nil -} - -// splitUSTARPath splits a path according to USTAR prefix and suffix rules. -// If the path is not splittable, then it will return ("", "", false). -func splitUSTARPath(name string) (prefix, suffix string, ok bool) { - length := len(name) - if length <= nameSize || !isASCII(name) { - return "", "", false - } else if length > prefixSize+1 { - length = prefixSize + 1 - } else if name[length-1] == '/' { - length-- - } - - i := strings.LastIndex(name[:length], "/") - nlen := len(name) - i - 1 // nlen is length of suffix - plen := i // plen is length of prefix - if i <= 0 || nlen > nameSize || nlen == 0 || plen > prefixSize { - return "", "", false - } - return name[:i], name[i+1:], true -} - -// Write writes to the current file in the tar archive. -// Write returns the error ErrWriteTooLong if more than -// Header.Size bytes are written after WriteHeader. -// -// Calling Write on special types like TypeLink, TypeSymlink, TypeChar, -// TypeBlock, TypeDir, and TypeFifo returns (0, ErrWriteTooLong) regardless -// of what the Header.Size claims. -func (tw *Writer) Write(b []byte) (int, error) { - if tw.err != nil { - return 0, tw.err - } - n, err := tw.curr.Write(b) - if err != nil && err != ErrWriteTooLong { - tw.err = err - } - return n, err -} - -// readFrom populates the content of the current file by reading from r. -// The bytes read must match the number of remaining bytes in the current file. -// -// If the current file is sparse and r is an io.ReadSeeker, -// then readFrom uses Seek to skip past holes defined in Header.SparseHoles, -// assuming that skipped regions are all NULs. -// This always reads the last byte to ensure r is the right size. -// -// TODO(dsnet): Re-export this when adding sparse file support. -// See https://golang.org/issue/22735 -func (tw *Writer) readFrom(r io.Reader) (int64, error) { - if tw.err != nil { - return 0, tw.err - } - n, err := tw.curr.ReadFrom(r) - if err != nil && err != ErrWriteTooLong { - tw.err = err - } - return n, err -} - -// Close closes the tar archive by flushing the padding, and writing the footer. -// If the current file (from a prior call to WriteHeader) is not fully written, -// then this returns an error. -func (tw *Writer) Close() error { - if tw.err == ErrWriteAfterClose { - return nil - } - if tw.err != nil { - return tw.err - } - - // Trailer: two zero blocks. - err := tw.Flush() - for i := 0; i < 2 && err == nil; i++ { - _, err = tw.w.Write(zeroBlock[:]) - } - - // Ensure all future actions are invalid. - tw.err = ErrWriteAfterClose - return err // Report IO errors -} - -// regFileWriter is a fileWriter for writing data to a regular file entry. -type regFileWriter struct { - w io.Writer // Underlying Writer - nb int64 // Number of remaining bytes to write -} - -func (fw *regFileWriter) Write(b []byte) (n int, err error) { - overwrite := int64(len(b)) > fw.nb - if overwrite { - b = b[:fw.nb] - } - if len(b) > 0 { - n, err = fw.w.Write(b) - fw.nb -= int64(n) - } - switch { - case err != nil: - return n, err - case overwrite: - return n, ErrWriteTooLong - default: - return n, nil - } -} - -func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) { - return io.Copy(struct{ io.Writer }{fw}, r) -} - -func (fw regFileWriter) LogicalRemaining() int64 { - return fw.nb -} -func (fw regFileWriter) PhysicalRemaining() int64 { - return fw.nb -} - -// sparseFileWriter is a fileWriter for writing data to a sparse file entry. -type sparseFileWriter struct { - fw fileWriter // Underlying fileWriter - sp sparseDatas // Normalized list of data fragments - pos int64 // Current position in sparse file -} - -func (sw *sparseFileWriter) Write(b []byte) (n int, err error) { - overwrite := int64(len(b)) > sw.LogicalRemaining() - if overwrite { - b = b[:sw.LogicalRemaining()] - } - - b0 := b - endPos := sw.pos + int64(len(b)) - for endPos > sw.pos && err == nil { - var nf int // Bytes written in fragment - dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() - if sw.pos < dataStart { // In a hole fragment - bf := b[:min(int64(len(b)), dataStart-sw.pos)] - nf, err = zeroWriter{}.Write(bf) - } else { // In a data fragment - bf := b[:min(int64(len(b)), dataEnd-sw.pos)] - nf, err = sw.fw.Write(bf) - } - b = b[nf:] - sw.pos += int64(nf) - if sw.pos >= dataEnd && len(sw.sp) > 1 { - sw.sp = sw.sp[1:] // Ensure last fragment always remains - } - } - - n = len(b0) - len(b) - switch { - case err == ErrWriteTooLong: - return n, errMissData // Not possible; implies bug in validation logic - case err != nil: - return n, err - case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: - return n, errUnrefData // Not possible; implies bug in validation logic - case overwrite: - return n, ErrWriteTooLong - default: - return n, nil - } -} - -func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) { - rs, ok := r.(io.ReadSeeker) - if ok { - if _, err := rs.Seek(0, io.SeekCurrent); err != nil { - ok = false // Not all io.Seeker can really seek - } - } - if !ok { - return io.Copy(struct{ io.Writer }{sw}, r) - } - - var readLastByte bool - pos0 := sw.pos - for sw.LogicalRemaining() > 0 && !readLastByte && err == nil { - var nf int64 // Size of fragment - dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() - if sw.pos < dataStart { // In a hole fragment - nf = dataStart - sw.pos - if sw.PhysicalRemaining() == 0 { - readLastByte = true - nf-- - } - _, err = rs.Seek(nf, io.SeekCurrent) - } else { // In a data fragment - nf = dataEnd - sw.pos - nf, err = io.CopyN(sw.fw, rs, nf) - } - sw.pos += nf - if sw.pos >= dataEnd && len(sw.sp) > 1 { - sw.sp = sw.sp[1:] // Ensure last fragment always remains - } - } - - // If the last fragment is a hole, then seek to 1-byte before EOF, and - // read a single byte to ensure the file is the right size. - if readLastByte && err == nil { - _, err = mustReadFull(rs, []byte{0}) - sw.pos++ - } - - n = sw.pos - pos0 - switch { - case err == io.EOF: - return n, io.ErrUnexpectedEOF - case err == ErrWriteTooLong: - return n, errMissData // Not possible; implies bug in validation logic - case err != nil: - return n, err - case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: - return n, errUnrefData // Not possible; implies bug in validation logic - default: - return n, ensureEOF(rs) - } -} - -func (sw sparseFileWriter) LogicalRemaining() int64 { - return sw.sp[len(sw.sp)-1].endOffset() - sw.pos -} -func (sw sparseFileWriter) PhysicalRemaining() int64 { - return sw.fw.PhysicalRemaining() -} - -// zeroWriter may only be written with NULs, otherwise it returns errWriteHole. -type zeroWriter struct{} - -func (zeroWriter) Write(b []byte) (int, error) { - for i, c := range b { - if c != 0 { - return i, errWriteHole - } - } - return len(b), nil -} - -// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so. -func ensureEOF(r io.Reader) error { - n, err := tryReadFull(r, []byte{0}) - switch { - case n > 0: - return ErrWriteTooLong - case err == io.EOF: - return nil - default: - return err - } -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 63de39bc..ba3374be 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -432,10 +432,6 @@ github.com/containerd/log # github.com/containerd/platforms v1.0.0-rc.4 ## explicit; go 1.24 github.com/containerd/platforms -# github.com/containerd/stargz-snapshotter/estargz v0.18.2 -## explicit; go 1.24.0 -github.com/containerd/stargz-snapshotter/estargz -github.com/containerd/stargz-snapshotter/estargz/errorutil # github.com/containerd/typeurl/v2 v2.2.3 ## explicit; go 1.21 github.com/containerd/typeurl/v2 @@ -852,11 +848,10 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value -# github.com/google/go-containerregistry v0.21.5 +# github.com/google/go-containerregistry v0.21.6 ## explicit; go 1.25.0 github.com/google/go-containerregistry/internal/and github.com/google/go-containerregistry/internal/compression -github.com/google/go-containerregistry/internal/estargz github.com/google/go-containerregistry/internal/gzip github.com/google/go-containerregistry/internal/redact github.com/google/go-containerregistry/internal/retry @@ -1469,9 +1464,6 @@ github.com/uudashr/iface/internal/directive github.com/uudashr/iface/opaque github.com/uudashr/iface/unexported github.com/uudashr/iface/unused -# github.com/vbatts/tar-split v0.12.2 -## explicit; go 1.17 -github.com/vbatts/tar-split/archive/tar # github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb ## explicit # github.com/xen0n/gosmopolitan v1.3.0