diff --git a/0-image-OCIncompressedLayer.patch b/0-image-OCIncompressedLayer.patch new file mode 100644 index 000000000..c5e549965 --- /dev/null +++ b/0-image-OCIncompressedLayer.patch @@ -0,0 +1,16 @@ +diff --git a/container/go/pkg/compat/image.go b/container/go/pkg/compat/image.go +index f04512d..c806818 100644 +--- a/container/go/pkg/compat/image.go ++++ b/container/go/pkg/compat/image.go +@@ -172,6 +172,11 @@ func (li *legacyImage) genManifest() error { + size = fl.size + urls = fl.urls + } ++ if mediaType == types.OCIUncompressedLayer { ++ // The remote pusher always stores blobs in compressed format, even if the ++ // original media type was uncompressed. ++ mediaType = types.OCILayer ++ } + li.manifest.Layers = append(li.manifest.Layers, v1.Descriptor{ + MediaType: mediaType, + Digest: digest, diff --git a/1-image-config_stripper.py b/1-image-config_stripper.py new file mode 100644 index 000000000..033963805 --- /dev/null +++ b/1-image-config_stripper.py @@ -0,0 +1,95 @@ +diff --git a/docker/util/config_stripper.py b/docker/util/config_stripper.py +index 6f8bb33..2424e04 100644 +--- a/docker/util/config_stripper.py ++++ b/docker/util/config_stripper.py +@@ -28,8 +28,6 @@ import threading + + _TIMESTAMP = '1970-01-01T00:00:00Z' + +-WHITELISTED_PREFIXES = ['sha256:', 'manifest', 'repositories'] +- + _BUF_SIZE = 4096 + + def main(): +@@ -70,7 +68,7 @@ def strip_tar(input, output): + # that symlinks to a lower layer that hasn't been extracted yet. Just + # reversing the iteration order avoids this problem. + for layer in reversed(image['Layers']): +- (new_layer_name, new_diff_id) = strip_layer(os.path.join(tempdir, layer)) ++ (new_layer_name, new_diff_id) = strip_layer(tempdir, layer) + + new_layers.append(new_layer_name) + new_diff_ids.append(new_diff_id) +@@ -83,8 +81,8 @@ def strip_tar(input, output): + image['Layers'] = new_layers + + config = image['Config'] +- cfg_path = os.path.join(tempdir, config) +- new_cfg_path = strip_config(cfg_path, new_diff_ids) ++ ++ new_cfg_path = strip_config(tempdir, config, new_diff_ids) + + # Update the name of the config in the metadata object + # to match it's new digest. +@@ -98,10 +96,9 @@ def strip_tar(input, output): + files_to_add = [] + for root, _, files in os.walk(tempdir): + for f in files: +- if os.path.basename(f).startswith(tuple(WHITELISTED_PREFIXES)): +- name = os.path.join(root, f) +- os.utime(name, (0,0)) +- files_to_add.append(name) ++ name = os.path.join(root, f) ++ os.utime(name, (0,0)) ++ files_to_add.append(name) + + with tarfile.open(name=output, mode='w') as ot: + for f in sorted(files_to_add): +@@ -112,10 +109,11 @@ def strip_tar(input, output): + shutil.rmtree(tempdir) + return 0 + +-def strip_layer(path): ++def strip_layer(work_dir, layer): + # The original layer tar is of the form /layer.tar, the + # working directory is one level up from where layer.tar is. +- original_dir = os.path.normpath(os.path.join(os.path.dirname(path), '..')) ++ path = os.path.join(work_dir, layer) ++ original_dir = os.path.dirname(path) + + # Write compressed tar to a temporary name. We'll rename it to the correct + # name after we compute the hash. +@@ -206,14 +204,15 @@ def strip_layer(path): + diffid = 'sha256:%s' % uncompressed_sha.hexdigest() + + # Rename into correct location now that we know the hash. +- new_name = 'sha256:%s' % compressed_sha.hexdigest() +- os.rename(gz_out.name, os.path.join(original_dir, new_name)) ++ new_name = os.path.join(original_dir, compressed_sha.hexdigest()) ++ os.rename(gz_out.name, new_name) + +- shutil.rmtree(os.path.dirname(path)) +- return (new_name, diffid) ++ os.remove(path) ++ return (os.path.relpath(new_name, work_dir), diffid) + + +-def strip_config(path, new_diff_ids): ++def strip_config(work_dir, config, new_diff_ids): ++ path = os.path.join(work_dir, config) + with open(path, 'r') as f: + config = json.load(f) + config['created'] = _TIMESTAMP +@@ -239,9 +238,9 @@ def strip_config(path, new_diff_ids): + + # Calculate the new file path + sha = hashlib.sha256(config_str.encode("utf-8")).hexdigest() +- new_path = 'sha256:%s' % sha +- os.rename(path, os.path.join(os.path.dirname(path), new_path)) +- return new_path ++ new_path = os.path.join(os.path.dirname(path), sha) ++ os.rename(path, new_path) ++ return os.path.relpath(new_path, work_dir) + + + if __name__ == "__main__": diff --git a/container/go/pkg/compat/image.go b/container/go/pkg/compat/image.go index f04512dc9..c80681821 100644 --- a/container/go/pkg/compat/image.go +++ b/container/go/pkg/compat/image.go @@ -172,6 +172,11 @@ func (li *legacyImage) genManifest() error { size = fl.size urls = fl.urls } + if mediaType == types.OCIUncompressedLayer { + // The remote pusher always stores blobs in compressed format, even if the + // original media type was uncompressed. + mediaType = types.OCILayer + } li.manifest.Layers = append(li.manifest.Layers, v1.Descriptor{ MediaType: mediaType, Digest: digest, diff --git a/docker/util/config_stripper.py b/docker/util/config_stripper.py index 6f8bb3359..2424e045c 100644 --- a/docker/util/config_stripper.py +++ b/docker/util/config_stripper.py @@ -28,8 +28,6 @@ _TIMESTAMP = '1970-01-01T00:00:00Z' -WHITELISTED_PREFIXES = ['sha256:', 'manifest', 'repositories'] - _BUF_SIZE = 4096 def main(): @@ -70,7 +68,7 @@ def strip_tar(input, output): # that symlinks to a lower layer that hasn't been extracted yet. Just # reversing the iteration order avoids this problem. for layer in reversed(image['Layers']): - (new_layer_name, new_diff_id) = strip_layer(os.path.join(tempdir, layer)) + (new_layer_name, new_diff_id) = strip_layer(tempdir, layer) new_layers.append(new_layer_name) new_diff_ids.append(new_diff_id) @@ -83,8 +81,8 @@ def strip_tar(input, output): image['Layers'] = new_layers config = image['Config'] - cfg_path = os.path.join(tempdir, config) - new_cfg_path = strip_config(cfg_path, new_diff_ids) + + new_cfg_path = strip_config(tempdir, config, new_diff_ids) # Update the name of the config in the metadata object # to match it's new digest. @@ -98,10 +96,9 @@ def strip_tar(input, output): files_to_add = [] for root, _, files in os.walk(tempdir): for f in files: - if os.path.basename(f).startswith(tuple(WHITELISTED_PREFIXES)): - name = os.path.join(root, f) - os.utime(name, (0,0)) - files_to_add.append(name) + name = os.path.join(root, f) + os.utime(name, (0,0)) + files_to_add.append(name) with tarfile.open(name=output, mode='w') as ot: for f in sorted(files_to_add): @@ -112,10 +109,11 @@ def strip_tar(input, output): shutil.rmtree(tempdir) return 0 -def strip_layer(path): +def strip_layer(work_dir, layer): # The original layer tar is of the form /layer.tar, the # working directory is one level up from where layer.tar is. - original_dir = os.path.normpath(os.path.join(os.path.dirname(path), '..')) + path = os.path.join(work_dir, layer) + original_dir = os.path.dirname(path) # Write compressed tar to a temporary name. We'll rename it to the correct # name after we compute the hash. @@ -206,14 +204,15 @@ def do_gzip_stderr(): diffid = 'sha256:%s' % uncompressed_sha.hexdigest() # Rename into correct location now that we know the hash. - new_name = 'sha256:%s' % compressed_sha.hexdigest() - os.rename(gz_out.name, os.path.join(original_dir, new_name)) + new_name = os.path.join(original_dir, compressed_sha.hexdigest()) + os.rename(gz_out.name, new_name) - shutil.rmtree(os.path.dirname(path)) - return (new_name, diffid) + os.remove(path) + return (os.path.relpath(new_name, work_dir), diffid) -def strip_config(path, new_diff_ids): +def strip_config(work_dir, config, new_diff_ids): + path = os.path.join(work_dir, config) with open(path, 'r') as f: config = json.load(f) config['created'] = _TIMESTAMP @@ -239,9 +238,9 @@ def strip_config(path, new_diff_ids): # Calculate the new file path sha = hashlib.sha256(config_str.encode("utf-8")).hexdigest() - new_path = 'sha256:%s' % sha - os.rename(path, os.path.join(os.path.dirname(path), new_path)) - return new_path + new_path = os.path.join(os.path.dirname(path), sha) + os.rename(path, new_path) + return os.path.relpath(new_path, work_dir) if __name__ == "__main__": diff --git a/repositories/py_repositories.bzl b/repositories/py_repositories.bzl index 8b511e9ee..4bd2cafd0 100644 --- a/repositories/py_repositories.bzl +++ b/repositories/py_repositories.bzl @@ -19,7 +19,7 @@ Provides functions to pull all Python external package dependencies of this repository. """ -load("@rules_python//python:pip.bzl", "pip_install") +load("@rules_python//python:pip.bzl", "pip_parse") def py_deps(): """Pull in external Python packages needed by py binaries in this repo. @@ -31,7 +31,7 @@ def py_deps(): """ excludes = native.existing_rules().keys() if "io_bazel_rules_docker_pip_deps" not in excludes: - pip_install( + pip_parse( name = "io_bazel_rules_docker_pip_deps", - requirements = "@io_bazel_rules_docker//repositories:requirements-pip.txt", + requirements_lock = "@io_bazel_rules_docker//repositories:requirements-pip.txt", ) diff --git a/repositories/repositories.bzl b/repositories/repositories.bzl index 4175ba6a4..3ddc3821b 100644 --- a/repositories/repositories.bzl +++ b/repositories/repositories.bzl @@ -37,7 +37,7 @@ def repositories(): name = "go_puller_linux_amd64", executable = True, sha256 = "08b8963cce9234f57055bafc7cadd1624cdce3c5990048cea1df453d7d288bc6", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-linux-amd64")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-linux-amd64")], ) if "go_puller_linux_arm64" not in excludes: @@ -45,7 +45,7 @@ def repositories(): name = "go_puller_linux_arm64", executable = True, sha256 = "912ee7c469b3e4bf15ba5d1f0ee500e7ec6724518862703fa8b09e4d58ce3ee6", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-linux-arm64")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-linux-arm64")], ) if "go_puller_linux_s390x" not in excludes: @@ -53,7 +53,7 @@ def repositories(): name = "go_puller_linux_s390x", executable = True, sha256 = "a5527b7b3b4a266e4680a4ad8939429665d4173f26b35d5d317385134369e438", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-linux-s390x")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-linux-s390x")], ) if "go_puller_darwin" not in excludes: @@ -61,7 +61,7 @@ def repositories(): name = "go_puller_darwin", executable = True, sha256 = "4855c4f5927f8fb0f885510ab3e2a166d5fa7cde765fbe9aec97dc6b2761bb22", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-darwin-amd64")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/puller-darwin-amd64")], ) if "loader_linux_amd64" not in excludes: @@ -69,7 +69,7 @@ def repositories(): name = "loader_linux_amd64", executable = True, sha256 = "5e5ada66beff07f9188bdc1f99c3fa37c407fc0048cd78b9c2047e9c5516f20b", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-linux-amd64")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-linux-amd64")], ) if "loader_linux_arm64" not in excludes: @@ -77,7 +77,7 @@ def repositories(): name = "loader_linux_arm64", executable = True, sha256 = "a80966d17b25dbc9313e9fc1cae74ded5916fa64dba0d33438c8adad338b44d3", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-linux-arm64")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-linux-arm64")], ) if "loader_linux_s390x" not in excludes: @@ -85,7 +85,7 @@ def repositories(): name = "loader_linux_s390x", executable = True, sha256 = "0c0ebc3e0a502542547a38b51f4686a049897eeb4cbc0e2f07fc25276c57866f", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-linux-s390x")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-linux-s390x")], ) if "loader_darwin" not in excludes: @@ -93,7 +93,7 @@ def repositories(): name = "loader_darwin", executable = True, sha256 = "8c9986b2b506febbff737090d9ec485cec1376c52789747573521a85194341c1", - urls = [("https://storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-darwin-amd64")], + urls = [("https://mirror.bazel.build/storage.googleapis.com/rules_docker/" + RULES_DOCKER_GO_BINARY_RELEASE + "/loader-darwin-amd64")], ) if "containerregistry" not in excludes: