From 03cee1957a4636ac811c2c0cbfb5d9a23b4d872d Mon Sep 17 00:00:00 2001 From: Thomas Fricke Date: Tue, 22 Jul 2025 12:16:20 +0200 Subject: [PATCH 1/6] added working and manually tested image based on the Minikube Docker container multistage build --- kubernetes/Dockerfile | 46 +++++++++++++++++++++++++++++++++++++++++++ kubernetes/Makefile | 18 +++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 kubernetes/Dockerfile create mode 100644 kubernetes/Makefile diff --git a/kubernetes/Dockerfile b/kubernetes/Dockerfile new file mode 100644 index 0000000..7170fab --- /dev/null +++ b/kubernetes/Dockerfile @@ -0,0 +1,46 @@ +# +# Go build stage +# +FROM gcr.io/k8s-minikube/kicbase:v0.0.47 AS builder +WORKDIR /build + + +RUN apt-get update +RUN apt install -y git wget cmake pkg-config +RUN apt-get install build-essential libfuse3-dev -y + +RUN wget https://go.dev/dl/go1.23.6.linux-amd64.tar.gz +RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go1.23.6.linux-amd64.tar.gz && rm go1.23.6.linux-amd64.tar.gz +ENV PATH=$PATH:/usr/local/go/bin + +# +# Clone and make install +# +RUN git clone https://github.com/negativa-ai/BLAFS +WORKDIR BLAFS +COPY Makefile . +RUN make install + +# +# use the original Minikube image +# +FROM gcr.io/k8s-minikube/kicbase:v0.0.47 + +RUN apt update + +RUN apt install libfuse3-3 -y + +# +# copy the tools build +# + +COPY --from=builder /usr/bin/baffs /usr/bin/baffs +COPY --from=builder /usr/bin/debloated_fs /usr/bin/debloated_fs + +# +# configure timezone +# +ENV TZ=Europe/Berlin +RUN ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime +RUN echo "$TZ" > /etc/timezone + diff --git a/kubernetes/Makefile b/kubernetes/Makefile new file mode 100644 index 0000000..ec8bc9d --- /dev/null +++ b/kubernetes/Makefile @@ -0,0 +1,18 @@ +debloated_fs: + mkdir -p build + cd build && cmake ../fs && cmake --build . + +baffs: + CGO_ENABLED=0 go build -ldflags="-extldflags=-static" -o baffs + + +install: debloated_fs baffs + cp build/debloated_fs /usr/bin/debloated_fs + cp baffs /usr/bin/baffs + +test: debloated_fs + ./build/all_test + go test -v ./... + +integration_test: install + ./tests/test.sh From 7f2fac597221c861ad4aa219432c74dcc278da9f Mon Sep 17 00:00:00 2001 From: Thomas Fricke Date: Tue, 22 Jul 2025 19:48:37 +0200 Subject: [PATCH 2/6] readpipe is now systemd service unbloatpipe writepipe added README updated --- kubernetes/README.md | 40 ++++++++++++++++++++++++++ kubernetes/scripts/unbloatpipe | 29 +++++++++++++++++++ kubernetes/scripts/unbloatpipe.service | 15 ++++++++++ 3 files changed, 84 insertions(+) create mode 100644 kubernetes/README.md create mode 100755 kubernetes/scripts/unbloatpipe create mode 100644 kubernetes/scripts/unbloatpipe.service diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 0000000..acd2151 --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,40 @@ +# Kubernetes setup + +This defines the files needed to run `baffs` in Kubernetes. + +## Images + +It derives everything from Minikube docker images. +As this runs a full featured Ubuntu 22.04 LTS inside, +it is compatible with the BLAFS containers. + + +The actual Dockerfile can reuse a lot of the original Minikube image. +The compile task has been separated in an early build stage derived +from the very same image used by Minikube for compatibility reasons +(the Golang build container artefacts were not compatible and the +`docker run ...` did not succeed on a shadowed container) + +The Docker version inside is used, not need to install another version. + +## Howto use + +Build the image with `docker build . -t minikube-baffs` and run Minikube +with `minikube start --driver=docker --base-image=minikube-baffs`. + +In the Minikube container trigger download of the image and the initial +shadowin by using `writepipe redis:7.4.1` + +You now can create a deplopment using this image by + +```bash +kubectl create deployment redis-shadow --image=redis:7.4.1 +``` + +Test your application as much as possible. + +Now stop the deployment by `kubect delete deployment redis-shadow` +and retrigger the debloat by using `writepipe redis:7.4.1` +in Minikube again. + +Restart the deployment, it is using the debloated container image now! diff --git a/kubernetes/scripts/unbloatpipe b/kubernetes/scripts/unbloatpipe new file mode 100755 index 0000000..a16b98e --- /dev/null +++ b/kubernetes/scripts/unbloatpipe @@ -0,0 +1,29 @@ +#!/bin/bash + +pipe=/tmp/testpipe + +trap "rm -f $pipe" EXIT + +if [[ ! -p $pipe ]]; then + mkfifo $pipe +fi + +while true +do + if read line <$pipe; then + echo "working on $line" + IMAGE=$(docker image list $line --format "table {{.Repository}}:{{.Tag}} {{.ID}}" | grep -v REPOSITORY | cut -f1 -d" ") + if [ -z $IMAGE ] + then + echo "pull and shadow..." + docker pull $line + docker tag $line "$line-nobaffs" + baffs shadow --images=$line + else + echo "debloat terminated shadowed $line" + baffs debloat --images=$line + docker tag "$line-baffs" $line + fi + + fi +done diff --git a/kubernetes/scripts/unbloatpipe.service b/kubernetes/scripts/unbloatpipe.service new file mode 100644 index 0000000..61b7b7e --- /dev/null +++ b/kubernetes/scripts/unbloatpipe.service @@ -0,0 +1,15 @@ +[Unit] +Description=pipe to unbloat images + +[Service] +Type=simple +WorkingDirectory=/home/docker +ExecStart=/usr/local/bin/unbloatpipe +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure +User=root +Group=root + +[Install] +WantedBy=multi-user.target From 2d3f9cf516e8491dd9db8d2ef27e7c07c02a191f Mon Sep 17 00:00:00 2001 From: Thomas Fricke Date: Wed, 23 Jul 2025 13:21:49 +0200 Subject: [PATCH 3/6] move scripts to reasonable names cleaned up the pipe limited lifetime of writepipe job to get rid of the need to delete it --- kubernetes/Dockerfile | 10 +++++++ kubernetes/README.md | 12 +++++---- kubernetes/scripts/unbloatpipe | 48 +++++++++++++++++++++++----------- kubernetes/writepipe.yaml | 26 ++++++++++++++++++ 4 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 kubernetes/writepipe.yaml diff --git a/kubernetes/Dockerfile b/kubernetes/Dockerfile index 7170fab..1774a88 100644 --- a/kubernetes/Dockerfile +++ b/kubernetes/Dockerfile @@ -44,3 +44,13 @@ ENV TZ=Europe/Berlin RUN ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime RUN echo "$TZ" > /etc/timezone +# +# configure systemd +# + +COPY scripts/unbloatpipe.service /etc/systemd/system/unbloatpipe.service +COPY scripts/unbloatpipe /usr/local/bin/unbloatpipe + +RUN systemctl enable unbloatpipe + + diff --git a/kubernetes/README.md b/kubernetes/README.md index acd2151..05a5ac2 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -17,13 +17,17 @@ from the very same image used by Minikube for compatibility reasons The Docker version inside is used, not need to install another version. -## Howto use +## How to use Build the image with `docker build . -t minikube-baffs` and run Minikube with `minikube start --driver=docker --base-image=minikube-baffs`. In the Minikube container trigger download of the image and the initial -shadowin by using `writepipe redis:7.4.1` +shadowing by using + +```bash + sed s/CONTAINER/redis:7.4.1/ writepipe.yaml | kubectl create -f - +``` You now can create a deplopment using this image by @@ -33,8 +37,6 @@ kubectl create deployment redis-shadow --image=redis:7.4.1 Test your application as much as possible. -Now stop the deployment by `kubect delete deployment redis-shadow` -and retrigger the debloat by using `writepipe redis:7.4.1` -in Minikube again. +Now stop the deployment by `kubectl delete deployment redis-shadow`. Restart the deployment, it is using the debloated container image now! diff --git a/kubernetes/scripts/unbloatpipe b/kubernetes/scripts/unbloatpipe index a16b98e..a4c759e 100755 --- a/kubernetes/scripts/unbloatpipe +++ b/kubernetes/scripts/unbloatpipe @@ -1,6 +1,6 @@ #!/bin/bash -pipe=/tmp/testpipe +pipe=/var/run/unbloatpipe trap "rm -f $pipe" EXIT @@ -11,19 +11,37 @@ fi while true do if read line <$pipe; then - echo "working on $line" - IMAGE=$(docker image list $line --format "table {{.Repository}}:{{.Tag}} {{.ID}}" | grep -v REPOSITORY | cut -f1 -d" ") - if [ -z $IMAGE ] - then - echo "pull and shadow..." - docker pull $line - docker tag $line "$line-nobaffs" - baffs shadow --images=$line - else - echo "debloat terminated shadowed $line" - baffs debloat --images=$line - docker tag "$line-baffs" $line - fi - + + if [ ! -z `docker images $line-nobaffs --format="table {{.Repository}}:{{.Tag}}" | grep -v REPOSITORY` ] + then + echo "## image $line is already processed" + continue + fi + + echo "## pull and shadow $line" + docker pull $line + docker tag $line "$line-nobaffs" + baffs shadow --images=$line + systemctl restart kubelet + + + echo "## wait until container is in use" + while true; + do + IMAGE_ID=$(docker image list $line --format "table {{.Repository}}:{{.Tag}} {{.ID}}" | grep -v REPOSITORY | cut -f2 -d" ") + (docker ps | grep $IMAGE_ID) && break + sleep 5 + done + + echo "## wait until container terminates" + CONTAINER_ID=$(docker ps | grep $IMAGE_ID | cut -f1 -d" ") + docker container wait $CONTAINER_ID + + echo "## debloat terminated shadowed $line" + baffs debloat --images=$line + systemctl restart kubelet + docker tag "$line-baffs" $line + + echo "## finished debloating $line" fi done diff --git a/kubernetes/writepipe.yaml b/kubernetes/writepipe.yaml new file mode 100644 index 0000000..eff4bb6 --- /dev/null +++ b/kubernetes/writepipe.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: writepipe +spec: + ttlSecondsAfterFinished: 10 + template: + metadata: + name: writepipe + spec: + containers: + - name: writepipe + image: busybox + args: + - /bin/sh + - -c + - echo CONTAINER > /tmp/run/unbloatpipe + volumeMounts: + - name: run + mountPath: /tmp/run + restartPolicy: Never + volumes: + - name: run + hostPath: + path: /var/run + type: Directory From c84051192e0eca20f2e824f14e833ac45f3ff4a3 Mon Sep 17 00:00:00 2001 From: Thomas Fricke Date: Mon, 28 Jul 2025 13:58:57 +0200 Subject: [PATCH 4/6] semi working listener unbloatpipe does forks now --- kubernetes/podeventlistener.sh | 48 ++++++++++++++++++++++++++++++ kubernetes/scripts/unbloatpipe | 54 +++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 24 deletions(-) create mode 100755 kubernetes/podeventlistener.sh diff --git a/kubernetes/podeventlistener.sh b/kubernetes/podeventlistener.sh new file mode 100755 index 0000000..2720613 --- /dev/null +++ b/kubernetes/podeventlistener.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# +# gets the original owner of a pod +# + +set -x + +getowner(){ + local object=$1 + local owner=$(kubectl get $object -o jsonpath='{ .metadata.ownerReferences[].kind}/{.metadata.ownerReferences[].name}' || return 1) + + + if [ -z $owner ] + then + return 1 + fi + + if [ $owner == '/' ] + then + echo $object + return 0 + else + getowner $owner + return $? + fi +} +kubectl get event -w --field-selector reason=Started,involvedObject.kind=Pod \ + -o go-template='{{printf "%s\n" .involvedObject.name}}' | +( +while true +do + read line + if [ ! -z $line ] + then + owner=$(getowner pod/$line) + if [[ ! -z $owner && ! $owner == / ]] + then + sleep 10 + if kubectl delete pod $line + then + sleep 10 + kubectl get $owner -o yaml |grep -v -E 'image:.*-baffs$' | sed 's/image:.*$/&-baffs/' | kubectl apply -f - + fi + fi + fi +done +) diff --git a/kubernetes/scripts/unbloatpipe b/kubernetes/scripts/unbloatpipe index a4c759e..a2e4bf0 100755 --- a/kubernetes/scripts/unbloatpipe +++ b/kubernetes/scripts/unbloatpipe @@ -1,5 +1,7 @@ #!/bin/bash +# set -x + pipe=/var/run/unbloatpipe trap "rm -f $pipe" EXIT @@ -8,40 +10,44 @@ if [[ ! -p $pipe ]]; then mkfifo $pipe fi -while true -do - if read line <$pipe; then - +debloat_cycle(){ + local line=$1 + if [ ! -z `docker images $line-nobaffs --format="table {{.Repository}}:{{.Tag}}" | grep -v REPOSITORY` ] then - echo "## image $line is already processed" + echo "## image $line is already processed" continue - fi + fi - echo "## pull and shadow $line" - docker pull $line + echo "## pull and shadow $line" + docker pull $line docker tag $line "$line-nobaffs" baffs shadow --images=$line - systemctl restart kubelet echo "## wait until container is in use" - while true; - do - IMAGE_ID=$(docker image list $line --format "table {{.Repository}}:{{.Tag}} {{.ID}}" | grep -v REPOSITORY | cut -f2 -d" ") - (docker ps | grep $IMAGE_ID) && break - sleep 5 - done - - echo "## wait until container terminates" - CONTAINER_ID=$(docker ps | grep $IMAGE_ID | cut -f1 -d" ") + while true; + do + IMAGE_ID=$(docker image list $line --format "table {{.Repository}}:{{.Tag}} {{.ID}}" | grep -v REPOSITORY | cut -f2 -d" ") + (docker ps | grep $IMAGE_ID) && break + sleep 5 + done + + echo "## wait until container terminates" + CONTAINER_ID=$(docker ps | grep $IMAGE_ID | cut -f1 -d" ") docker container wait $CONTAINER_ID - - echo "## debloat terminated shadowed $line" - baffs debloat --images=$line - systemctl restart kubelet + + echo "## debloat terminated shadowed $line" + baffs debloat --images=$line docker tag "$line-baffs" $line - echo "## finished debloating $line" - fi + echo "## finished debloating $line" + +} + +while true +do + read line <$pipe + debloat_cycle $line & done + From 7fc923cc4ff124c85907f1848f78dd1b07b8c77d Mon Sep 17 00:00:00 2001 From: Thomas Fricke Date: Tue, 29 Jul 2025 13:20:31 +0200 Subject: [PATCH 5/6] semi working variant --- kubernetes/podeventlistener.sh | 51 +++++++++++++++++++++++++++------- kubernetes/scripts/unbloatpipe | 17 ++++++++++-- kubernetes/writepipe.yaml | 5 ++-- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/kubernetes/podeventlistener.sh b/kubernetes/podeventlistener.sh index 2720613..0ba5a3e 100755 --- a/kubernetes/podeventlistener.sh +++ b/kubernetes/podeventlistener.sh @@ -4,7 +4,22 @@ # gets the original owner of a pod # -set -x +#set -x + +cleanup() { + # kill all processes whose parent is this process + pkill -P $$ +} + +for sig in INT QUIT HUP TERM; do + trap " + cleanup + trap - $sig EXIT + kill -s $sig "'"$$"' "$sig" +done +trap cleanup EXIT + + getowner(){ local object=$1 @@ -25,24 +40,40 @@ getowner(){ return $? fi } -kubectl get event -w --field-selector reason=Started,involvedObject.kind=Pod \ - -o go-template='{{printf "%s\n" .involvedObject.name}}' | -( -while true -do - read line + +debloat_cycle(){ + local line=$1 if [ ! -z $line ] then owner=$(getowner pod/$line) if [[ ! -z $owner && ! $owner == / ]] - then - sleep 10 + then + for image in $(kubectl get pod $line -o jsonpath="{ .spec['initContainers', 'containers'][*].image} + ") + do + echo -n "## " + ( echo $image | grep baffs ) && echo ignored && return + cat writepipe.yaml | sed s/CONTAINER/$image/ | sed s/NAME/writepipe-$RANDOM/| kubectl create -f - + done + + sleep 20 if kubectl delete pod $line then - sleep 10 + sleep 20 + echo patching $owner kubectl get $owner -o yaml |grep -v -E 'image:.*-baffs$' | sed 's/image:.*$/&-baffs/' | kubectl apply -f - fi fi fi + +} + +kubectl get event -w --field-selector reason=Started,involvedObject.kind=Pod \ + -o go-template='{{printf "%s\n" .involvedObject.name}}' | +( +while true +do + read line + debloat_cycle $line # & done ) diff --git a/kubernetes/scripts/unbloatpipe b/kubernetes/scripts/unbloatpipe index a2e4bf0..7b9912c 100755 --- a/kubernetes/scripts/unbloatpipe +++ b/kubernetes/scripts/unbloatpipe @@ -4,7 +4,20 @@ pipe=/var/run/unbloatpipe -trap "rm -f $pipe" EXIT +cleanup() { + # kill all processes whose parent is this process + rm -f $pipe + pkill -P $$ +} + +for sig in INT QUIT HUP TERM; do + trap " + cleanup + trap - $sig EXIT + kill -s $sig "'"$$"' "$sig" +done +trap cleanup EXIT + if [[ ! -p $pipe ]]; then mkfifo $pipe @@ -48,6 +61,6 @@ debloat_cycle(){ while true do read line <$pipe - debloat_cycle $line & + debloat_cycle $line # & done diff --git a/kubernetes/writepipe.yaml b/kubernetes/writepipe.yaml index eff4bb6..0002572 100644 --- a/kubernetes/writepipe.yaml +++ b/kubernetes/writepipe.yaml @@ -1,7 +1,8 @@ apiVersion: batch/v1 kind: Job metadata: - name: writepipe + name: NAME + namespace: kube-system spec: ttlSecondsAfterFinished: 10 template: @@ -10,7 +11,7 @@ spec: spec: containers: - name: writepipe - image: busybox + image: alpine args: - /bin/sh - -c From 21f677048e5b606c5eda3390dbc046c7c48baf33 Mon Sep 17 00:00:00 2001 From: Thomas Fricke Date: Tue, 29 Jul 2025 19:30:17 +0200 Subject: [PATCH 6/6] 0.1 version which allows several deployments without restart --- kubernetes/README.md | 60 +++++++++++++++++++++++++++++----- kubernetes/podeventlistener.sh | 45 +++++++++++++++++-------- kubernetes/test.sh | 12 +++++++ 3 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 kubernetes/test.sh diff --git a/kubernetes/README.md b/kubernetes/README.md index 05a5ac2..893b4b4 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -17,26 +17,68 @@ from the very same image used by Minikube for compatibility reasons The Docker version inside is used, not need to install another version. -## How to use +## How to build the Minikube container with baffs inside Build the image with `docker build . -t minikube-baffs` and run Minikube -with `minikube start --driver=docker --base-image=minikube-baffs`. - -In the Minikube container trigger download of the image and the initial -shadowing by using +with ```bash - sed s/CONTAINER/redis:7.4.1/ writepipe.yaml | kubectl create -f - +minikube start --driver=docker --cpus=8 --memory=16384 --base-image=minikube-baffs`. ``` +In a terminal run `podeventlistener.sh`. That is all! + +## How it works + +In the Minikube container there is a Systemd service `unbloatpipe.service` +installed responsible for the shadowing process. + +- __pulling the image__ +- waiting for the image +- __creating the shadow image with `baffs`__ +- waiting the image to be + - used + - and freed +- __debloating the image with `baffs`__ + +The usage of the image is started from the Kubernetes side creating a pod and deleting it later by patching the object owning the pod. + +This is triggered by the `podeventlistener.sh` script. It listens to the event pipeline +for the creation of pod, finds the owning object, normally a deployment or a daemonset +and patches it appending the suffix `-baffs` to the image version. + +This way an update is triggered which can only be successfull after the debloat +process in the unbloatpipe is done. In between while the new container +might not exist the old pod is continued. + +It uses `writepipe.yaml` to create a job connecting to the Minikube container + +## Usage + You now can create a deplopment using this image by ```bash kubectl create deployment redis-shadow --image=redis:7.4.1 ``` -Test your application as much as possible. +Test your application as much as possible. This is emulated by a waiting period +of 20s for demo purposes. + +The object owning the pod is automatically patched and will replace the old image by +deleting the initial pod. + +## Testing + +The `test.sh` script starts several deployment to show how it works and +if the scripts are successfully recovering from restarting the services +in Minikube. + +## Maturity + +This is a proof of concept. __Do not use it in production or for anything important.__ +The `baffs` script restarts the docker daemon for each call and the `unbloatpipe` +additionally kills the Kubelet process. -Now stop the deployment by `kubectl delete deployment redis-shadow`. +To be ready for production it needs to be parallized and `baffs` must avoid to +restart the daemon. -Restart the deployment, it is using the debloated container image now! diff --git a/kubernetes/podeventlistener.sh b/kubernetes/podeventlistener.sh index 0ba5a3e..51f04aa 100755 --- a/kubernetes/podeventlistener.sh +++ b/kubernetes/podeventlistener.sh @@ -51,29 +51,48 @@ debloat_cycle(){ for image in $(kubectl get pod $line -o jsonpath="{ .spec['initContainers', 'containers'][*].image} ") do - echo -n "## " + echo "## processing image $image in pod/$line with owner $owner" ( echo $image | grep baffs ) && echo ignored && return cat writepipe.yaml | sed s/CONTAINER/$image/ | sed s/NAME/writepipe-$RANDOM/| kubectl create -f - done sleep 20 - if kubectl delete pod $line - then - sleep 20 - echo patching $owner - kubectl get $owner -o yaml |grep -v -E 'image:.*-baffs$' | sed 's/image:.*$/&-baffs/' | kubectl apply -f - - fi + echo patching $owner + kubectl get $owner -o yaml |grep -v -E 'image:.*-baffs$' | sed 's/image:.*$/&-baffs/' | kubectl apply -f - + echo deleting pod $line + while ! kubectl delete pod $line + do + if kubectl get pods + then + break + fi + echo Retry in 5 seconds + sleep 5 + done fi fi } -kubectl get event -w --field-selector reason=Started,involvedObject.kind=Pod \ - -o go-template='{{printf "%s\n" .involvedObject.name}}' | -( while true do - read line - debloat_cycle $line # & + kubectl get event -w --field-selector reason=Started,involvedObject.kind=Pod \ + -o go-template='{{printf "%s\n" .involvedObject.name}}' | \ + while read line + do + debloat_cycle $line # & + kubectl get pods -o go-template --template='{{"\nImages\n"}} + {{- range .items -}} + {{- printf "%s\n" .metadata.name -}} + {{- range .spec.initContainers -}} + {{"* "}} {{ .image }} + {{- end -}} + {{- range .spec.containers -}} + {{"- "}} {{ .image }} + {{- end -}}{{"\n"}} +{{- end -}}' | grep --color=always -E '^.*-baffs|$' + + done + sleep 10 done -) + diff --git a/kubernetes/test.sh b/kubernetes/test.sh new file mode 100644 index 0000000..390e7c4 --- /dev/null +++ b/kubernetes/test.sh @@ -0,0 +1,12 @@ + +kubectl create deployment redis-shadow-1 --image=redis:7.4.1 +kubectl create deployment redis-shadow-2 --image=redis:7.4.2 + +kubectl create deployment nginx-shadow-23 --image=nginx:1.23.0 +kubectl create deployment nginx-shadow-24 --image=nginx:1.24.0 +kubectl create deployment nginx-shadow-25 --image=nginx:1.25.0 + +kubectl create deployment redis-shadow-3 --image=redis:7.4.3 +kubectl create deployment redis-shadow-4 --image=redis:7.4.4 +kubectl create deployment redis-shadow-5 --image=redis:7.4.5 +