diff --git a/kubernetes/Dockerfile b/kubernetes/Dockerfile new file mode 100644 index 0000000..1774a88 --- /dev/null +++ b/kubernetes/Dockerfile @@ -0,0 +1,56 @@ +# +# Go build stage +# +FROM gcr.io/k8s-minikube/kicbase:v0.0.47 AS builder +WORKDIR /build + + +RUN apt-get update +RUN apt install -y git wget cmake pkg-config +RUN apt-get install build-essential libfuse3-dev -y + +RUN wget https://go.dev/dl/go1.23.6.linux-amd64.tar.gz +RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go1.23.6.linux-amd64.tar.gz && rm go1.23.6.linux-amd64.tar.gz +ENV PATH=$PATH:/usr/local/go/bin + +# +# Clone and make install +# +RUN git clone https://github.com/negativa-ai/BLAFS +WORKDIR BLAFS +COPY Makefile . +RUN make install + +# +# use the original Minikube image +# +FROM gcr.io/k8s-minikube/kicbase:v0.0.47 + +RUN apt update + +RUN apt install libfuse3-3 -y + +# +# copy the tools build +# + +COPY --from=builder /usr/bin/baffs /usr/bin/baffs +COPY --from=builder /usr/bin/debloated_fs /usr/bin/debloated_fs + +# +# configure timezone +# +ENV TZ=Europe/Berlin +RUN ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime +RUN echo "$TZ" > /etc/timezone + +# +# configure systemd +# + +COPY scripts/unbloatpipe.service /etc/systemd/system/unbloatpipe.service +COPY scripts/unbloatpipe /usr/local/bin/unbloatpipe + +RUN systemctl enable unbloatpipe + + diff --git a/kubernetes/Makefile b/kubernetes/Makefile new file mode 100644 index 0000000..ec8bc9d --- /dev/null +++ b/kubernetes/Makefile @@ -0,0 +1,18 @@ +debloated_fs: + mkdir -p build + cd build && cmake ../fs && cmake --build . + +baffs: + CGO_ENABLED=0 go build -ldflags="-extldflags=-static" -o baffs + + +install: debloated_fs baffs + cp build/debloated_fs /usr/bin/debloated_fs + cp baffs /usr/bin/baffs + +test: debloated_fs + ./build/all_test + go test -v ./... + +integration_test: install + ./tests/test.sh diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 0000000..893b4b4 --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,84 @@ +# Kubernetes setup + +This defines the files needed to run `baffs` in Kubernetes. + +## Images + +It derives everything from Minikube docker images. +As this runs a full featured Ubuntu 22.04 LTS inside, +it is compatible with the BLAFS containers. + + +The actual Dockerfile can reuse a lot of the original Minikube image. +The compile task has been separated in an early build stage derived +from the very same image used by Minikube for compatibility reasons +(the Golang build container artefacts were not compatible and the +`docker run ...` did not succeed on a shadowed container) + +The Docker version inside is used, not need to install another version. + +## How to build the Minikube container with baffs inside + +Build the image with `docker build . -t minikube-baffs` and run Minikube +with + +```bash +minikube start --driver=docker --cpus=8 --memory=16384 --base-image=minikube-baffs`. +``` + +In a terminal run `podeventlistener.sh`. That is all! + +## How it works + +In the Minikube container there is a Systemd service `unbloatpipe.service` +installed responsible for the shadowing process. + +- __pulling the image__ +- waiting for the image +- __creating the shadow image with `baffs`__ +- waiting the image to be + - used + - and freed +- __debloating the image with `baffs`__ + +The usage of the image is started from the Kubernetes side creating a pod and deleting it later by patching the object owning the pod. + +This is triggered by the `podeventlistener.sh` script. It listens to the event pipeline +for the creation of pod, finds the owning object, normally a deployment or a daemonset +and patches it appending the suffix `-baffs` to the image version. + +This way an update is triggered which can only be successfull after the debloat +process in the unbloatpipe is done. In between while the new container +might not exist the old pod is continued. + +It uses `writepipe.yaml` to create a job connecting to the Minikube container + +## Usage + +You now can create a deplopment using this image by + +```bash +kubectl create deployment redis-shadow --image=redis:7.4.1 +``` + +Test your application as much as possible. This is emulated by a waiting period +of 20s for demo purposes. + +The object owning the pod is automatically patched and will replace the old image by +deleting the initial pod. + +## Testing + +The `test.sh` script starts several deployment to show how it works and +if the scripts are successfully recovering from restarting the services +in Minikube. + +## Maturity + +This is a proof of concept. __Do not use it in production or for anything important.__ +The `baffs` script restarts the docker daemon for each call and the `unbloatpipe` +additionally kills the Kubelet process. + +To be ready for production it needs to be parallized and `baffs` must avoid to +restart the daemon. + diff --git a/kubernetes/podeventlistener.sh b/kubernetes/podeventlistener.sh new file mode 100755 index 0000000..51f04aa --- /dev/null +++ b/kubernetes/podeventlistener.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# +# +# gets the original owner of a pod +# + +#set -x + +cleanup() { + # kill all processes whose parent is this process + pkill -P $$ +} + +for sig in INT QUIT HUP TERM; do + trap " + cleanup + trap - $sig EXIT + kill -s $sig "'"$$"' "$sig" +done +trap cleanup EXIT + + + +getowner(){ + local object=$1 + local owner=$(kubectl get $object -o jsonpath='{ .metadata.ownerReferences[].kind}/{.metadata.ownerReferences[].name}' || return 1) + + + if [ -z $owner ] + then + return 1 + fi + + if [ $owner == '/' ] + then + echo $object + return 0 + else + getowner $owner + return $? + fi +} + +debloat_cycle(){ + local line=$1 + if [ ! -z $line ] + then + owner=$(getowner pod/$line) + if [[ ! -z $owner && ! $owner == / ]] + then + for image in $(kubectl get pod $line -o jsonpath="{ .spec['initContainers', 'containers'][*].image} + ") + do + echo "## processing image $image in pod/$line with owner $owner" + ( echo $image | grep baffs ) && echo ignored && return + cat writepipe.yaml | sed s/CONTAINER/$image/ | sed s/NAME/writepipe-$RANDOM/| kubectl create -f - + done + + sleep 20 + echo patching $owner + kubectl get $owner -o yaml |grep -v -E 'image:.*-baffs$' | sed 's/image:.*$/&-baffs/' | kubectl apply -f - + echo deleting pod $line + while ! kubectl delete pod $line + do + if kubectl get pods + then + break + fi + echo Retry in 5 seconds + sleep 5 + done + fi + fi + +} + +while true +do + kubectl get event -w --field-selector reason=Started,involvedObject.kind=Pod \ + -o go-template='{{printf "%s\n" .involvedObject.name}}' | \ + while read line + do + debloat_cycle $line # & + kubectl get pods -o go-template --template='{{"\nImages\n"}} + {{- range .items -}} + {{- printf "%s\n" .metadata.name -}} + {{- range .spec.initContainers -}} + {{"* "}} {{ .image }} + {{- end -}} + {{- range .spec.containers -}} + {{"- "}} {{ .image }} + {{- end -}}{{"\n"}} +{{- end -}}' | grep --color=always -E '^.*-baffs|$' + + done + sleep 10 +done + diff --git a/kubernetes/scripts/unbloatpipe b/kubernetes/scripts/unbloatpipe new file mode 100755 index 0000000..7b9912c --- /dev/null +++ b/kubernetes/scripts/unbloatpipe @@ -0,0 +1,66 @@ +#!/bin/bash + +# set -x + +pipe=/var/run/unbloatpipe + +cleanup() { + # kill all processes whose parent is this process + rm -f $pipe + pkill -P $$ +} + +for sig in INT QUIT HUP TERM; do + trap " + cleanup + trap - $sig EXIT + kill -s $sig "'"$$"' "$sig" +done +trap cleanup EXIT + + +if [[ ! -p $pipe ]]; then + mkfifo $pipe +fi + +debloat_cycle(){ + local line=$1 + + if [ ! -z `docker images $line-nobaffs --format="table {{.Repository}}:{{.Tag}}" | grep -v REPOSITORY` ] + then + echo "## image $line is already processed" + continue + fi + + echo "## pull and shadow $line" + docker pull $line + docker tag $line "$line-nobaffs" + baffs shadow --images=$line + + + echo "## wait until container is in use" + while true; + do + IMAGE_ID=$(docker image list $line --format "table {{.Repository}}:{{.Tag}} {{.ID}}" | grep -v REPOSITORY | cut -f2 -d" ") + (docker ps | grep $IMAGE_ID) && break + sleep 5 + done + + echo "## wait until container terminates" + CONTAINER_ID=$(docker ps | grep $IMAGE_ID | cut -f1 -d" ") + docker container wait $CONTAINER_ID + + echo "## debloat terminated shadowed $line" + baffs debloat --images=$line + docker tag "$line-baffs" $line + + echo "## finished debloating $line" + +} + +while true +do + read line <$pipe + debloat_cycle $line # & +done + diff --git a/kubernetes/scripts/unbloatpipe.service b/kubernetes/scripts/unbloatpipe.service new file mode 100644 index 0000000..61b7b7e --- /dev/null +++ b/kubernetes/scripts/unbloatpipe.service @@ -0,0 +1,15 @@ +[Unit] +Description=pipe to unbloat images + +[Service] +Type=simple +WorkingDirectory=/home/docker +ExecStart=/usr/local/bin/unbloatpipe +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure +User=root +Group=root + +[Install] +WantedBy=multi-user.target diff --git a/kubernetes/test.sh b/kubernetes/test.sh new file mode 100644 index 0000000..390e7c4 --- /dev/null +++ b/kubernetes/test.sh @@ -0,0 +1,12 @@ + +kubectl create deployment redis-shadow-1 --image=redis:7.4.1 +kubectl create deployment redis-shadow-2 --image=redis:7.4.2 + +kubectl create deployment nginx-shadow-23 --image=nginx:1.23.0 +kubectl create deployment nginx-shadow-24 --image=nginx:1.24.0 +kubectl create deployment nginx-shadow-25 --image=nginx:1.25.0 + +kubectl create deployment redis-shadow-3 --image=redis:7.4.3 +kubectl create deployment redis-shadow-4 --image=redis:7.4.4 +kubectl create deployment redis-shadow-5 --image=redis:7.4.5 + diff --git a/kubernetes/writepipe.yaml b/kubernetes/writepipe.yaml new file mode 100644 index 0000000..0002572 --- /dev/null +++ b/kubernetes/writepipe.yaml @@ -0,0 +1,27 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: NAME + namespace: kube-system +spec: + ttlSecondsAfterFinished: 10 + template: + metadata: + name: writepipe + spec: + containers: + - name: writepipe + image: alpine + args: + - /bin/sh + - -c + - echo CONTAINER > /tmp/run/unbloatpipe + volumeMounts: + - name: run + mountPath: /tmp/run + restartPolicy: Never + volumes: + - name: run + hostPath: + path: /var/run + type: Directory