Kubernetes

kubernetes/Dockerfile

@@ -0,0 +1,56 @@
+#
+# Go build stage
+#
+FROM gcr.io/k8s-minikube/kicbase:v0.0.47 AS builder
+WORKDIR /build
+
+
+RUN apt-get update
+RUN apt install -y git wget cmake pkg-config 
+RUN apt-get install build-essential libfuse3-dev -y
+
+RUN wget https://go.dev/dl/go1.23.6.linux-amd64.tar.gz
+RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go1.23.6.linux-amd64.tar.gz && rm go1.23.6.linux-amd64.tar.gz
+ENV PATH=$PATH:/usr/local/go/bin
+
+#
+# Clone and make install
+# 
+RUN git clone https://github.com/negativa-ai/BLAFS
+WORKDIR BLAFS
+COPY Makefile .
+RUN make install 
+
+#
+# use the original Minikube image
+#
+FROM gcr.io/k8s-minikube/kicbase:v0.0.47
+
+RUN apt update
+
+RUN apt install libfuse3-3 -y 
+
+#
+# copy the tools build
+#
+
+COPY --from=builder /usr/bin/baffs /usr/bin/baffs
+COPY --from=builder /usr/bin/debloated_fs /usr/bin/debloated_fs
+
+#
+# configure timezone
+#
+ENV TZ=Europe/Berlin
+RUN ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime
+RUN echo "$TZ" > /etc/timezone
+
+#
+# configure systemd
+#
+
+COPY scripts/unbloatpipe.service /etc/systemd/system/unbloatpipe.service
+COPY scripts/unbloatpipe  /usr/local/bin/unbloatpipe
+
+RUN systemctl enable unbloatpipe
+
+

kubernetes/Makefile

@@ -0,0 +1,18 @@
+debloated_fs:
+	mkdir -p build
+	cd build && cmake ../fs && cmake --build .
+
+baffs:
+	CGO_ENABLED=0  go build -ldflags="-extldflags=-static" -o baffs
+
+
+install: debloated_fs baffs
+	cp build/debloated_fs /usr/bin/debloated_fs
+	cp baffs /usr/bin/baffs
+
+test: debloated_fs
+	./build/all_test
+	go test -v ./...
+
+integration_test: install
+	./tests/test.sh

kubernetes/README.md

@@ -0,0 +1,84 @@
+# Kubernetes setup
+
+This defines the files needed to run `baffs` in Kubernetes.
+
+## Images
+
+It derives everything from Minikube docker images.
+As this runs a full featured Ubuntu 22.04 LTS inside,
+it is compatible with the BLAFS containers.
+
+
+The actual Dockerfile can reuse a lot of the original Minikube image.
+The compile task has been separated in an early build stage derived
+from the very same image used by Minikube for compatibility reasons
+(the Golang build container artefacts were not compatible and the 
+`docker run ...` did not succeed on a shadowed container)
+
+The Docker version inside is used, not need to install another version.
+
+## How to build the Minikube container with baffs inside
+
+Build the image with `docker build .  -t minikube-baffs` and run Minikube
+with 
+
+```bash
+minikube start --driver=docker --cpus=8 --memory=16384  --base-image=minikube-baffs`. 
+```
+
+In a terminal run `podeventlistener.sh`. That is all!
+
+## How it works
+
+In the Minikube container there is a Systemd service `unbloatpipe.service`
+installed responsible for the shadowing process.
+
+- __pulling the image__
+- waiting for the image
+- __creating the shadow image with `baffs`__
+- waiting the image to be
+  - used 
+  - and freed
+- __debloating the image with `baffs`__ 
+
+The usage of the image is started from the Kubernetes side creating a pod and deleting it later by patching the object owning the pod.
+
+This is triggered by the `podeventlistener.sh` script. It listens to the event pipeline
+for the creation of pod, finds the owning object, normally a deployment or a daemonset
+and patches it appending the suffix `-baffs` to the image version.
+
+This way an update is triggered which can only be successfull after the debloat
+process in the unbloatpipe is done. In between while the new container
+might not exist the old pod is continued.
+
+It uses `writepipe.yaml` to create a job connecting to the Minikube container
+
+## Usage
+
+You now can create a deplopment using this image by
+
+```bash
+kubectl create deployment redis-shadow --image=redis:7.4.1
+```
+
+Test your application as much as possible. This is emulated by a waiting period
+of 20s for demo purposes.
+
+The object owning the pod is automatically patched and will replace the old image by
+deleting the initial pod.
+
+## Testing
+
+The `test.sh` script starts several deployment to show how it works and
+if the scripts are successfully recovering from restarting the services
+in Minikube.
+
+## Maturity
+
+This is a proof of concept. __Do not use it in production or for anything important.__
+The `baffs` script restarts the docker daemon for each call and the `unbloatpipe`
+additionally kills the Kubelet process.
+
+To be ready for production it needs to be parallized and `baffs` must avoid to
+restart the daemon.
+

kubernetes/podeventlistener.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+#
+# gets the original owner of a pod
+#
+
+#set -x
+
+cleanup() {
+    # kill all processes whose parent is this process
+    pkill -P $$
+}
+
+for sig in INT QUIT HUP TERM; do
+  trap "
+    cleanup
+    trap - $sig EXIT
+    kill -s $sig "'"$$"' "$sig"
+done
+trap cleanup EXIT
+
+
+
+getowner(){
+  local object=$1
+  local owner=$(kubectl get $object -o jsonpath='{ .metadata.ownerReferences[].kind}/{.metadata.ownerReferences[].name}' || return 1)
+
+
+  if [ -z $owner ]
+  then
+    return 1
+  fi
+
+  if [ $owner == '/' ] 
+  then 
+    echo $object
+    return 0
+  else
+    getowner $owner
+    return $?
+  fi
+}
+
+debloat_cycle(){
+  local line=$1
+  if [ ! -z $line ]
+  then
+    owner=$(getowner pod/$line)
+    if [[ ! -z $owner && ! $owner == / ]]
+    then
+      for image in $(kubectl get pod $line  -o jsonpath="{ .spec['initContainers', 'containers'][*].image}
+            ")
+      do 
+	 echo  "## processing image $image in pod/$line with owner $owner"
+         ( echo $image | grep baffs ) && echo ignored && return 
+         cat writepipe.yaml | sed s/CONTAINER/$image/ | sed s/NAME/writepipe-$RANDOM/| kubectl create -f -
+      done
+
+      sleep 20
+      echo patching $owner
+      kubectl get $owner -o yaml |grep -v -E 'image:.*-baffs$' | sed 's/image:.*$/&-baffs/' | kubectl apply -f -
+      echo deleting pod $line
+      while ! kubectl delete pod $line 
+      do
+	      if  kubectl get pods
+	      then 
+		      break
+	      fi
+	      echo Retry in 5 seconds
+	      sleep 5
+      done
+    fi
+  fi
+
+}
+
+while true
+do
+    kubectl get event -w --field-selector reason=Started,involvedObject.kind=Pod \
+       	-o go-template='{{printf "%s\n" .involvedObject.name}}' | \
+    while read line
+    do
+      debloat_cycle $line # &
+      kubectl get pods -o go-template --template='{{"\nImages\n"}}
+    {{- range .items -}}
+      {{- printf "%s\n" .metadata.name -}}
+      {{- range .spec.initContainers -}}
+       {{"*  "}} {{ .image }}
+    {{- end -}}
+    {{- range .spec.containers -}}
+       {{"-  "}} {{ .image }}
+    {{- end -}}{{"\n"}}
+{{- end -}}' | grep --color=always -E '^.*-baffs|$'
+
+    done
+    sleep 10
+done
+

kubernetes/scripts/unbloatpipe

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# set -x
+
+pipe=/var/run/unbloatpipe
+
+cleanup() {
+    # kill all processes whose parent is this process
+    rm -f $pipe
+    pkill -P $$
+}
+
+for sig in INT QUIT HUP TERM; do
+  trap "
+    cleanup
+    trap - $sig EXIT
+    kill -s $sig "'"$$"' "$sig"
+done
+trap cleanup EXIT
+
+
+if [[ ! -p $pipe ]]; then
+    mkfifo $pipe
+fi
+
+debloat_cycle(){
+        local line=$1
+
+        if [ ! -z  `docker images $line-nobaffs --format="table {{.Repository}}:{{.Tag}}" | grep -v REPOSITORY` ] 
+        then
+          echo "## image $line is already processed"
+          continue
+        fi
+
+        echo "## pull and shadow $line"    
+        docker pull $line
+        docker tag $line "$line-nobaffs"
+        baffs shadow --images=$line
+
+
+        echo "## wait until container is in use"
+        while true; 
+        do
+                IMAGE_ID=$(docker image list $line --format "table {{.Repository}}:{{.Tag}} {{.ID}}" | grep -v REPOSITORY | cut -f2 -d" ")
+                (docker ps | grep $IMAGE_ID)  && break
+                sleep 5
+        done
+
+        echo "## wait until container terminates"
+        CONTAINER_ID=$(docker ps | grep $IMAGE_ID | cut -f1 -d" ")
+        docker container wait $CONTAINER_ID
+
+        echo "## debloat terminated shadowed $line"
+        baffs debloat --images=$line
+        docker tag "$line-baffs" $line
+
+        echo "## finished debloating $line"     
+
+}
+
+while true
+do
+    read line <$pipe
+    debloat_cycle $line # &
+done
+

kubernetes/scripts/unbloatpipe.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=pipe to unbloat images
+
+[Service]
+Type=simple
+WorkingDirectory=/home/docker
+ExecStart=/usr/local/bin/unbloatpipe
+ExecReload=/bin/kill -HUP $MAINPID
+KillMode=process
+Restart=on-failure
+User=root
+Group=root
+
+[Install]
+WantedBy=multi-user.target

kubernetes/test.sh

@@ -0,0 +1,12 @@
+
+kubectl create deployment redis-shadow-1 --image=redis:7.4.1
+kubectl create deployment redis-shadow-2 --image=redis:7.4.2
+
+kubectl create deployment nginx-shadow-23 --image=nginx:1.23.0
+kubectl create deployment nginx-shadow-24 --image=nginx:1.24.0
+kubectl create deployment nginx-shadow-25 --image=nginx:1.25.0
+
+kubectl create deployment redis-shadow-3 --image=redis:7.4.3
+kubectl create deployment redis-shadow-4 --image=redis:7.4.4
+kubectl create deployment redis-shadow-5 --image=redis:7.4.5
+

kubernetes/writepipe.yaml

@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: NAME
+  namespace: kube-system
+spec:
+  ttlSecondsAfterFinished: 10
+  template:
+    metadata:
+      name: writepipe
+    spec:
+      containers:
+      - name: writepipe
+        image: alpine
+        args:
+        - /bin/sh
+        - -c
+        - echo CONTAINER > /tmp/run/unbloatpipe
+        volumeMounts:
+        - name: run
+          mountPath: /tmp/run
+      restartPolicy: Never
+      volumes:
+      - name: run 
+        hostPath:
+          path: /var/run
+          type: Directory