Add all needed targets to perform airgap installs on GKE

Makefile

@@ -173,6 +173,18 @@ module-experimental-eirinifs:
 module-experimental-eirini_release:
 	$(MAKE) -C modules/experimental eirini_release
 
+.PHONY: module-experimental-local-registry
+module-experimental-local-registry:
+	$(MAKE) -C modules/experimental local-registry
+
+.PHONY: module-experimental-push-imagelist
+module-experimental-push-imagelist:
+	$(MAKE) -C modules/experimental push-imagelist
+
+.PHONY: module-experimental-podman-imagelist
+module-experimental-podman-imagelist:
+	$(MAKE) -C modules/experimental podman-imagelist
+
 .PHONY: module-experimental-airgap-up
 module-experimental-airgap-up:
 	$(MAKE) -C modules/experimental airgap-up

backend/aks/Makefile

@@ -35,3 +35,7 @@ all::
 	@echo 'WARNING: stil WIP. Use it on your own risk.'
 	@echo 'Kindly waiting for 5s…'; sleep 5
 all:: deploy
+
+.PHONY: airgap-up airgap-down
+airgap-up airgap-down:
+	echo "Not implemented yet" && exit 1

backend/caasp4os/Makefile

@@ -36,3 +36,7 @@ force-clean-cluster:
 
 .PHONY: all
 all: deps-caasp4os caasp4os-deploy caasp-prepare
+
+.PHONY: airgap-up airgap-down
+airgap-up airgap-down:
+	echo "Not implemented yet" && exit 1

backend/ekcp/Makefile

@@ -36,3 +36,7 @@ force-clean-cluster:
 
 .PHONY: all
 all: up kubeconfig prepare
+
+.PHONY: airgap-up airgap-down
+airgap-up airgap-down:
+	echo "Not implemented yet" && exit 1

backend/eks/Makefile

@@ -32,3 +32,7 @@ force-clean-cluster:
 
 .PHONY: all
 all: deploy
+
+.PHONY: airgap-up airgap-down
+airgap-up airgap-down:
+	echo "Not implemented yet" && exit 1

backend/gke/Makefile

@@ -30,3 +30,11 @@ force-clean-cluster:
 
 .PHONY: all
 all: deploy
+
+.PHONY: airgap-up
+airgap-up:
+	./airgap-up.sh
+
+.PHONY: airgap-down
+airgap-down:
+	./airgap-down.sh

backend/gke/airgap-down.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+. ./defaults.sh
+. ../../include/common.sh
+. .envrc
+
+export KUBECF_NAMESPACE=scf
+export QUARKS_NAMESPACE=cf-operator
+gke_isolate_network 0

backend/gke/airgap-up.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+. ./defaults.sh
+. ../../include/common.sh
+. .envrc
+
+export KUBECF_NAMESPACE=scf
+export QUARKS_NAMESPACE=cf-operator
+for ns in $KUBECF_NAMESPACE $QUARKS_NAMESPACE default; do
+    kubectl create namespace $ns || true
+    kubectl label namespaces --overwrite $ns airgap=true
+done
+
+gke_isolate_network 1

backend/kind/Makefile

@@ -54,3 +54,7 @@ stop:
 .PHONY: restart
 restart:
 	./restart.sh
+
+.PHONY: airgap-up airgap-down
+airgap-up airgap-down:
+	echo "Not implemented yet" && exit 1

backend/minikube/Makefile

@@ -38,3 +38,7 @@ start:
 .PHONY: stop
 stop:
 	./stop.sh
+
+.PHONY: airgap-up airgap-down
+airgap-up airgap-down:
+	echo "Not implemented yet" && exit 1

include/func.sh

@@ -375,3 +375,140 @@ wait_for_cf-operator() {
       wait_for "kubectl delete -f ../kube/cf-operator/qstatefulset_tolerations.yaml --namespace=scf"
     fi
 }
+
+
+gke_isolate_network() {
+    enable="${1:-1}"
+
+    if [[ $enable == 1 ]]; then
+      # Complaint wrong. The echo generates a traling newline the `blue` doesn't.
+      # shellcheck disable=SC2005
+      echo "$(blue "Configure cluster network: Deny egress external")"
+      # enable isolation
+      # ingress - allows all incoming traffic
+      # egress  - allows dns traffic anywhere
+      #         - allows traffic to all ports, pods, namespaces
+      #           (but no whitelisting of external ips!)
+      # references
+      # - BASE = https://github.com/ahmetb/kubernetes-network-policy-recipes
+      # - (BASE)/blob/master/02a-allow-all-traffic-to-an-application.md
+      # - (BASE)/blob/master/14-deny-external-egress-traffic.md
+      # - See also https://www.youtube.com/watch?v=3gGpMmYeEO8 (31min)
+      #   - Egress info wrt disallow external see 17:20-17:52
+      #
+      # __ATTENTION__
+      # Requires a networking plugin to enforce, else ignored
+      # (if not directly supported by platform)
+      # - Example plugins: Calico, WeaveNet, Romana
+      #
+      # GKE: Uses Calico, Use `--enable-network-policy` when
+      # creating a cluster (`gcloud`).
+      # Minikube needs special setup.
+      # KinD used by our Drone setup may have support.
+
+      cat <<EOF | kubectl apply -f -
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: airgap-quarks
+  namespace: ${QUARKS_NAMESPACE}
+spec:
+  podSelector: {}
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  - from:
+    - ipBlock:
+        cidr: 10.0.0.0/8
+    - namespaceSelector: {}
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 10.0.0.0/8
+    - namespaceSelector: {}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: airgap-kubecf
+  namespace: ${KUBECF_NAMESPACE}
+spec:
+  podSelector: {}
+  policyTypes:
+  - Ingress
+  - Egress
+  ingress:
+  - from:
+    - ipBlock:
+        cidr: 10.0.0.0/8
+    - namespaceSelector: {}
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 10.0.0.0/8
+    - namespaceSelector: {}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: dnsoutbound-quarks
+  namespace: ${QUARKS_NAMESPACE}
+spec:
+  podSelector: {}
+  policyTypes:
+  - Egress
+  egress:
+    - ports:
+      - port: 53
+        protocol: UDP
+      - port: 53
+        protocol: TCP
+    - to:
+      - namespaceSelector: {}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: dnsoutbound-kubecf
+  namespace: ${KUBECF_NAMESPACE}
+spec:
+  podSelector: {}
+  policyTypes:
+  - Egress
+  egress:
+    - ports:
+      - port: 53
+        protocol: UDP
+      - port: 53
+        protocol: TCP
+    - to:
+      - namespaceSelector: {}
+EOF
+      # For debugging, show what kube thinks of it.
+      kubectl describe networkpolicies \
+              --namespace "${QUARKS_NAMESPACE}" \
+              airgap-quarks
+      kubectl describe networkpolicies \
+        --namespace "${KUBECF_NAMESPACE}" \
+        airgap-kubecf
+      kubectl describe networkpolicies \
+              --namespace "${QUARKS_NAMESPACE}" \
+              dnsoutbound-quarks
+      kubectl describe networkpolicies \
+              --namespace "${KUBECF_NAMESPACE}" \
+              dnsoutbound-kubecf
+    else
+      # shellcheck disable=SC2005
+      echo "$(blue "Configure cluster network: Full access")"
+      # disable isolation
+      kubectl delete networkpolicies \
+        --namespace "${KUBECF_NAMESPACE}" \
+        --ignore-not-found \
+        airgap-kubecf dnsoutbound-kubecf
+      kubectl delete networkpolicies \
+        --namespace "${QUARKS_NAMESPACE}" \
+        --ignore-not-found \
+        airgap-quarks dnsoutbound-quarks
+    fi
+}

modules/experimental/Makefile

@@ -9,8 +9,20 @@ eirini_release:
 eirinifs:
 	./eirinifs.sh
 
+.PHONY: local-registry
+local-registry:
+	./local-registry.sh
+
+.PHONY: push-imagelist
+push-imagelist:
+	./push-imagelist.sh
+
+.PHONY: podman-imagelist
+podman-imagelist:
+	./podman-imagelist.sh
+
 .PHONY: airgap-up
-airgap-up: airgap-down
+airgap-up:
 	./airgap_up.sh
 
 PHONY: airgap-down

modules/experimental/airgap_down.sh

@@ -33,6 +33,13 @@ sudo -s << 'EOS'
     echo "Could not remove DROP 0.0.0.0/0 in OUTPUT chain. Skipping other iptable deletions"
   fi
 EOS
+EOF
+
+  # Remove insecure registry
+  # shellcheck disable=SC2087
+  ssh -T sles@${kube_node} << EOF
+sudo sed 's/^\(CRIO_OPTIONS\s*=\s*\).*$/\1""/' \
+  /etc/sysconfig/crio
 EOF
 }
 

modules/experimental/airgap_up.sh

@@ -9,6 +9,8 @@ if [[ ${BACKEND} != "caasp4os" ]]; then
   exit 1
 fi
 
+DOCKER_REGISTRY=registry.scf.svc.cluster.local
+
 airgap_up_node() {
   local kube_node host_ip
   kube_node=$1
@@ -30,6 +32,13 @@ sudo -s << 'EOS'
   iptables -A OUTPUT -j ACCEPT -d 192.168.0.0/16
   iptables -A OUTPUT -j DROP -d 0.0.0.0/0
 EOS
+EOF
+
+  # Include insecure registry
+  # shellcheck disable=SC2087
+  ssh -T sles@${kube_node} << EOF
+sudo sed 's/^\(CRIO_OPTIONS\s*=\s*\).*$/\1"--insecure-registry=${DOCKER_REGISTRY}"/' \
+  /etc/sysconfig/crio
 EOF
 }
 
@@ -44,5 +53,12 @@ kubectl create namespace scf 2>/dev/null || true
 kubectl create -n cf-operator -f ../modules/experimental/cilium-block-egress.yaml
 kubectl create -n scf -f ../modules/experimental/cilium-block-egress.yaml
 
+# # test that indeed it is airgapped:
+# if ! kubectl run hello-world --image=hello-world >/dev/null; then
+#     err "Airgap enabled, but could download an external hello-world container image"
+#     exit 1
+# fi
+# kubectl delete pod hello-world --ignore-not-found
+
 info "Cluster ${CLUSTER_NAME} is now running a simulated airgapped setup. Run \`make module-experimental-airgap-down\` to restore internet access"
 

modules/experimental/docker/Dockerfile.skopeo

@@ -0,0 +1,5 @@
+FROM opensuse/leap:latest
+LABEL MAINTAINER="Víctor Cuadrado Juan <vcuadradojuan@suse.com>, Christian Richter <crichter@suse.com>"
+RUN zypper ref \
+  && zypper in -y --no-recommends skopeo \
+  && zypper clean --all

modules/experimental/local-registry.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+. ./defaults.sh
+. ../../include/common.sh
+. .envrc
+
+info "Creating local registry at local-registry.default.svc.cluster.local…"
+
+kubectl apply -f - <<HEREDOC
+---
+# PVC for local registry service
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: registry-data-pvc
+  labels:
+    app: local-registry
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 30Gi
+---
+# Docker registry pod definition
+apiVersion: v1
+kind: Pod
+metadata:
+  name: local-registry
+  labels:
+    app: local-registry
+  namespace: default
+spec:
+  volumes:
+    - name: registry-data-vol
+      persistentVolumeClaim:
+        claimName: registry-data-pvc
+  containers:
+    - name: local-registry
+      image: registry:2
+      imagePullPolicy: Always
+      ports:
+        - containerPort: 5000
+      volumeMounts:
+        - mountPath: /var/lib/registry
+          name: registry-data-vol
+---
+# Registry service definition
+kind: Service
+apiVersion: v1
+metadata:
+  name: local-registry
+  namespace: default
+spec:
+  selector:
+    app: local-registry
+  ports:
+    - port: 80
+      targetPort: 5000
+HEREDOC
+
+wait_ns default
+
+ok "Registry created"
+kubectl get services local-registry -n default