diff --git a/Makefile b/Makefile index dea07b5d..6cf3d4e8 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,18 @@ module-experimental-eirinifs: module-experimental-eirini_release: $(MAKE) -C modules/experimental eirini_release +.PHONY: module-experimental-local-registry +module-experimental-local-registry: + $(MAKE) -C modules/experimental local-registry + +.PHONY: module-experimental-push-imagelist +module-experimental-push-imagelist: + $(MAKE) -C modules/experimental push-imagelist + +.PHONY: module-experimental-podman-imagelist +module-experimental-podman-imagelist: + $(MAKE) -C modules/experimental podman-imagelist + .PHONY: module-experimental-airgap-up module-experimental-airgap-up: $(MAKE) -C modules/experimental airgap-up diff --git a/backend/aks/Makefile b/backend/aks/Makefile index 682c6b7e..f3c89f11 100644 --- a/backend/aks/Makefile +++ b/backend/aks/Makefile @@ -35,3 +35,7 @@ all:: @echo 'WARNING: stil WIP. Use it on your own risk.' @echo 'Kindly waiting for 5s…'; sleep 5 all:: deploy + +.PHONY: airgap-up airgap-down +airgap-up airgap-down: + echo "Not implemented yet" && exit 1 diff --git a/backend/caasp4os/Makefile b/backend/caasp4os/Makefile index 5cac6a23..199b8dc9 100644 --- a/backend/caasp4os/Makefile +++ b/backend/caasp4os/Makefile @@ -36,3 +36,7 @@ force-clean-cluster: .PHONY: all all: deps-caasp4os caasp4os-deploy caasp-prepare + +.PHONY: airgap-up airgap-down +airgap-up airgap-down: + echo "Not implemented yet" && exit 1 diff --git a/backend/ekcp/Makefile b/backend/ekcp/Makefile index b2d3aefd..4417c910 100644 --- a/backend/ekcp/Makefile +++ b/backend/ekcp/Makefile @@ -36,3 +36,7 @@ force-clean-cluster: .PHONY: all all: up kubeconfig prepare + +.PHONY: airgap-up airgap-down +airgap-up airgap-down: + echo "Not implemented yet" && exit 1 diff --git a/backend/eks/Makefile b/backend/eks/Makefile index 216038e6..b90344a5 100644 --- a/backend/eks/Makefile +++ b/backend/eks/Makefile @@ -32,3 +32,7 @@ force-clean-cluster: .PHONY: all all: deploy + +.PHONY: airgap-up airgap-down +airgap-up airgap-down: + echo "Not implemented yet" && exit 1 diff --git a/backend/gke/Makefile b/backend/gke/Makefile index b0d02c6b..59a0c592 100644 --- a/backend/gke/Makefile +++ b/backend/gke/Makefile @@ -30,3 +30,11 @@ force-clean-cluster: .PHONY: all all: deploy + +.PHONY: airgap-up +airgap-up: + ./airgap-up.sh + +.PHONY: airgap-down +airgap-down: + ./airgap-down.sh diff --git a/backend/gke/airgap-down.sh b/backend/gke/airgap-down.sh new file mode 100755 index 00000000..88b462a6 --- /dev/null +++ b/backend/gke/airgap-down.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +. ./defaults.sh +. ../../include/common.sh +. .envrc + +export KUBECF_NAMESPACE=scf +export QUARKS_NAMESPACE=cf-operator +gke_isolate_network 0 diff --git a/backend/gke/airgap-up.sh b/backend/gke/airgap-up.sh new file mode 100755 index 00000000..458de73f --- /dev/null +++ b/backend/gke/airgap-up.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +. ./defaults.sh +. ../../include/common.sh +. .envrc + +export KUBECF_NAMESPACE=scf +export QUARKS_NAMESPACE=cf-operator +for ns in $KUBECF_NAMESPACE $QUARKS_NAMESPACE default; do + kubectl create namespace $ns || true + kubectl label namespaces --overwrite $ns airgap=true +done + +gke_isolate_network 1 diff --git a/backend/kind/Makefile b/backend/kind/Makefile index f3a4a051..89cfa3de 100644 --- a/backend/kind/Makefile +++ b/backend/kind/Makefile @@ -54,3 +54,7 @@ stop: .PHONY: restart restart: ./restart.sh + +.PHONY: airgap-up airgap-down +airgap-up airgap-down: + echo "Not implemented yet" && exit 1 diff --git a/backend/minikube/Makefile b/backend/minikube/Makefile index c4352332..5e4bbee6 100644 --- a/backend/minikube/Makefile +++ b/backend/minikube/Makefile @@ -38,3 +38,7 @@ start: .PHONY: stop stop: ./stop.sh + +.PHONY: airgap-up airgap-down +airgap-up airgap-down: + echo "Not implemented yet" && exit 1 diff --git a/include/func.sh b/include/func.sh index 5ab11125..4a3da5ec 100644 --- a/include/func.sh +++ b/include/func.sh @@ -375,3 +375,140 @@ wait_for_cf-operator() { wait_for "kubectl delete -f ../kube/cf-operator/qstatefulset_tolerations.yaml --namespace=scf" fi } + + +gke_isolate_network() { + enable="${1:-1}" + + if [[ $enable == 1 ]]; then + # Complaint wrong. The echo generates a traling newline the `blue` doesn't. + # shellcheck disable=SC2005 + echo "$(blue "Configure cluster network: Deny egress external")" + # enable isolation + # ingress - allows all incoming traffic + # egress - allows dns traffic anywhere + # - allows traffic to all ports, pods, namespaces + # (but no whitelisting of external ips!) + # references + # - BASE = https://github.com/ahmetb/kubernetes-network-policy-recipes + # - (BASE)/blob/master/02a-allow-all-traffic-to-an-application.md + # - (BASE)/blob/master/14-deny-external-egress-traffic.md + # - See also https://www.youtube.com/watch?v=3gGpMmYeEO8 (31min) + # - Egress info wrt disallow external see 17:20-17:52 + # + # __ATTENTION__ + # Requires a networking plugin to enforce, else ignored + # (if not directly supported by platform) + # - Example plugins: Calico, WeaveNet, Romana + # + # GKE: Uses Calico, Use `--enable-network-policy` when + # creating a cluster (`gcloud`). + # Minikube needs special setup. + # KinD used by our Drone setup may have support. + + cat </dev/null || true kubectl create -n cf-operator -f ../modules/experimental/cilium-block-egress.yaml kubectl create -n scf -f ../modules/experimental/cilium-block-egress.yaml +# # test that indeed it is airgapped: +# if ! kubectl run hello-world --image=hello-world >/dev/null; then +# err "Airgap enabled, but could download an external hello-world container image" +# exit 1 +# fi +# kubectl delete pod hello-world --ignore-not-found + info "Cluster ${CLUSTER_NAME} is now running a simulated airgapped setup. Run \`make module-experimental-airgap-down\` to restore internet access" diff --git a/modules/experimental/docker/Dockerfile.skopeo b/modules/experimental/docker/Dockerfile.skopeo new file mode 100644 index 00000000..4cf67d45 --- /dev/null +++ b/modules/experimental/docker/Dockerfile.skopeo @@ -0,0 +1,5 @@ +FROM opensuse/leap:latest +LABEL MAINTAINER="Víctor Cuadrado Juan , Christian Richter " +RUN zypper ref \ + && zypper in -y --no-recommends skopeo \ + && zypper clean --all \ No newline at end of file diff --git a/modules/experimental/local-registry.sh b/modules/experimental/local-registry.sh new file mode 100755 index 00000000..e629bf22 --- /dev/null +++ b/modules/experimental/local-registry.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +. ./defaults.sh +. ../../include/common.sh +. .envrc + +info "Creating local registry at local-registry.default.svc.cluster.local…" + +kubectl apply -f - <> /etc/containers/registries.conf + echo '[registries.Insecure]' >> /etc/containers/registries.conf + echo "registries = ['local-registry.default.svc.cluster.local']" >> /etc/containers/registries.conf + apk add --no-cache skopeo + while true; do sleep 100; done +HEREDOC + +# create push-imagelist.sh script +cat >push-imagelist.sh << 'HEREDOC' +#!/usr/bin/env sh + +set -e + +while IFS= read -r file +do + for SOURCE_IMAGE in $(cat "$imagelist_file"); do + MIRROR='local-registry.default.svc.cluster.local' + echo ">>>>> Mirroring image: ${SOURCE_IMAGE}" + # add registry.suse.com/cap/ in front if it's not already there + if [[ $( echo $SOURCE_IMAGE | grep -o '/' | tr -d '\r\n' | wc -c) -lt 1 ]]; then + # SOURCE_IMAGE=docker.io/"$SOURCE_IMAGE" # upstream + SOURCE_IMAGE=registry.suse.com/cap/$SOURCE_IMAGE # cap + fi + + org=$(echo "$SOURCE_IMAGE" | cut -d/ -f2) + image=$(echo "$SOURCE_IMAGE" | cut -d/ -f3) + + skopeo copy "docker://${SOURCE_IMAGE}" "docker://${MIRROR}/${org}/${image} 2>&1 >/dev/null" & + done +done < <(find . -name "imagelist.txt") + + +wait +echo +echo "Done pushing images" +HEREDOC +chmod +x push-imagelist.sh + +# wait for the pod to be up +sleep 30 + +# copy push-imagelist.sh script +kubectl cp push-imagelist.sh "${KUBECF_NAMESPACE}"/push-imagelist-pod:/tmp/push-imagelist.sh + +# copy all imagelist.txt files +while IFS= read -r imagelist_file +do + kubectl cp -n "${KUBECF_NAMESPACE}" "$imagelist_file" \ + "${KUBECF_NAMESPACE}"/push-imagelist-pod:/tmp/"$imagelist_file" +done < <(find . -name "imagelist.txt") + +# start the script in the pod +# TODO +# for now, just cd tmp; ./push-imagelist.sh diff --git a/modules/experimental/push-imagelist.sh b/modules/experimental/push-imagelist.sh new file mode 100755 index 00000000..36d3baaa --- /dev/null +++ b/modules/experimental/push-imagelist.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +. ./defaults.sh +. ../../include/common.sh +. .envrc + +info "Pushing all imagelist.txt images to local-registry.default.svc.cluster.local" + +kubectl create namespace push-imagelist 2>/dev/null || true + +# POSIX compliant: +while IFS= read -r imagelist_file +do + # Create a kube job per image in imagelist.txt. There, pull it, retag it, + # and push the image to the local registry + for SOURCE_IMAGE in $(cat "$imagelist_file"); do + JOB_NAME=${SOURCE_IMAGE%%\:*} # remove suffix after ':' + JOB_NAME=${JOB_NAME////-} # substitute '/' with '-' + JOB_NAME=${JOB_NAME:0:63} # truncate to 63 charts for kubernetes + TARGET_REGISTRY='local-registry.default.svc.cluster.local' + kubectl apply --overwrite=false -f - <> /etc/containers/registries.conf + echo '[registries.Insecure]' >> /etc/containers/registries.conf + echo "registries = ['${TARGET_REGISTRY}']" >> /etc/containers/registries.conf + + # add registry.suse.com/cap/ in front if it's not already there + if [[ $( echo "$SOURCE_IMAGE" | grep -o '/' | tr -d '\r\n' | wc -c) -lt 1 ]]; then + # SOURCE_IMAGE=docker.io/"$SOURCE_IMAGE" # upstream + SOURCE_IMAGE=registry.suse.com/cap/$SOURCE_IMAGE # cap + fi + + org=\$(echo \$SOURCE_IMAGE | cut -d/ -f2) + image=\$(echo \$SOURCE_IMAGE | cut -d/ -f3) + + echo "Mirroring image \$SOURCE_IMAGE to ${TARGET_REGISTRY}" + skopeo copy "docker://\${SOURCE_IMAGE}" "docker://${TARGET_REGISTRY}/\$org/\$image" + echo "Mirrored image \$SOURCE_IMAGE in ${TARGET_REGISTRY}" + restartPolicy: Never + backoffLimit: 4 +HEREDOC + done +done < <(find . -name "imagelist.txt") + +info "Waiting for jobs to finish pushing images…" +kubectl -n push-imagelist wait --for=condition=complete job --all --timeout=-1s + +ok "All images pushed to local-registry.default.svc.cluster.local" diff --git a/modules/tests/kubecf-test.sh b/modules/tests/kubecf-test.sh index dabb3f88..aaaec494 100755 --- a/modules/tests/kubecf-test.sh +++ b/modules/tests/kubecf-test.sh @@ -58,72 +58,6 @@ wait_for_tests_pod() { wait_for_timeout 300 is_pod_started "${pod_name}" "${container_name}" } -isolate_network() { - enable="${1:-1}" - - if [[ $enable == 1 ]]; then - # Complaint wrong. The echo generates a traling newline the `blue` doesn't. - # shellcheck disable=SC2005 - echo "$(blue "Configure cluster network: Deny egress external")" - # enable isolation - # ingress - allows all incoming traffic - # egress - allows dns traffic anywhere - # - allows traffic to all ports, pods, namespaces - # (but no whitelisting of external ips!) - # references - # - BASE = https://github.com/ahmetb/kubernetes-network-policy-recipes - # - (BASE)/blob/master/02a-allow-all-traffic-to-an-application.md - # - (BASE)/blob/master/14-deny-external-egress-traffic.md - # - See also https://www.youtube.com/watch?v=3gGpMmYeEO8 (31min) - # - Egress info wrt disallow external see 17:20-17:52 - # - # __ATTENTION__ - # Requires a networking plugin to enforce, else ignored - # (if not directly supported by platform) - # - Example plugins: Calico, WeaveNet, Romana - # - # GKE: Uses Calico, Use `--enable-network-policy` when - # creating a cluster (`gcloud`). - # Minikube needs special setup. - # KinD used by our Drone setup may have support. - - cat <