CORS-4334: Konnectivity

data/data/bootstrap/files/opt/openshift/egress-selector-config.yaml

@@ -0,0 +1,15 @@
+apiVersion: apiserver.k8s.io/v1beta1
+kind: EgressSelectorConfiguration
+egressSelections:
+- name: cluster
+  connection:
+    proxyProtocol: GRPC
+    transport:
+      uds:
+        udsName: /etc/kubernetes/config/konnectivity-server.socket
+- name: controlplane
+  connection:
+    proxyProtocol: Direct
+- name: etcd
+  connection:
+    proxyProtocol: Direct

data/data/bootstrap/files/opt/openshift/konnectivity-agent-certs-secret.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: konnectivity-agent-certs
+  namespace: openshift-bootstrap-konnectivity
+  labels:
+    app: konnectivity-agent
+    openshift.io/bootstrap-only: "true"
+type: Opaque
+data:
+  tls.crt: ${KONNECTIVITY_AGENT_CERT_BASE64}
+  tls.key: ${KONNECTIVITY_AGENT_KEY_BASE64}
+  ca.crt: ${KONNECTIVITY_CA_CERT_BASE64}

data/data/bootstrap/files/opt/openshift/konnectivity-agent-daemonset.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: konnectivity-agent
+  namespace: openshift-bootstrap-konnectivity
+  labels:
+    app: konnectivity-agent
+    openshift.io/bootstrap-only: "true"
+spec:
+  selector:
+    matchLabels:
+      app: konnectivity-agent
+  template:
+    metadata:
+      labels:
+        app: konnectivity-agent
+    spec:
+      hostNetwork: true
+      dnsPolicy: Default
+      priorityClassName: system-node-critical
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: konnectivity-agent
+        image: ${KONNECTIVITY_IMAGE}
+        command:
+        - /usr/bin/proxy-agent
+        args:
+        - --logtostderr=true
+        - --ca-cert=/etc/konnectivity/ca.crt
+        - --agent-cert=/etc/konnectivity/tls.crt
+        - --agent-key=/etc/konnectivity/tls.key
+        - --proxy-server-host=${BOOTSTRAP_NODE_IP}
+        - --proxy-server-port=8091
+        - --health-server-port=2041
+        - --agent-identifiers=default-route=true
+        - --keepalive-time=30s
+        - --probe-interval=5s
+        - --sync-interval=5s
+        - --sync-interval-cap=30s
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 2041
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        resources:
+          requests:
+            cpu: 40m
+            memory: 50Mi
+        volumeMounts:
+        - name: konnectivity-certs
+          mountPath: /etc/konnectivity
+          readOnly: true
+      volumes:
+      - name: konnectivity-certs
+        secret:
+          secretName: konnectivity-agent-certs

data/data/bootstrap/files/opt/openshift/konnectivity-config-override.yaml

@@ -0,0 +1,5 @@
+apiVersion: kubecontrolplane.config.openshift.io/v1
+kind: KubeAPIServerConfig
+apiServerArguments:
+  egress-selector-config-file:
+  - "/etc/kubernetes/config/egress-selector-config.yaml"

data/data/bootstrap/files/opt/openshift/konnectivity-namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openshift-bootstrap-konnectivity
+  labels:
+    openshift.io/bootstrap-only: "true"

data/data/bootstrap/files/opt/openshift/konnectivity-server-pod.yaml

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: konnectivity-server
+  namespace: kube-system
+  labels:
+    app: konnectivity-server
+spec:
+  hostNetwork: true
+  priorityClassName: system-node-critical
+  containers:
+  - name: konnectivity-server
+    image: ${KONNECTIVITY_IMAGE}
+    command:
+    - /usr/bin/proxy-server
+    args:
+    - --logtostderr=true
+    - --cluster-cert=/etc/konnectivity/server.crt
+    - --cluster-key=/etc/konnectivity/server.key
+    - --cluster-ca-cert=/etc/konnectivity/ca.crt
+    - --uds-name=/etc/kubernetes/bootstrap-configs/konnectivity-server.socket
+    - --server-port=0
+    - --agent-port=8091
+    - --health-port=2041
+    - --mode=grpc
+    - --proxy-strategies=destHost,defaultRoute
+    - --keepalive-time=30s
+    - --frontend-keepalive-time=30s
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: 2041
+      initialDelaySeconds: 10
+      periodSeconds: 10
+    resources:
+      requests:
+        cpu: 40m
+        memory: 50Mi
+    volumeMounts:
+    - name: config-dir
+      mountPath: /etc/kubernetes/bootstrap-configs
+    - name: konnectivity-certs
+      mountPath: /etc/konnectivity
+      readOnly: true
+  volumes:
+  - name: config-dir
+    hostPath:
+      path: /etc/kubernetes/bootstrap-configs
+      type: DirectoryOrCreate
+  - name: konnectivity-certs
+    hostPath:
+      path: /opt/openshift/tls/konnectivity
+      type: Directory

data/data/bootstrap/files/usr/local/bin/bootkube.sh.template

@@ -10,6 +10,8 @@ set -euoE pipefail ## -E option will cause functions to inherit trap
 . /usr/local/bin/bootstrap-cluster-gather.sh
 # shellcheck source=bootstrap-verify-api-server-urls.sh
 . /usr/local/bin/bootstrap-verify-api-server-urls.sh
+# shellcheck source=konnectivity.sh.template
+. /usr/local/bin/konnectivity.sh
 
 mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests}
 
@@ -245,6 +247,8 @@ then
     record_service_stage_success
 fi
 
+konnectivity_setup
+
 if [ ! -f kube-apiserver-bootstrap.done ]
 then
     record_service_stage_start "kube-apiserver-bootstrap"
@@ -269,9 +273,16 @@ then
 		--infra-config-file=/assets/manifests/cluster-infrastructure-02-config.yml \
 		--rendered-manifest-files=/assets/manifests \
 		--payload-version=$VERSION \
-		--operand-kubernetes-version="${KUBERNETES_VERSION}"
+		--operand-kubernetes-version="${KUBERNETES_VERSION}" \
+{{- if .KonnectivityEnabled }}
+		--config-override-files=/assets/konnectivity-config-override.yaml
+{{- end }}
 
 	cp kube-apiserver-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-apiserver-config.yaml
+{{- if .KonnectivityEnabled }}
+	# Copy egress selector config to bootstrap-configs where KAS can read it
+	cp /opt/openshift/egress-selector-config.yaml /etc/kubernetes/bootstrap-configs/egress-selector-config.yaml
+{{- end }}
 	cp kube-apiserver-bootstrap/bootstrap-manifests/* bootstrap-manifests/
 	cp kube-apiserver-bootstrap/manifests/* manifests/
 
@@ -566,6 +577,8 @@ then
     record_service_stage_success
 fi
 
+konnectivity_manifests
+
 REQUIRED_PODS="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator"
 if [ "$BOOTSTRAP_INPLACE" = true ]
 then
@@ -651,6 +664,8 @@ if [ ! -f api-int-dns-check.done ]; then
   fi
 fi
 
+konnectivity_cleanup
+
 # Workaround for https://github.com/opencontainers/runc/pull/1807
 touch /opt/openshift/.bootkube.done
 echo "bootkube.service complete"

data/data/bootstrap/files/usr/local/bin/konnectivity-certs.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Generate Konnectivity certificates with a self-signed CA (1-day validity).
+# These are needed for mTLS between the Konnectivity server and agents
+# during the bootstrap phase.
+#
+# Usage: konnectivity-certs.sh <bootstrap-node-ip>
+
+BOOTSTRAP_NODE_IP="${1:?Usage: konnectivity-certs.sh <bootstrap-node-ip>}"
+
+KONNECTIVITY_CERT_DIR=/opt/openshift/tls/konnectivity
+mkdir -p "${KONNECTIVITY_CERT_DIR}"
+
+echo "Generating Konnectivity certificates in ${KONNECTIVITY_CERT_DIR}..."
+
+# Generate self-signed Konnectivity CA
+openssl req -x509 -newkey rsa:2048 -nodes \
+    -keyout "${KONNECTIVITY_CERT_DIR}/ca.key" \
+    -out "${KONNECTIVITY_CERT_DIR}/ca.crt" \
+    -days 1 \
+    -subj "/CN=konnectivity-signer/O=openshift"
+
+# Server certificate for agent endpoint (needs bootstrap IP as SAN)
+openssl req -new -newkey rsa:2048 -nodes \
+    -keyout "${KONNECTIVITY_CERT_DIR}/server.key" \
+    -out "${KONNECTIVITY_CERT_DIR}/server.csr" \
+    -subj "/CN=konnectivity-server/O=openshift"
+
+openssl x509 -req -in "${KONNECTIVITY_CERT_DIR}/server.csr" \
+    -CA "${KONNECTIVITY_CERT_DIR}/ca.crt" \
+    -CAkey "${KONNECTIVITY_CERT_DIR}/ca.key" \
+    -CAcreateserial \
+    -out "${KONNECTIVITY_CERT_DIR}/server.crt" \
+    -days 1 \
+    -extfile <(printf "extendedKeyUsage=serverAuth\nsubjectAltName=IP:%s" "${BOOTSTRAP_NODE_IP}")
+
+# Agent client certificate (shared by all agents)
+openssl req -new -newkey rsa:2048 -nodes \
+    -keyout "${KONNECTIVITY_CERT_DIR}/agent.key" \
+    -out "${KONNECTIVITY_CERT_DIR}/agent.csr" \
+    -subj "/CN=konnectivity-agent/O=openshift"
+
+openssl x509 -req -in "${KONNECTIVITY_CERT_DIR}/agent.csr" \
+    -CA "${KONNECTIVITY_CERT_DIR}/ca.crt" \
+    -CAkey "${KONNECTIVITY_CERT_DIR}/ca.key" \
+    -CAcreateserial \
+    -out "${KONNECTIVITY_CERT_DIR}/agent.crt" \
+    -days 1 \
+    -extfile <(printf "extendedKeyUsage=clientAuth")
+
+# Clean up CSR files
+rm -f "${KONNECTIVITY_CERT_DIR}"/*.csr
+
+echo "Konnectivity certificates generated successfully."

data/data/bootstrap/files/usr/local/bin/konnectivity.sh.template

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Konnectivity bootstrap functions.
+# Sourced by bootkube.sh — do not execute directly.
+
+# konnectivity_setup detects the bootstrap node IP, generates certificates,
+# and creates the konnectivity server static pod manifest.
+konnectivity_setup() {
+{{- if .KonnectivityEnabled }}
+{{- if .BootstrapNodeIP }}
+    # Use explicitly configured bootstrap node IP
+    export BOOTSTRAP_NODE_IP="{{.BootstrapNodeIP}}"
+    echo "Using configured bootstrap node IP: ${BOOTSTRAP_NODE_IP}"
+{{- else }}
+    # Detect bootstrap node IP at runtime using the default route source address.
+    # Konnectivity agents use this to connect back to the bootstrap server.
+    {{- if .UseIPv6ForNodeIP }}
+        export BOOTSTRAP_NODE_IP=$(ip -6 -j route get 2001:4860:4860::8888 | jq -r '.[0].prefsrc')
+    {{- else }}
+        export BOOTSTRAP_NODE_IP=$(ip -j route get 1.1.1.1 | jq -r '.[0].prefsrc')
+    {{- end }}
+    echo "Detected bootstrap node IP: ${BOOTSTRAP_NODE_IP}"
+{{- end }}
+
+    if [ ! -f konnectivity-certs.done ]; then
+        record_service_stage_start "konnectivity-certs"
+        /usr/local/bin/konnectivity-certs.sh "${BOOTSTRAP_NODE_IP}"
+        touch konnectivity-certs.done
+        record_service_stage_success
+    fi
+
+    if [ ! -f konnectivity-server-bootstrap.done ]; then
+        record_service_stage_start "konnectivity-server-bootstrap"
+        echo "Creating Konnectivity server static pod manifest..."
+        export KONNECTIVITY_IMAGE=$(image_for apiserver-network-proxy)
+        envsubst < /opt/openshift/konnectivity-server-pod.yaml > /etc/kubernetes/manifests/konnectivity-server-pod.yaml
+        touch konnectivity-server-bootstrap.done
+        record_service_stage_success
+    fi
+{{- end }}
+    :
+}
+
+# konnectivity_manifests creates the agent namespace, secret, and daemonset
+# manifests for cluster deployment.
+konnectivity_manifests() {
+{{- if .KonnectivityEnabled }}
+    if [ ! -f konnectivity-agent-manifest.done ]; then
+        record_service_stage_start "konnectivity-agent-manifest"
+        echo "Creating Konnectivity agent manifests..."
+
+        export KONNECTIVITY_IMAGE=$(image_for apiserver-network-proxy)
+        KONNECTIVITY_CERT_DIR=/opt/openshift/tls/konnectivity
+
+        cp /opt/openshift/konnectivity-namespace.yaml manifests/konnectivity-namespace.yaml
+
+        export KONNECTIVITY_AGENT_CERT_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/agent.crt")
+        export KONNECTIVITY_AGENT_KEY_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/agent.key")
+        export KONNECTIVITY_CA_CERT_BASE64=$(base64 -w0 "${KONNECTIVITY_CERT_DIR}/ca.crt")
+        envsubst < /opt/openshift/konnectivity-agent-certs-secret.yaml > manifests/konnectivity-agent-certs.yaml
+
+        envsubst < /opt/openshift/konnectivity-agent-daemonset.yaml > manifests/konnectivity-agent-daemonset.yaml
+
+        touch konnectivity-agent-manifest.done
+        record_service_stage_success
+    fi
+{{- end }}
+    :
+}
+
+# konnectivity_cleanup removes bootstrap konnectivity resources by deleting
+# the namespace (cascading to DaemonSet and Secret) and the server static pod.
+konnectivity_cleanup() {
+{{- if .KonnectivityEnabled }}
+    if [ ! -f konnectivity-cleanup.done ]; then
+        record_service_stage_start "konnectivity-cleanup"
+        echo "Cleaning up bootstrap konnectivity resources..."
+        oc delete namespace openshift-bootstrap-konnectivity \
+            --kubeconfig=/opt/openshift/auth/kubeconfig \
+            --ignore-not-found=true
+        rm -f /etc/kubernetes/manifests/konnectivity-server-pod.yaml
+        touch konnectivity-cleanup.done
+        record_service_stage_success
+    fi
+{{- end }}
+    :
+}

pkg/asset/ignition/bootstrap/common.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/utils/ptr"
 
 	configv1 "github.com/openshift/api/config/v1"
+	"github.com/openshift/api/features"
 	"github.com/openshift/installer/data"
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/ignition"
@@ -99,6 +100,7 @@ type bootstrapTemplateData struct {
 	Invoker               string
 	ClusterDomain         string
 	OSImageStream         types.OSImageStream
+	KonnectivityEnabled   bool
 }
 
 // platformTemplateData is the data to use to replace values in bootstrap
@@ -397,6 +399,8 @@ func (a *Common) getTemplateData(dependencies asset.Parents, bootstrapInPlace bo
 		}
 		pullSecret = merged
 	}
+	konnectivityFeatureGateEnabled := (installConfig.Config.Enabled(features.FeatureGateCRDCompatibilityRequirementOperator) ||
+		installConfig.Config.Enabled(features.FeatureGateClusterAPIMachineManagement))
 
 	return &bootstrapTemplateData{
 		AdditionalTrustBundle: installConfig.Config.AdditionalTrustBundle,
@@ -422,6 +426,7 @@ func (a *Common) getTemplateData(dependencies asset.Parents, bootstrapInPlace bo
 		Invoker:               openshiftInstallInvoker,
 		ClusterDomain:         installConfig.Config.ClusterDomain(),
 		OSImageStream:         installConfig.Config.OSImageStream,
+		KonnectivityEnabled:   konnectivityFeatureGateEnabled && !bootstrapInPlace,
 	}
 }