Feature/integrationtests logs

Jenkinsfile

@@ -8,7 +8,7 @@ pipeline {
     }
 
     options {
-        buildDiscarder(logRotator(numToKeepStr: '5'))
+        buildDiscarder(logRotator(numToKeepStr: '20'))
         timestamps()
         timeout(time: 120, unit: 'MINUTES')
     }
@@ -106,16 +106,56 @@ pipeline {
                                 profiles = [params.chooseProfile]
                             }
 
-                            def withK3dCluster = { body ->
+                            def dumpKubernetesDebugInfo = { profile ->
+                                def dumpDir = "target/k8s-debug/${profile}"
+
+                                docker.image("${env.GOLANG_IMAGE}").inside(env.INTEGRATION_TEST_DOCKER_ARGS) {
+                                    sh(script: """
+                                        set +e
+                                        apk add --no-cache kubectl
+                                        mkdir -p '${dumpDir}'
+                                        export KUBECONFIG='${env.WORKSPACE}/.kubeconfig.yaml'
+
+                                        kubectl cluster-info > '${dumpDir}/cluster-info.txt' 2>&1
+                                        kubectl get nodes -o wide > '${dumpDir}/nodes.txt' 2>&1
+                                        kubectl get namespaces -o wide > '${dumpDir}/namespaces.txt' 2>&1
+                                        kubectl get pods -A -o wide > '${dumpDir}/pods.txt' 2>&1
+                                        kubectl get events -A --sort-by=.lastTimestamp > '${dumpDir}/events.txt' 2>&1
+                                        kubectl get pvc,pv -A -o wide > '${dumpDir}/volumes.txt' 2>&1
+                                        kubectl get ingress -A -o wide > '${dumpDir}/ingress.txt' 2>&1
+                                        kubectl describe all -A > '${dumpDir}/describe-all.txt' 2>&1
+
+                                        : > '${dumpDir}/container-logs.txt'
+                                        kubectl get pods -A --no-headers \\
+                                          -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name |
+                                        while read -r namespace pod; do
+                                          echo "===== \${namespace}/\${pod} current =====" >> '${dumpDir}/container-logs.txt'
+                                          kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --tail=200 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1
+
+                                          echo >> '${dumpDir}/container-logs.txt'
+                                          echo "===== \${namespace}/\${pod} previous =====" >> '${dumpDir}/container-logs.txt'
+                                          kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --previous --tail=200 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1
+                                          echo >> '${dumpDir}/container-logs.txt'
+                                        done
+                                    """, returnStatus: true)
+                                }
+
+                                  archiveArtifacts artifacts: "${dumpDir}/**", allowEmptyArchive: true
+                              }
+
+                            def withK3dCluster = { profile, body ->
                                 try {
                                     sh "yes | KUBECONFIG=${env.WORKSPACE}/.kubeconfig.yaml ./scripts/init-cluster.sh --cluster-name=${env.K3D_CLUSTER_NAME}"
                                     body()
+                                } catch(Throwable t) {
+                                    dumpKubernetesDebugInfo(profile)
+                                    throw t
                                 } finally {
                                     sh "KUBECONFIG=${env.WORKSPACE}/.kubeconfig.yaml $HOME/.local/bin/k3d cluster delete ${env.K3D_CLUSTER_NAME}"
                                 }}
 
                             profiles.each { profile ->
-                                withK3dCluster {
+                                withK3dCluster(profile) {
 
                                     if (profile.startsWith('operator')) {
                                         docker.image("${env.GOLANG_IMAGE}").inside(env.INTEGRATION_TEST_DOCKER_ARGS) {

src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy

@@ -8,6 +8,9 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicReference
 import groovy.util.logging.Slf4j
 
+import io.fabric8.kubernetes.api.model.ContainerStatus
+import io.fabric8.kubernetes.api.model.Namespace
+import io.fabric8.kubernetes.api.model.Pod
 import io.fabric8.kubernetes.client.KubernetesClient
 import io.fabric8.kubernetes.client.KubernetesClientBuilder
 import io.fabric8.kubernetes.client.KubernetesClientException
@@ -20,6 +23,22 @@ import org.awaitility.Awaitility
 @Slf4j
 class TestK8sHelper {
 
+	static final int DEFAULT_WAIT_MINUTES = 5
+	static final int DEFAULT_POLL_SECONDS = 5
+	static final String RUNNING = "Running"
+	static final String FAILED = "Failed"
+	static final String SUCCEEDED = "Succeeded"
+	static final Set<String> FATAL_CONTAINER_WAITING_REASONS = [
+		"CrashLoopBackOff",
+		"CreateContainerConfigError",
+		"CreateContainerError",
+		"ErrImagePull",
+		"ImageInspectError",
+		"ImagePullBackOff",
+		"InvalidImageName",
+		"RunContainerError"
+	] as Set
+
 	/**
 	 * This method logs Namespace and contining Pods to namespace.*/
 	static void dumpNamespacesAndPods() {
@@ -113,23 +132,256 @@ class TestK8sHelper {
 	}
 
 	/**
-	 * Test defined namespace and check if all pods running or specific pod. Pod is find by name which startWith...
+	 * Checks the current Kubernetes state once and verifies that every matching pod is running.
+	 * Use a waitFor... variant when the tested resource may still be rolling out.
 	 * @param namespace
-	 * @param podNameStartsWith
+	 * @param podNameStartsWith optional pod name prefix. Empty string matches all pods in the namespace.
 	 */
 	static boolean checkAllPodsRunningInNamespace(String namespace, String podNameStartsWith = "") {
-		String running = "Running"
 		try (KubernetesClient client = new KubernetesClientBuilder().build()) {
 			// Check Pod
-			def actualPods = client.pods().inNamespace(namespace).list().getItems().findAll { it.metadata.name.startsWith(podNameStartsWith) }
+			List<Pod> actualPods = client.pods().inNamespace(namespace).list().getItems().findAll { Pod pod ->
+				pod.getMetadata().getName().startsWith(podNameStartsWith)
+			}
 			assert !actualPods.isEmpty(): "No pods found in namespace: ${namespace} with name ${podNameStartsWith}"
-			def notRunningPods = actualPods.findAll { pod -> pod.getStatus().getPhase() != running }
+			failOnFatalPods(namespace, actualPods)
+			List<Pod> notRunningPods = actualPods.findAll { Pod pod -> !isPodRunning(pod) }
+
+			assert notRunningPods.isEmpty(): "These pods in ${namespace} are not yet running: ${describePods(notRunningPods)}"
+			return true
+		} catch (KubernetesClientException ex) {
+			fail("Unexpected Kubernetes exception", ex)
+			return false
+		}
+	}
+
+	/**
+	 * Waits until at least one matching pod exists and all matching pods are running.
+	 */
+	static boolean waitForAllPodsRunningInNamespace(String namespace,
+		String podNameStartsWith = "",
+		int timeout = DEFAULT_WAIT_MINUTES,
+		TimeUnit timeoutUnit = TimeUnit.MINUTES) {
+		Awaitility.await()
+			.atMost(timeout, timeoutUnit)
+			.pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS)
+			.untilAsserted {
+				checkAllPodsRunningInNamespace(namespace, podNameStartsWith)
+			}
+		return true
+	}
+
+	/**
+	 * Checks the current Kubernetes state once and verifies one running pod for each expected name prefix.
+	 * Extra pods in the namespace are ignored, which keeps the check stable during rollouts.
+	 */
+	static boolean checkPodPrefixesRunningInNamespace(String namespace, List<String> expectedPodPrefixes) {
+		try (KubernetesClient client = new KubernetesClientBuilder().build()) {
+			List<Pod> actualPods = client.pods().inNamespace(namespace).list().getItems()
+			expectedPodPrefixes.each { String prefix ->
+				List<Pod> matchingPods = actualPods.findAll { Pod pod -> pod.getMetadata().getName().startsWith(prefix) }
+				failIfOnlyFatalPodsMatch(namespace, prefix, matchingPods)
+			}
+
+			List<String> missingPods = expectedPodPrefixes.findAll { String prefix ->
+				!actualPods.any { Pod pod -> pod.getMetadata().getName().startsWith(prefix) }
+			}
+			assert missingPods.isEmpty(): "Missing these pods in ${namespace}: ${missingPods}"
+
+			List<String> notRunningPodPrefixes = expectedPodPrefixes.findAll { String prefix ->
+				List<Pod> matchingPods = actualPods.findAll { Pod pod -> pod.getMetadata().getName().startsWith(prefix) }
+				!matchingPods.any { Pod pod -> isPodRunning(pod) }
+			}
+			assert notRunningPodPrefixes.isEmpty(): "No running pod found in ${namespace} for: ${notRunningPodPrefixes}. Current pods: ${describePods(actualPods)}"
+			return true
+		} catch (KubernetesClientException ex) {
+			fail("Unexpected Kubernetes exception", ex)
+			return false
+		}
+	}
+
+	/**
+	 * Waits until each expected pod name prefix has at least one running pod.
+	 */
+	static boolean waitForPodPrefixesRunningInNamespace(String namespace,
+		List<String> expectedPodPrefixes,
+		int timeout = DEFAULT_WAIT_MINUTES,
+		TimeUnit timeoutUnit = TimeUnit.MINUTES) {
+		Awaitility.await()
+			.atMost(timeout, timeoutUnit)
+			.pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS)
+			.untilAsserted {
+				checkPodPrefixesRunningInNamespace(namespace, expectedPodPrefixes)
+			}
+		return true
+	}
+
+	/**
+	 * Checks the current Kubernetes state once using named pod matchers.
+	 * Use this when simple prefixes are ambiguous, for example when one pod name is a prefix of another.
+	 */
+	static boolean checkPodsMatchingRunningInNamespace(String namespace, Map<String, Closure<Boolean>> expectedPods) {
+		try (KubernetesClient client = new KubernetesClientBuilder().build()) {
+			List<Pod> actualPods = client.pods().inNamespace(namespace).list().getItems()
+			expectedPods.each { String expectedPod, Closure<Boolean> podNameMatches ->
+				List<Pod> matchingPods = actualPods.findAll { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) }
+				failIfOnlyFatalPodsMatch(namespace, expectedPod, matchingPods)
+			}
+
+			List<String> missingPods = expectedPods.findAll { String expectedPod, Closure<Boolean> podNameMatches ->
+				!actualPods.any { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) }
+			}.keySet() as List<String>
+			assert missingPods.isEmpty(): "Missing these pods in ${namespace}: ${missingPods}"
+
+			List<String> notRunningPods = expectedPods.findAll { String expectedPod, Closure<Boolean> podNameMatches ->
+				List<Pod> matchingPods = actualPods.findAll { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) }
+				!matchingPods.any { Pod pod -> isPodRunning(pod) }
+			}.keySet() as List<String>
+			assert notRunningPods.isEmpty(): "No running pod found in ${namespace} for: ${notRunningPods}. Current pods: ${describePods(actualPods)}"
+			return true
+		} catch (KubernetesClientException ex) {
+			fail("Unexpected Kubernetes exception", ex)
+			return false
+		}
+	}
 
-			assert notRunningPods.isEmpty(): "These pods in ${namespace} are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}"
+	/**
+	 * Waits until every named pod matcher resolves to at least one running pod.
+	 */
+	static boolean waitForPodsMatchingRunningInNamespace(String namespace,
+		Map<String, Closure<Boolean>> expectedPods,
+		int timeout = DEFAULT_WAIT_MINUTES,
+		TimeUnit timeoutUnit = TimeUnit.MINUTES) {
+		Awaitility.await()
+			.atMost(timeout, timeoutUnit)
+			.pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS)
+			.untilAsserted {
+				checkPodsMatchingRunningInNamespace(namespace, expectedPods)
+			}
+		return true
+	}
+
+	/**
+	 * Checks the current Kubernetes state once and verifies that all expected namespaces exist.
+	 */
+	static boolean checkNamespacesExist(List<String> expectedNamespaces) {
+		try (KubernetesClient client = new KubernetesClientBuilder().build()) {
+			List<Namespace> currentNamespaces = client.namespaces().list().getItems()
+			List<String> missingNamespaces = expectedNamespaces.findAll { String expectedNamespace ->
+				!currentNamespaces.any { Namespace currentNamespace -> currentNamespace.getMetadata().getName() == expectedNamespace }
+			}
+			assert missingNamespaces.isEmpty(): "Missing these Namespaces: ${missingNamespaces}"
 			return true
 		} catch (KubernetesClientException ex) {
 			fail("Unexpected Kubernetes exception", ex)
 			return false
 		}
 	}
+
+	/**
+	 * Waits until all expected namespaces exist.
+	 */
+	static boolean waitForNamespaces(List<String> expectedNamespaces,
+		int timeout = DEFAULT_WAIT_MINUTES,
+		TimeUnit timeoutUnit = TimeUnit.MINUTES) {
+		Awaitility.await()
+			.atMost(timeout, timeoutUnit)
+			.pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS)
+			.untilAsserted {
+				checkNamespacesExist(expectedNamespaces)
+			}
+		return true
+	}
+
+	private static void failOnFatalPods(String namespace, Collection<Pod> pods) {
+		Collection<Pod> fatalPods = pods.findAll { Pod pod -> isPodFatal(pod) }
+		if (!fatalPods.isEmpty()) {
+			throw new IllegalStateException("Pods in ${namespace} reached a terminal or unrecoverable state: ${describePods(fatalPods)}")
+		}
+	}
+
+	private static void failIfOnlyFatalPodsMatch(String namespace, String expectedPod, Collection<Pod> matchingPods) {
+		if (matchingPods.isEmpty() || matchingPods.any { Pod pod -> isPodRunning(pod) }) {
+			return
+		}
+
+		if (matchingPods.every { Pod pod -> isPodFatal(pod) }) {
+			throw new IllegalStateException(
+				"No recoverable pod found in ${namespace} for ${expectedPod}. Matching pods: ${describePods(matchingPods)}"
+			)
+		}
+	}
+
+	private static boolean isPodRunning(Pod pod) {
+		return pod.getStatus()?.getPhase() == RUNNING && !hasFatalContainerState(pod)
+	}
+
+	private static boolean isPodFatal(Pod pod) {
+		String phase = pod.getStatus()?.getPhase()
+		return phase == FAILED || phase == SUCCEEDED || hasFatalContainerState(pod)
+	}
+
+	private static boolean hasFatalContainerState(Pod pod) {
+		return containerStatusesFor(pod).any { ContainerStatus status ->
+			def waiting = status.getState()?.getWaiting()
+			def terminated = status.getState()?.getTerminated()
+			(waiting != null && FATAL_CONTAINER_WAITING_REASONS.contains(waiting.getReason()))
+				|| (terminated != null && terminated.getExitCode() != null && terminated.getExitCode() != 0)
+		}
+	}
+
+	private static List<ContainerStatus> containerStatusesFor(Pod pod) {
+		List<ContainerStatus> statuses = []
+		statuses.addAll(pod.getStatus()?.getInitContainerStatuses() ?: [])
+		statuses.addAll(pod.getStatus()?.getContainerStatuses() ?: [])
+		return statuses
+	}
+
+	private static String describePods(Collection<Pod> pods) {
+		return pods.collect { Pod pod ->
+			String podName = pod.getMetadata().getName()
+			String phase = pod.getStatus()?.getPhase() ?: "<unknown>"
+			List<ContainerStatus> containerStatuses = pod.getStatus()?.getContainerStatuses()
+			String readyContainers = containerStatuses == null
+				? "0/0"
+				: "${containerStatuses.count { ContainerStatus status -> Boolean.TRUE == status.getReady() }}/${containerStatuses.size()}"
+			String details = podProblemDetails(pod)
+			"${podName}:${phase}:ready=${readyContainers}${details ? ":${details}" : ""}"
+		}.join(', ')
+	}
+
+	private static String podProblemDetails(Pod pod) {
+		List<String> details = []
+		if (pod.getStatus()?.getReason()) {
+			details << "reason=" + pod.getStatus().getReason()
+		}
+		if (pod.getStatus()?.getMessage()) {
+			details << "message=" + shorten(pod.getStatus().getMessage())
+		}
+		containerStatusesFor(pod).each { ContainerStatus status ->
+			String containerState = describeContainerState(status)
+			if (containerState) {
+				details << containerState
+			}
+		}
+		return details.isEmpty() ? "" : "details=[${details.join('; ')}]"
+	}
+
+	private static String describeContainerState(ContainerStatus status) {
+		def waiting = status.getState()?.getWaiting()
+		if (waiting != null) {
+			return "container=${status.getName()} waiting=${waiting.getReason() ?: '<unknown>'}${waiting.getMessage() ? " message=${shorten(waiting.getMessage())}" : ''}"
+		}
+
+		def terminated = status.getState()?.getTerminated()
+		if (terminated != null) {
+			return "container=${status.getName()} terminated=${terminated.getReason() ?: '<unknown>'} exit=${terminated.getExitCode()}"
+		}
+
+		return null
+	}
+
+	private static String shorten(String value) {
+		return value.length() <= 160 ? value : "${value.take(157)}..."
+	}
 }