From 28a0668ee6a5efb479e1a62a708a93f383f05540 Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Fri, 15 May 2026 13:11:00 +0200 Subject: [PATCH 1/9] Improve integration test failure diagnostics --- Jenkinsfile | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28b3e1315..22959cf4e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -106,16 +106,75 @@ pipeline { profiles = [params.chooseProfile] } - def withK3dCluster = { body -> + def dumpKubernetesDebugInfo = { profile -> + def dumpDir = "target/k8s-debug/${profile}" + + sh(script: """ + set +e + mkdir -p '${dumpDir}' + export KUBECONFIG='${env.WORKSPACE}/.kubeconfig.yaml' + + { + echo '# cluster-info' + kubectl cluster-info + + echo + echo '# nodes' + kubectl get nodes -o wide + + echo + echo '# namespaces' + kubectl get namespaces -o wide + + echo + echo '# pods' + kubectl get pods -A -o wide + + echo + echo '# events' + kubectl get events -A --sort-by=.lastTimestamp + + echo + echo '# pvc / pv' + kubectl get pvc,pv -A -o wide + + echo + echo '# ingress' + kubectl get ingress -A -o wide + } > '${dumpDir}/overview.txt' 2>&1 + + kubectl describe all -A > '${dumpDir}/describe-all.txt' 2>&1 + + : > '${dumpDir}/container-logs.txt' + kubectl get pods -A --no-headers \\ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + while read -r namespace pod; do + echo "===== \${namespace}/\${pod} current =====" >> '${dumpDir}/container-logs.txt' + kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --tail=100 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1 + + echo >> '${dumpDir}/container-logs.txt' + echo "===== \${namespace}/\${pod} previous =====" >> '${dumpDir}/container-logs.txt' + kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --previous --tail=100 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1 + echo >> '${dumpDir}/container-logs.txt' + done + """, returnStatus: true) + + archiveArtifacts artifacts: "${dumpDir}/**", allowEmptyArchive: true + } + + def withK3dCluster = { profile, body -> try { sh "yes | KUBECONFIG=${env.WORKSPACE}/.kubeconfig.yaml ./scripts/init-cluster.sh --cluster-name=${env.K3D_CLUSTER_NAME}" body() + } catch(Throwable t) { + dumpKubernetesDebugInfo(profile) + throw t } finally { sh "KUBECONFIG=${env.WORKSPACE}/.kubeconfig.yaml $HOME/.local/bin/k3d cluster delete ${env.K3D_CLUSTER_NAME}" }} profiles.each { profile -> - withK3dCluster { + withK3dCluster(profile) { if (profile.startsWith('operator')) { docker.image("${env.GOLANG_IMAGE}").inside(env.INTEGRATION_TEST_DOCKER_ARGS) { From dc7c8f65dd00f06f794f93abd207c921e587de64 Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Fri, 15 May 2026 16:28:18 +0200 Subject: [PATCH 2/9] Add an intentionally failing test --- .../gitops/integration/profiles/ArgoCDProfileTestIT.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy index 893d830ad..ad784456f 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy @@ -28,6 +28,11 @@ class ArgoCDProfileTestIT extends ProfileTestSetup { println "###### Integration ArgoCD test ######" } + @Test + void forceFailureForJenkinsDebugDump() { + fail("Intentional failure to verify Jenkins Kubernetes debug dump") + } + @Test void ensureNamespaceExists() { From 7e15169655a95d5c2b94aec8603bafed8d208be3 Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Fri, 15 May 2026 16:39:40 +0200 Subject: [PATCH 3/9] modify integration test dump --- Jenkinsfile | 83 +++++++++++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 51 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 22959cf4e..3c07e5958 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -107,57 +107,38 @@ pipeline { } def dumpKubernetesDebugInfo = { profile -> - def dumpDir = "target/k8s-debug/${profile}" - - sh(script: """ - set +e - mkdir -p '${dumpDir}' - export KUBECONFIG='${env.WORKSPACE}/.kubeconfig.yaml' - - { - echo '# cluster-info' - kubectl cluster-info - - echo - echo '# nodes' - kubectl get nodes -o wide - - echo - echo '# namespaces' - kubectl get namespaces -o wide - - echo - echo '# pods' - kubectl get pods -A -o wide - - echo - echo '# events' - kubectl get events -A --sort-by=.lastTimestamp - - echo - echo '# pvc / pv' - kubectl get pvc,pv -A -o wide - - echo - echo '# ingress' - kubectl get ingress -A -o wide - } > '${dumpDir}/overview.txt' 2>&1 - - kubectl describe all -A > '${dumpDir}/describe-all.txt' 2>&1 - - : > '${dumpDir}/container-logs.txt' - kubectl get pods -A --no-headers \\ - -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | - while read -r namespace pod; do - echo "===== \${namespace}/\${pod} current =====" >> '${dumpDir}/container-logs.txt' - kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --tail=100 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1 - - echo >> '${dumpDir}/container-logs.txt' - echo "===== \${namespace}/\${pod} previous =====" >> '${dumpDir}/container-logs.txt' - kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --previous --tail=100 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1 - echo >> '${dumpDir}/container-logs.txt' - done - """, returnStatus: true) + def dumpDir = "target/k8s-debug/${profile}" + + docker.image("${env.GOLANG_IMAGE}").inside(env.INTEGRATION_TEST_DOCKER_ARGS) { + sh(script: """ + set +e + apk add --no-cache kubectl + mkdir -p '${dumpDir}' + export KUBECONFIG='${env.WORKSPACE}/.kubeconfig.yaml' + + kubectl cluster-info > '${dumpDir}/cluster-info.txt' 2>&1 + kubectl get nodes -o wide > '${dumpDir}/nodes.txt' 2>&1 + kubectl get namespaces -o wide > '${dumpDir}/namespaces.txt' 2>&1 + kubectl get pods -A -o wide > '${dumpDir}/pods.txt' 2>&1 + kubectl get events -A --sort-by=.lastTimestamp > '${dumpDir}/events.txt' 2>&1 + kubectl get pvc,pv -A -o wide > '${dumpDir}/volumes.txt' 2>&1 + kubectl get ingress -A -o wide > '${dumpDir}/ingress.txt' 2>&1 + kubectl describe all -A > '${dumpDir}/describe-all.txt' 2>&1 + + : > '${dumpDir}/container-logs.txt' + kubectl get pods -A --no-headers \\ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name | + while read -r namespace pod; do + echo "===== \${namespace}/\${pod} current =====" >> '${dumpDir}/container-logs.txt' + kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --tail=200 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1 + + echo >> '${dumpDir}/container-logs.txt' + echo "===== \${namespace}/\${pod} previous =====" >> '${dumpDir}/container-logs.txt' + kubectl logs -n "\${namespace}" "\${pod}" --all-containers=true --previous --tail=200 --prefix=true >> '${dumpDir}/container-logs.txt' 2>&1 + echo >> '${dumpDir}/container-logs.txt' + done + """, returnStatus: true) + } archiveArtifacts artifacts: "${dumpDir}/**", allowEmptyArchive: true } From dbbe6bfac344dd29777365c2f1ee88af32b337d5 Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Fri, 15 May 2026 17:01:19 +0200 Subject: [PATCH 4/9] Revert "Add an intentionally failing test" This reverts commit dc7c8f65dd00f06f794f93abd207c921e587de64. --- .../gitops/integration/profiles/ArgoCDProfileTestIT.groovy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy index ad784456f..893d830ad 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy @@ -28,11 +28,6 @@ class ArgoCDProfileTestIT extends ProfileTestSetup { println "###### Integration ArgoCD test ######" } - @Test - void forceFailureForJenkinsDebugDump() { - fail("Intentional failure to verify Jenkins Kubernetes debug dump") - } - @Test void ensureNamespaceExists() { From d6c18dc5a8b5e564800817688fc1f8be65361c39 Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Wed, 27 May 2026 16:25:52 +0200 Subject: [PATCH 5/9] remove jenkinsfile email post --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3c07e5958..2e8a94707 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -213,7 +213,7 @@ pipeline { } - post { + /*post { changed { emailext( subject: "${currentBuild.result}: ${env.JOB_NAME} #${env.BUILD_NUMBER}", @@ -225,5 +225,5 @@ pipeline { ] ) } - } + }*/ } \ No newline at end of file From 11e64ee13ad0e04d25a890c7f66b828076801b34 Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Wed, 27 May 2026 16:29:35 +0200 Subject: [PATCH 6/9] change jenkins builddiscarder to 20 --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e8a94707..e9216ae8e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,7 @@ pipeline { } options { - buildDiscarder(logRotator(numToKeepStr: '5')) + buildDiscarder(logRotator(numToKeepStr: '20')) timestamps() timeout(time: 120, unit: 'MINUTES') } From df2eb2c129337a5ba3513b39cd7b76852e30b350 Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Wed, 27 May 2026 18:33:50 +0200 Subject: [PATCH 7/9] Stabilize Kubernetes integration test readiness checks; Add reusable wait helpers for pod and namespace readiness --- .../gitops/integration/TestK8sHelper.groovy | 162 +++++++++++++++++- .../features/CertManagerTestIT.groovy | 48 ++---- .../features/MonitoringTestIT.groovy | 45 ++--- .../profiles/ArgoCDProfileTestIT.groovy | 42 +---- .../profiles/FullProfileTestIT.groovy | 94 ++-------- .../profiles/MandantProfileTestIT.groovy | 28 +-- .../profiles/PetclinicProfileTestIT.groovy | 38 +--- .../profiles/PrefixProfileTestIT.groovy | 51 +----- 8 files changed, 229 insertions(+), 279 deletions(-) diff --git a/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy b/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy index e6533101d..6103f4695 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy @@ -8,6 +8,9 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicReference import groovy.util.logging.Slf4j +import io.fabric8.kubernetes.api.model.ContainerStatus +import io.fabric8.kubernetes.api.model.Namespace +import io.fabric8.kubernetes.api.model.Pod import io.fabric8.kubernetes.client.KubernetesClient import io.fabric8.kubernetes.client.KubernetesClientBuilder import io.fabric8.kubernetes.client.KubernetesClientException @@ -20,6 +23,10 @@ import org.awaitility.Awaitility @Slf4j class TestK8sHelper { + static final int DEFAULT_WAIT_MINUTES = 5 + static final int DEFAULT_POLL_SECONDS = 5 + static final String RUNNING = "Running" + /** * This method logs Namespace and contining Pods to namespace.*/ static void dumpNamespacesAndPods() { @@ -113,23 +120,166 @@ class TestK8sHelper { } /** - * Test defined namespace and check if all pods running or specific pod. Pod is find by name which startWith... + * Checks the current Kubernetes state once and verifies that every matching pod is running. + * Use a waitFor... variant when the tested resource may still be rolling out. * @param namespace - * @param podNameStartsWith + * @param podNameStartsWith optional pod name prefix. Empty string matches all pods in the namespace. */ static boolean checkAllPodsRunningInNamespace(String namespace, String podNameStartsWith = "") { - String running = "Running" try (KubernetesClient client = new KubernetesClientBuilder().build()) { // Check Pod - def actualPods = client.pods().inNamespace(namespace).list().getItems().findAll { it.metadata.name.startsWith(podNameStartsWith) } + List actualPods = client.pods().inNamespace(namespace).list().getItems().findAll { Pod pod -> + pod.getMetadata().getName().startsWith(podNameStartsWith) + } assert !actualPods.isEmpty(): "No pods found in namespace: ${namespace} with name ${podNameStartsWith}" - def notRunningPods = actualPods.findAll { pod -> pod.getStatus().getPhase() != running } + List notRunningPods = actualPods.findAll { Pod pod -> pod.getStatus().getPhase() != RUNNING } + + assert notRunningPods.isEmpty(): "These pods in ${namespace} are not yet running: ${describePods(notRunningPods)}" + return true + } catch (KubernetesClientException ex) { + fail("Unexpected Kubernetes exception", ex) + return false + } + } + + /** + * Waits until at least one matching pod exists and all matching pods are running. + * This is the default choice for integration tests that observe resources created asynchronously. + */ + static boolean waitForAllPodsRunningInNamespace(String namespace, + String podNameStartsWith = "", + int timeout = DEFAULT_WAIT_MINUTES, + TimeUnit timeoutUnit = TimeUnit.MINUTES) { + Awaitility.await() + .atMost(timeout, timeoutUnit) + .pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS) + .untilAsserted { + checkAllPodsRunningInNamespace(namespace, podNameStartsWith) + } + return true + } + + /** + * Checks the current Kubernetes state once and verifies one running pod for each expected name prefix. + * Extra pods in the namespace are ignored, which keeps the check stable during rollouts. + */ + static boolean checkPodPrefixesRunningInNamespace(String namespace, List expectedPodPrefixes) { + try (KubernetesClient client = new KubernetesClientBuilder().build()) { + List actualPods = client.pods().inNamespace(namespace).list().getItems() + List missingPods = expectedPodPrefixes.findAll { String prefix -> + !actualPods.any { Pod pod -> pod.getMetadata().getName().startsWith(prefix) } + } + assert missingPods.isEmpty(): "Missing these pods in ${namespace}: ${missingPods}" - assert notRunningPods.isEmpty(): "These pods in ${namespace} are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" + List notRunningPodPrefixes = expectedPodPrefixes.findAll { String prefix -> + List matchingPods = actualPods.findAll { Pod pod -> pod.getMetadata().getName().startsWith(prefix) } + !matchingPods.any { Pod pod -> pod.getStatus().getPhase() == RUNNING } + } + assert notRunningPodPrefixes.isEmpty(): "No running pod found in ${namespace} for: ${notRunningPodPrefixes}. Current pods: ${describePods(actualPods)}" return true } catch (KubernetesClientException ex) { fail("Unexpected Kubernetes exception", ex) return false } } + + /** + * Waits until each expected pod name prefix has at least one running pod. + */ + static boolean waitForPodPrefixesRunningInNamespace(String namespace, + List expectedPodPrefixes, + int timeout = DEFAULT_WAIT_MINUTES, + TimeUnit timeoutUnit = TimeUnit.MINUTES) { + Awaitility.await() + .atMost(timeout, timeoutUnit) + .pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS) + .untilAsserted { + checkPodPrefixesRunningInNamespace(namespace, expectedPodPrefixes) + } + return true + } + + /** + * Checks the current Kubernetes state once using named pod matchers. + * Use this when simple prefixes are ambiguous, for example when one pod name is a prefix of another. + */ + static boolean checkPodsMatchingRunningInNamespace(String namespace, Map> expectedPods) { + try (KubernetesClient client = new KubernetesClientBuilder().build()) { + List actualPods = client.pods().inNamespace(namespace).list().getItems() + List missingPods = expectedPods.findAll { String expectedPod, Closure podNameMatches -> + !actualPods.any { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) } + }.keySet() as List + assert missingPods.isEmpty(): "Missing these pods in ${namespace}: ${missingPods}" + + List notRunningPods = expectedPods.findAll { String expectedPod, Closure podNameMatches -> + List matchingPods = actualPods.findAll { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) } + !matchingPods.any { Pod pod -> pod.getStatus().getPhase() == RUNNING } + }.keySet() as List + assert notRunningPods.isEmpty(): "No running pod found in ${namespace} for: ${notRunningPods}. Current pods: ${describePods(actualPods)}" + return true + } catch (KubernetesClientException ex) { + fail("Unexpected Kubernetes exception", ex) + return false + } + } + + /** + * Waits until every named pod matcher resolves to at least one running pod. + */ + static boolean waitForPodsMatchingRunningInNamespace(String namespace, + Map> expectedPods, + int timeout = DEFAULT_WAIT_MINUTES, + TimeUnit timeoutUnit = TimeUnit.MINUTES) { + Awaitility.await() + .atMost(timeout, timeoutUnit) + .pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS) + .untilAsserted { + checkPodsMatchingRunningInNamespace(namespace, expectedPods) + } + return true + } + + /** + * Checks the current Kubernetes state once and verifies that all expected namespaces exist. + */ + static boolean checkNamespacesExist(List expectedNamespaces) { + try (KubernetesClient client = new KubernetesClientBuilder().build()) { + List currentNamespaces = client.namespaces().list().getItems() + List missingNamespaces = expectedNamespaces.findAll { String expectedNamespace -> + !currentNamespaces.any { Namespace currentNamespace -> currentNamespace.getMetadata().getName() == expectedNamespace } + } + assert missingNamespaces.isEmpty(): "Missing these Namespaces: ${missingNamespaces}" + return true + } catch (KubernetesClientException ex) { + fail("Unexpected Kubernetes exception", ex) + return false + } + } + + /** + * Waits until all expected namespaces exist. + */ + static boolean waitForNamespaces(List expectedNamespaces, + int timeout = DEFAULT_WAIT_MINUTES, + TimeUnit timeoutUnit = TimeUnit.MINUTES) { + Awaitility.await() + .atMost(timeout, timeoutUnit) + .pollInterval(DEFAULT_POLL_SECONDS, TimeUnit.SECONDS) + .untilAsserted { + checkNamespacesExist(expectedNamespaces) + } + return true + } + + private static String describePods(Collection pods) { + return pods.collect { Pod pod -> + String podName = pod.getMetadata().getName() + String phase = pod.getStatus()?.getPhase() ?: "" + List containerStatuses = pod.getStatus()?.getContainerStatuses() + String readyContainers = containerStatuses == null + ? "0/0" + : "${containerStatuses.count { ContainerStatus status -> Boolean.TRUE == status.getReady() }}/${containerStatuses.size()}" + "${podName}:${phase}:ready=${readyContainers}" + }.join(', ') + } } \ No newline at end of file diff --git a/src/test/groovy/com/cloudogu/gitops/integration/features/CertManagerTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/features/CertManagerTestIT.groovy index 03f45e233..4de72c781 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/features/CertManagerTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/features/CertManagerTestIT.groovy @@ -1,10 +1,9 @@ package com.cloudogu.gitops.integration.features -import static org.assertj.core.api.Assertions.assertThat +import com.cloudogu.gitops.integration.TestK8sHelper import groovy.util.logging.Slf4j -import io.kubernetes.client.openapi.models.V1Pod import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty @@ -18,22 +17,14 @@ import org.junit.jupiter.api.condition.EnabledIfSystemProperty class CertManagerTestIT extends KubenetesApiTestSetup { String namespace = 'cert-manager' - int sumOfPods = 3 @Override boolean isReadyToStartTests() { - // cert-manager should has 3 running pods - def pods = api.listNamespacedPod(namespace).execute() - if (pods.items.size() != 3) { + try { + return TestK8sHelper.checkPodsMatchingRunningInNamespace(namespace, expectedCertManagerPods()) + } catch (AssertionError ignored) { return false } - for (V1Pod pod : pods.getItems()) { - println("Pod ${pod.getMetadata().name} with status ${pod.status.phase}") - if (!"Running".equals(pod.status.phase)) { - return false - } - } - return true } @BeforeAll @@ -43,29 +34,28 @@ class CertManagerTestIT extends KubenetesApiTestSetup { @Test void ensureNamespaceExists() { - def namespaces = api.listNamespace().execute() - assertThat(namespaces).isNotNull() - assertThat(namespaces.getItems().isEmpty()).isFalse() - def namespace = namespaces.getItems().find { namespace.equals(it.getMetadata().name) } - assertThat(namespace).isNotNull() - + TestK8sHelper.waitForNamespaces([namespace]) } @Test void ensureAllCertManagerPodsAreExist() { - - def pods = api.listNamespacedPod(namespace).execute() - assertThat(pods).isNotNull() - assertThat(pods.getItems().isEmpty()).isFalse() - + TestK8sHelper.waitForPodsMatchingRunningInNamespace(namespace, expectedCertManagerPods()) } @Test - void ensureNumberOfPodsAreEqualToSumOfPods() { - - def pods = api.listNamespacedPod(namespace).execute() - assertThat(pods.getItems().size()).isEqualTo(sumOfPods) - + void ensureExpectedCertManagerPodsAreRunning() { + TestK8sHelper.waitForPodsMatchingRunningInNamespace(namespace, expectedCertManagerPods()) } + private static Map> expectedCertManagerPods() { + [ + 'cert-manager' : { String podName -> + podName.startsWith('cert-manager-') && + !podName.startsWith('cert-manager-cainjector') && + !podName.startsWith('cert-manager-webhook') + }, + 'cert-manager-cainjector': { String podName -> podName.startsWith('cert-manager-cainjector') }, + 'cert-manager-webhook' : { String podName -> podName.startsWith('cert-manager-webhook') }, + ] + } } \ No newline at end of file diff --git a/src/test/groovy/com/cloudogu/gitops/integration/features/MonitoringTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/features/MonitoringTestIT.groovy index e0c0c60a4..8056cff4b 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/features/MonitoringTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/features/MonitoringTestIT.groovy @@ -2,6 +2,8 @@ package com.cloudogu.gitops.integration.features import static org.assertj.core.api.Assertions.assertThat +import com.cloudogu.gitops.integration.TestK8sHelper + import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test @@ -17,21 +19,17 @@ import org.junit.jupiter.api.condition.EnabledIfSystemProperty class MonitoringTestIT extends KubenetesApiTestSetup { String namespace = 'monitoring' - String grafanaPod = 'prometheus-stack-grafana' - String operatorPod = 'prometheus-stack-operator' - String prometheusPod = 'prometheus-stack-prometheus' + String grafanaPod = 'kube-prometheus-stack-grafana' + String operatorPod = 'kube-prometheus-stack-operator' + String prometheusPod = 'prometheus-kube-prometheus-stack-prometheus' @Override boolean isReadyToStartTests() { - - def pods = api.listNamespacedPod(namespace).execute() - if (pods && !pods.items.isEmpty()) { - def grafanaPod = pods.items.find { it.getMetadata().name.contains(grafanaPod) } - if (grafanaPod) { - return "Running".equals(grafanaPod.status.phase) - } + try { + return TestK8sHelper.checkAllPodsRunningInNamespace(namespace, grafanaPod) + } catch (AssertionError ignored) { + return false } - return false; } @BeforeAll @@ -41,36 +39,17 @@ class MonitoringTestIT extends KubenetesApiTestSetup { @Test void ensureNamespaceExists() { - def namespaces = api.listNamespace().execute() - assertThat(namespaces).isNotNull() - assertThat(namespaces.getItems().isEmpty()).isFalse() - def namespace = namespaces.getItems().find { namespace.equals(it.getMetadata().name) } - assertThat(namespace).isNotNull() - + TestK8sHelper.waitForNamespaces([namespace]) } @Test void ensureGrafanaIsStarted() { - - def pods = api.listNamespacedPod(namespace).execute() - assertThat(pods).isNotNull() - assertThat(pods.getItems().isEmpty()).isFalse() - - def grafanaPod = pods.items.find { it.getMetadata().name.contains(grafanaPod) } - assertThat(grafanaPod).isNotNull() - assertThat(grafanaPod.status.phase).isEqualTo("Running") + TestK8sHelper.waitForAllPodsRunningInNamespace(namespace, grafanaPod) } @Test void ensureOperatorIsStarted() { - - def pods = api.listNamespacedPod(namespace).execute() - assertThat(pods).isNotNull() - assertThat(pods.getItems().isEmpty()).isFalse() - - def operator = pods.items.find { it.getMetadata().name.contains(operatorPod) } - assertThat(operator).isNotNull() - assertThat(operator.status.phase).isEqualTo("Running") + TestK8sHelper.waitForAllPodsRunningInNamespace(namespace, operatorPod) } @Disabled("not start on jenkins") diff --git a/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy index 893d830ad..06ab8611c 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/profiles/ArgoCDProfileTestIT.groovy @@ -1,14 +1,7 @@ package com.cloudogu.gitops.integration.profiles -import static org.assertj.core.api.Assertions.assertThat -import static org.assertj.core.api.Assertions.fail +import com.cloudogu.gitops.integration.TestK8sHelper -import java.util.concurrent.TimeUnit - -import io.fabric8.kubernetes.client.KubernetesClient -import io.fabric8.kubernetes.client.KubernetesClientBuilder -import io.fabric8.kubernetes.client.KubernetesClientException -import org.awaitility.Awaitility import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty @@ -30,18 +23,7 @@ class ArgoCDProfileTestIT extends ProfileTestSetup { @Test void ensureNamespaceExists() { - - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - - def argocdNamespace = client.namespaces().withName(namespace).get() - - assertThat(argocdNamespace).isNotNull() - - } catch (KubernetesClientException ex) { - // Handle exception - assert fail("not expected exception was thrown. ", ex) - } - + TestK8sHelper.waitForNamespaces([namespace], 40) } /** @@ -57,24 +39,6 @@ class ArgoCDProfileTestIT extends ProfileTestSetup { List expectedPods = [expectedPod1, expectedPod2, /* expectedPod3,*/ expectedPod4, expectedPod5, expectedPod6,] - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - Awaitility.await().atMost(40, TimeUnit.MINUTES).untilAsserted { - def actualPods = client.pods().inNamespace(namespace).list().getItems() - - // 1. Verify all expected pods are present - def missingPods = expectedPods.findAll { prefix -> !actualPods.any { it.getMetadata().getName().startsWith(prefix) } - } - assert missingPods.isEmpty(): "Missing these pods in argocd: ${missingPods}" - - // 2. Verify all relevant pods are in 'Running' phase - def notRunningPods = actualPods.findAll { pod -> expectedPods.any { prefix -> pod.getMetadata().getName().startsWith(prefix) } - }.findAll { pod -> pod.getStatus().getPhase() != "Running" - } - - assert notRunningPods.isEmpty(): "These pods are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" - } - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) - } + TestK8sHelper.waitForPodPrefixesRunningInNamespace(namespace, expectedPods, 40) } } \ No newline at end of file diff --git a/src/test/groovy/com/cloudogu/gitops/integration/profiles/FullProfileTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/profiles/FullProfileTestIT.groovy index bc6532431..0f1463272 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/profiles/FullProfileTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/profiles/FullProfileTestIT.groovy @@ -1,16 +1,10 @@ package com.cloudogu.gitops.integration.profiles -import static org.assertj.core.api.Assertions.fail - import com.cloudogu.gitops.integration.TestK8sHelper import java.util.concurrent.TimeUnit import groovy.util.logging.Slf4j -import io.fabric8.kubernetes.client.KubernetesClient -import io.fabric8.kubernetes.client.KubernetesClientBuilder -import io.fabric8.kubernetes.client.KubernetesClientException -import org.awaitility.Awaitility import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfSystemProperty @@ -26,7 +20,6 @@ class FullProfileTestIT extends ProfileTestSetup { /** * Gets path to kubeconfig */ - static final String RUNNING = "Running" static final String EXAMPLE_APPS_NAMESPACE = 'example-apps-staging' @BeforeAll @@ -37,14 +30,12 @@ class FullProfileTestIT extends ProfileTestSetup { private static void waitUntilAllPodsRunning() { // if cert-manager is online, argocd is online, too! - Awaitility.await().atMost(40, TimeUnit.MINUTES).untilAsserted { - TestK8sHelper.checkAllPodsRunningInNamespace(EXAMPLE_APPS_NAMESPACE) - } + TestK8sHelper.waitForAllPodsRunningInNamespace(EXAMPLE_APPS_NAMESPACE, "", 40, TimeUnit.MINUTES) } @Test void ensureJenkinsPodIsStarted() { - TestK8sHelper.checkAllPodsRunningInNamespace('jenkins', 'jenkins') + TestK8sHelper.waitForAllPodsRunningInNamespace('jenkins', 'jenkins') } @Test @@ -58,31 +49,13 @@ class FullProfileTestIT extends ProfileTestSetup { List expectedPods = [expectedPod1, expectedPod2, /* expectedPod3,*/ expectedPod4, expectedPod5, expectedPod6,] - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - - def actualPods = client.pods().inNamespace('argocd').list().getItems() - - // 1. Verify all expected pods are present - def missingPods = expectedPods.findAll { prefix -> !actualPods.any { it.getMetadata().getName().startsWith(prefix) } - } - assert missingPods.isEmpty(): "Missing these pods in argocd: ${missingPods}" - - // 2. Verify all relevant pods are in 'Running' phase - def notRunningPods = actualPods.findAll { pod -> expectedPods.any { prefix -> pod.getMetadata().getName().startsWith(prefix) } - }.findAll { pod -> pod.getStatus().getPhase() != RUNNING - } - - assert notRunningPods.isEmpty(): "These pods are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" - - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) - } + TestK8sHelper.waitForPodPrefixesRunningInNamespace('argocd', expectedPods) } @Test void ensureScmmPodIsStarted() { - TestK8sHelper.checkAllPodsRunningInNamespace('scm-manager') + TestK8sHelper.waitForAllPodsRunningInNamespace('scm-manager') } @Test @@ -102,73 +75,42 @@ class FullProfileTestIT extends ProfileTestSetup { "monitoring", "secrets"] as List - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - - def currentNames = client.namespaces().list().getItems() - - // 1. Verify all expected pods are present - def missingNamespace = expectedNamespaces.findAll { prefix -> !currentNames.any { it.getMetadata().getName().startsWith(prefix) } - } - assert missingNamespace.isEmpty(): "Missing these Namespace: ${missingNamespace}" - - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) - } - + TestK8sHelper.waitForNamespaces(expectedNamespaces) } /** * tests searches for ingress services and ensure ingress is used as loadbalancer*/ @Test void ensureIngressIsOnline() { - TestK8sHelper.checkAllPodsRunningInNamespace('ingress', 'traefik') + TestK8sHelper.waitForAllPodsRunningInNamespace('ingress', 'traefik') } @Test void ensureCertManagerIsOnline() { - TestK8sHelper.checkAllPodsRunningInNamespace('cert-manager') + TestK8sHelper.waitForAllPodsRunningInNamespace('cert-manager') } @Test void ensureVaultIsOnline() { - TestK8sHelper.checkAllPodsRunningInNamespace('secrets', 'vault-0') + TestK8sHelper.waitForAllPodsRunningInNamespace('secrets', 'vault-0') } @Test void ensureRegistryIsOnline() { - TestK8sHelper.checkAllPodsRunningInNamespace('registry', 'docker-registry') + TestK8sHelper.waitForAllPodsRunningInNamespace('registry', 'docker-registry') } @Test void ensureExternalSecretsPodsRunning() { - - String expectedPod1 = "external-secrets-webhook" - String expectedPod2 = "external-secrets-cert-controller" - - List expectedPods = [expectedPod1, expectedPod2] - - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - - def actualPods = client.pods().inNamespace('secrets').list().getItems() - - // 1. Verify all expected pods are present - def missingPods = expectedPods.findAll { prefix -> !actualPods.any { it.getMetadata().getName().startsWith(prefix) } - } - assert missingPods.isEmpty(): "Missing these pods in secrets: ${missingPods}" - - // 2. Verify all relevant pods are in 'Running' phase - def notRunningPods = actualPods.findAll { pod -> expectedPods.any { prefix -> pod.getMetadata().getName().startsWith(prefix) } - }.findAll { pod -> pod.getStatus().getPhase() != RUNNING - } - - assert notRunningPods.isEmpty(): "These pods are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" - - // vault-0, external-secrets-webhook, external-secrets-, external-secrets-cert-controller - assert actualPods.size() == 4 - - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) - } + TestK8sHelper.waitForPodsMatchingRunningInNamespace('secrets', [ + 'external-secrets' : { String podName -> + podName.startsWith('external-secrets-') && + !podName.startsWith('external-secrets-webhook') && + !podName.startsWith('external-secrets-cert-controller') + }, + 'external-secrets-webhook' : { String podName -> podName.startsWith('external-secrets-webhook') }, + 'external-secrets-cert-controller': { String podName -> podName.startsWith('external-secrets-cert-controller') }, + ]) } } \ No newline at end of file diff --git a/src/test/groovy/com/cloudogu/gitops/integration/profiles/MandantProfileTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/profiles/MandantProfileTestIT.groovy index e8aff5b78..7101a3d7d 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/profiles/MandantProfileTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/profiles/MandantProfileTestIT.groovy @@ -41,7 +41,7 @@ class MandantProfileTestIT extends ProfileTestSetup { private static void waitUntilTenantIsReady() { // tenant is created very late after running GOP twice! - Awaitility.await().atMost(40, TimeUnit.MINUTES).untilAsserted { + Awaitility.await().atMost(40, TimeUnit.MINUTES).pollInterval(5, TimeUnit.SECONDS).untilAsserted { assert TestK8sHelper.checkAllPodsRunningInNamespace(TENANT_NAMESPACE_REGISTRY, "docker-registry") && TestK8sHelper.checkAllPodsRunningInNamespace(TENANT_NAMESPACE_SCM, 'scmm-') } } @@ -50,12 +50,12 @@ class MandantProfileTestIT extends ProfileTestSetup { // just local @Test void ensureJenkinsPodIsStartedOnTenant() { - TestK8sHelper.checkAllPodsRunningInNamespace('tenant1-jenkins', 'jenkins') + TestK8sHelper.waitForAllPodsRunningInNamespace('tenant1-jenkins', 'jenkins') } @Test void ensureRegistryPodIsStartedOnTenant() { - TestK8sHelper.checkAllPodsRunningInNamespace('tenant1-registry', 'docker-registry') + TestK8sHelper.waitForAllPodsRunningInNamespace('tenant1-registry', 'docker-registry') } @DisabledIfSystemProperty(named = "micronaut.environments", matches = "operator-mandants") @@ -63,11 +63,11 @@ class MandantProfileTestIT extends ProfileTestSetup { @Test void ensureArgocdPodsAreStartedOnTenant() { def argocdNamespace = TENANT_NAMESPACE_ARGOCD - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-application-controller') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-applicationset-controller') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-redis') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-repo-server') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-server') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-application-controller') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-applicationset-controller') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-redis') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-repo-server') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-server') } @DisabledIfSystemProperty(named = "micronaut.environments", matches = "operator-mandants") @@ -75,17 +75,17 @@ class MandantProfileTestIT extends ProfileTestSetup { @Test void ensureArgocdPodsAreStartedOnCentral() { def argocdNamespace = 'argocd' - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-application-controller') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-applicationset-controller') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-redis') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-repo-server') - TestK8sHelper.checkAllPodsRunningInNamespace(argocdNamespace, 'argocd-server') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-application-controller') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-applicationset-controller') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-redis') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-repo-server') + TestK8sHelper.waitForAllPodsRunningInNamespace(argocdNamespace, 'argocd-server') } @Test void ensureScmmPodIsStarted() { - TestK8sHelper.checkAllPodsRunningInNamespace('scm-manager') + TestK8sHelper.waitForAllPodsRunningInNamespace('scm-manager') } @DisabledIfSystemProperty(named = "micronaut.environments", matches = "operator-mandants") diff --git a/src/test/groovy/com/cloudogu/gitops/integration/profiles/PetclinicProfileTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/profiles/PetclinicProfileTestIT.groovy index 1c31d989b..2bce01154 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/profiles/PetclinicProfileTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/profiles/PetclinicProfileTestIT.groovy @@ -11,7 +11,6 @@ import groovy.util.logging.Slf4j import io.fabric8.kubernetes.client.KubernetesClient import io.fabric8.kubernetes.client.KubernetesClientBuilder import io.fabric8.kubernetes.client.KubernetesClientException -import org.awaitility.Awaitility import org.awaitility.core.ConditionTimeoutException import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -33,49 +32,16 @@ class PetclinicProfileTestIT extends ProfileTestSetup { println "###### Testing Petclinic ######" // petclinic need most of time to run. If online, we can start all tests. try { - Awaitility.await() - .atMost(40, TimeUnit.MINUTES) - .pollInterval(5, TimeUnit.SECONDS) - .untilAsserted { - waitUntilPetclinicIsRunning() - } + TestK8sHelper.waitForAllPodsRunningInNamespace(exampleStagingNs, "", 40, TimeUnit.MINUTES) } catch (ConditionTimeoutException timeoutEx) { TestK8sHelper.dumpNamespacesAndPods() fail('Cluster not ready, sth false.', timeoutEx) } } - // Start condition - private static void waitUntilPetclinicIsRunning() { - // Check Pod - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - def actualPods = client.pods().inNamespace(exampleStagingNs).list().getItems() - assert !actualPods.isEmpty(): "No pods found in petclinc - namespace: ${exampleStagingNs}" - def notRunningPods = actualPods.findAll { pod -> pod.getStatus().getPhase() != "Running" - } - assert !actualPods.isEmpty() && notRunningPods.isEmpty(): "These pods in ${exampleStagingNs} are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) - } - } - @Test void ensurePetclinicIsRunningOnStages() { - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - - // Check Pod - def actualPods = client.pods().inNamespace(exampleStagingNs).list().getItems() - - assert !actualPods.isEmpty(): "No pods found in petclinc - namespace: ${exampleStagingNs}" - - def notRunningPods = actualPods.findAll { pod -> pod.getStatus().getPhase() != "Running" - } - - assert notRunningPods.isEmpty(): "These pods in ${exampleStagingNs} are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" - - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) - } + TestK8sHelper.waitForAllPodsRunningInNamespace(exampleStagingNs) } @DisabledIfSystemProperty(named = "micronaut.environments", matches = "full|operator-full|content-examples") diff --git a/src/test/groovy/com/cloudogu/gitops/integration/profiles/PrefixProfileTestIT.groovy b/src/test/groovy/com/cloudogu/gitops/integration/profiles/PrefixProfileTestIT.groovy index f4fdbd646..801d68b3c 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/profiles/PrefixProfileTestIT.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/profiles/PrefixProfileTestIT.groovy @@ -7,10 +7,6 @@ import com.cloudogu.gitops.integration.TestK8sHelper import java.util.concurrent.TimeUnit import groovy.util.logging.Slf4j -import io.fabric8.kubernetes.client.KubernetesClient -import io.fabric8.kubernetes.client.KubernetesClientBuilder -import io.fabric8.kubernetes.client.KubernetesClientException -import org.awaitility.Awaitility import org.awaitility.core.ConditionTimeoutException import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -40,29 +36,10 @@ class PrefixProfileTestIT extends ProfileTestSetup { log.info "###### Integration test for Prefix ######" try { - Awaitility.await() - .atMost(40, TimeUnit.MINUTES) - .pollInterval(5, TimeUnit.SECONDS) - .untilAsserted { - waitUntilPetclinicIsRunning() - } + TestK8sHelper.waitForAllPodsRunningInNamespace(exampleStagingNs, "", 40, TimeUnit.MINUTES) } catch (ConditionTimeoutException timeoutEx) { TestK8sHelper.dumpNamespacesAndPods() - fail('Cluster not ready, sth false.') - } - } - - // Start condition - private static void waitUntilPetclinicIsRunning() { - // Check Pod - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - def actualPods = client.pods().inNamespace(exampleStagingNs).list().getItems() - assert !actualPods.isEmpty(): "No pods found in petclinc - namespace: ${exampleStagingNs}" - def notRunningPods = actualPods.findAll { pod -> pod.getStatus().getPhase() != "Running" - } - assert !actualPods.isEmpty() && notRunningPods.isEmpty(): "These pods in ${exampleStagingNs} are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) + fail('Cluster not ready, sth false.', timeoutEx) } } @@ -79,17 +56,7 @@ class PrefixProfileTestIT extends ProfileTestSetup { exampleProductionNs, exampleStagingNs] - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - def currentNames = client.namespaces().list().getItems() - - // 1. Verify all expected pods are present - def missingNamespace = expectedNamespaces.findAll { prefix -> !currentNames.any { it.getMetadata().getName().startsWith(prefix) } - } - assert missingNamespace.isEmpty(): "Missing these Namespace: ${missingNamespace}" - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) - } - + TestK8sHelper.waitForNamespaces(expectedNamespaces) } @Test @@ -99,16 +66,8 @@ class PrefixProfileTestIT extends ProfileTestSetup { registryNs, certManagerNs, monitoringNs] - try (KubernetesClient client = new KubernetesClientBuilder().build()) { - namespacesToCheck.each { ns -> - def actualPods = client.pods().inNamespace(ns).list().getItems() - assert !actualPods.isEmpty(): "No pods found in namespace: ${ns}" - def notRunningPods = actualPods.findAll { pod -> pod.getStatus().getPhase() != "Running" - } - assert notRunningPods.isEmpty(): "These pods in ${ns} are not yet running: ${notRunningPods.collect { it.getMetadata().getName() + ':' + it.getStatus().getPhase() }}" - } - } catch (KubernetesClientException ex) { - fail("Unexpected Kubernetes exception", ex) + namespacesToCheck.each { String ns -> + TestK8sHelper.waitForAllPodsRunningInNamespace(ns) } } } \ No newline at end of file From 843132c365be3767dbe878f1dccd0a974966c43d Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Thu, 28 May 2026 11:45:27 +0200 Subject: [PATCH 8/9] Revert "remove jenkinsfile email post" This reverts commit d6c18dc5a8b5e564800817688fc1f8be65361c39. --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e9216ae8e..f5283d70f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -213,7 +213,7 @@ pipeline { } - /*post { + post { changed { emailext( subject: "${currentBuild.result}: ${env.JOB_NAME} #${env.BUILD_NUMBER}", @@ -225,5 +225,5 @@ pipeline { ] ) } - }*/ + } } \ No newline at end of file From 44008b46b1b71f563ba5c1879a1c6ec8f4c4722b Mon Sep 17 00:00:00 2001 From: Felix Wende Date: Thu, 28 May 2026 14:07:07 +0200 Subject: [PATCH 9/9] add fast fail mechanism for integration tests in case of failing pod and add pod logs in case of failure --- .../gitops/integration/TestK8sHelper.groovy | 112 +++++++++++++++++- 1 file changed, 107 insertions(+), 5 deletions(-) diff --git a/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy b/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy index 6103f4695..edc708d40 100644 --- a/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy +++ b/src/test/groovy/com/cloudogu/gitops/integration/TestK8sHelper.groovy @@ -26,6 +26,18 @@ class TestK8sHelper { static final int DEFAULT_WAIT_MINUTES = 5 static final int DEFAULT_POLL_SECONDS = 5 static final String RUNNING = "Running" + static final String FAILED = "Failed" + static final String SUCCEEDED = "Succeeded" + static final Set FATAL_CONTAINER_WAITING_REASONS = [ + "CrashLoopBackOff", + "CreateContainerConfigError", + "CreateContainerError", + "ErrImagePull", + "ImageInspectError", + "ImagePullBackOff", + "InvalidImageName", + "RunContainerError" + ] as Set /** * This method logs Namespace and contining Pods to namespace.*/ @@ -132,7 +144,8 @@ class TestK8sHelper { pod.getMetadata().getName().startsWith(podNameStartsWith) } assert !actualPods.isEmpty(): "No pods found in namespace: ${namespace} with name ${podNameStartsWith}" - List notRunningPods = actualPods.findAll { Pod pod -> pod.getStatus().getPhase() != RUNNING } + failOnFatalPods(namespace, actualPods) + List notRunningPods = actualPods.findAll { Pod pod -> !isPodRunning(pod) } assert notRunningPods.isEmpty(): "These pods in ${namespace} are not yet running: ${describePods(notRunningPods)}" return true @@ -144,7 +157,6 @@ class TestK8sHelper { /** * Waits until at least one matching pod exists and all matching pods are running. - * This is the default choice for integration tests that observe resources created asynchronously. */ static boolean waitForAllPodsRunningInNamespace(String namespace, String podNameStartsWith = "", @@ -166,6 +178,11 @@ class TestK8sHelper { static boolean checkPodPrefixesRunningInNamespace(String namespace, List expectedPodPrefixes) { try (KubernetesClient client = new KubernetesClientBuilder().build()) { List actualPods = client.pods().inNamespace(namespace).list().getItems() + expectedPodPrefixes.each { String prefix -> + List matchingPods = actualPods.findAll { Pod pod -> pod.getMetadata().getName().startsWith(prefix) } + failIfOnlyFatalPodsMatch(namespace, prefix, matchingPods) + } + List missingPods = expectedPodPrefixes.findAll { String prefix -> !actualPods.any { Pod pod -> pod.getMetadata().getName().startsWith(prefix) } } @@ -173,7 +190,7 @@ class TestK8sHelper { List notRunningPodPrefixes = expectedPodPrefixes.findAll { String prefix -> List matchingPods = actualPods.findAll { Pod pod -> pod.getMetadata().getName().startsWith(prefix) } - !matchingPods.any { Pod pod -> pod.getStatus().getPhase() == RUNNING } + !matchingPods.any { Pod pod -> isPodRunning(pod) } } assert notRunningPodPrefixes.isEmpty(): "No running pod found in ${namespace} for: ${notRunningPodPrefixes}. Current pods: ${describePods(actualPods)}" return true @@ -206,6 +223,11 @@ class TestK8sHelper { static boolean checkPodsMatchingRunningInNamespace(String namespace, Map> expectedPods) { try (KubernetesClient client = new KubernetesClientBuilder().build()) { List actualPods = client.pods().inNamespace(namespace).list().getItems() + expectedPods.each { String expectedPod, Closure podNameMatches -> + List matchingPods = actualPods.findAll { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) } + failIfOnlyFatalPodsMatch(namespace, expectedPod, matchingPods) + } + List missingPods = expectedPods.findAll { String expectedPod, Closure podNameMatches -> !actualPods.any { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) } }.keySet() as List @@ -213,7 +235,7 @@ class TestK8sHelper { List notRunningPods = expectedPods.findAll { String expectedPod, Closure podNameMatches -> List matchingPods = actualPods.findAll { Pod pod -> podNameMatches.call(pod.getMetadata().getName()) } - !matchingPods.any { Pod pod -> pod.getStatus().getPhase() == RUNNING } + !matchingPods.any { Pod pod -> isPodRunning(pod) } }.keySet() as List assert notRunningPods.isEmpty(): "No running pod found in ${namespace} for: ${notRunningPods}. Current pods: ${describePods(actualPods)}" return true @@ -271,6 +293,50 @@ class TestK8sHelper { return true } + private static void failOnFatalPods(String namespace, Collection pods) { + Collection fatalPods = pods.findAll { Pod pod -> isPodFatal(pod) } + if (!fatalPods.isEmpty()) { + throw new IllegalStateException("Pods in ${namespace} reached a terminal or unrecoverable state: ${describePods(fatalPods)}") + } + } + + private static void failIfOnlyFatalPodsMatch(String namespace, String expectedPod, Collection matchingPods) { + if (matchingPods.isEmpty() || matchingPods.any { Pod pod -> isPodRunning(pod) }) { + return + } + + if (matchingPods.every { Pod pod -> isPodFatal(pod) }) { + throw new IllegalStateException( + "No recoverable pod found in ${namespace} for ${expectedPod}. Matching pods: ${describePods(matchingPods)}" + ) + } + } + + private static boolean isPodRunning(Pod pod) { + return pod.getStatus()?.getPhase() == RUNNING && !hasFatalContainerState(pod) + } + + private static boolean isPodFatal(Pod pod) { + String phase = pod.getStatus()?.getPhase() + return phase == FAILED || phase == SUCCEEDED || hasFatalContainerState(pod) + } + + private static boolean hasFatalContainerState(Pod pod) { + return containerStatusesFor(pod).any { ContainerStatus status -> + def waiting = status.getState()?.getWaiting() + def terminated = status.getState()?.getTerminated() + (waiting != null && FATAL_CONTAINER_WAITING_REASONS.contains(waiting.getReason())) + || (terminated != null && terminated.getExitCode() != null && terminated.getExitCode() != 0) + } + } + + private static List containerStatusesFor(Pod pod) { + List statuses = [] + statuses.addAll(pod.getStatus()?.getInitContainerStatuses() ?: []) + statuses.addAll(pod.getStatus()?.getContainerStatuses() ?: []) + return statuses + } + private static String describePods(Collection pods) { return pods.collect { Pod pod -> String podName = pod.getMetadata().getName() @@ -279,7 +345,43 @@ class TestK8sHelper { String readyContainers = containerStatuses == null ? "0/0" : "${containerStatuses.count { ContainerStatus status -> Boolean.TRUE == status.getReady() }}/${containerStatuses.size()}" - "${podName}:${phase}:ready=${readyContainers}" + String details = podProblemDetails(pod) + "${podName}:${phase}:ready=${readyContainers}${details ? ":${details}" : ""}" }.join(', ') } + + private static String podProblemDetails(Pod pod) { + List details = [] + if (pod.getStatus()?.getReason()) { + details << "reason=" + pod.getStatus().getReason() + } + if (pod.getStatus()?.getMessage()) { + details << "message=" + shorten(pod.getStatus().getMessage()) + } + containerStatusesFor(pod).each { ContainerStatus status -> + String containerState = describeContainerState(status) + if (containerState) { + details << containerState + } + } + return details.isEmpty() ? "" : "details=[${details.join('; ')}]" + } + + private static String describeContainerState(ContainerStatus status) { + def waiting = status.getState()?.getWaiting() + if (waiting != null) { + return "container=${status.getName()} waiting=${waiting.getReason() ?: ''}${waiting.getMessage() ? " message=${shorten(waiting.getMessage())}" : ''}" + } + + def terminated = status.getState()?.getTerminated() + if (terminated != null) { + return "container=${status.getName()} terminated=${terminated.getReason() ?: ''} exit=${terminated.getExitCode()}" + } + + return null + } + + private static String shorten(String value) { + return value.length() <= 160 ? value : "${value.take(157)}..." + } } \ No newline at end of file