From d21210bcebe7b3244fe8dfe69bd718ac44f3a9f2 Mon Sep 17 00:00:00 2001 From: devantler Date: Sat, 27 Jun 2026 10:21:51 +0200 Subject: [PATCH] ci: migrate Flux diagnostics to the shared diagnose-flux composite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the three copy-pasted '🩺 Diagnose Flux on failure' blocks (ci.yaml system-test + deploy-prod, cd.yaml deploy-prod) with the shared devantler-tech/actions/diagnose-flux@v7.1.0 composite extracted in devantler-tech/actions#368. The two prod-deploy paths previously carried a reduced variant (no nodes/all-pods/failing-pod/Jobs dump, shorter --tail) and now inherit the richest diagnostics, so the paths can never drift again. The HCLOUD_TOKEN env on the prod diagnose steps was vestigial — kubectl uses the Talos cert-based admin@prod kubeconfig. Part of devantler-tech/actions#367. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/cd.yaml | 41 +++---------- .github/workflows/ci.yaml | 119 ++++---------------------------------- 2 files changed, 18 insertions(+), 142 deletions(-) diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 8b933bc..94fa254 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -272,37 +272,10 @@ jobs: - name: 🩺 Diagnose Flux on failure if: failure() && steps.reconcile.conclusion == 'failure' - env: - HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - run: | - set +e - echo "::group::Flux Kustomizations" - kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Flux HelmReleases" - kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Flux OCIRepositories" - kubectl get ocirepositories.source.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Failing Kustomizations describe" - for k in infrastructure-controllers infrastructure apps; do - echo "--- $k ---" - kubectl describe kustomizations.kustomize.toolkit.fluxcd.io "$k" -n flux-system 2>&1 | tail -60 - done - echo "::endgroup::" - echo "::group::Flux controller pods" - kubectl get pods -n flux-system -o wide - echo "::endgroup::" - echo "::group::kustomize-controller logs" - kubectl logs -n flux-system -l app=kustomize-controller --tail=200 - echo "::endgroup::" - echo "::group::helm-controller logs" - kubectl logs -n flux-system -l app=helm-controller --tail=200 - echo "::endgroup::" - echo "::group::source-controller logs" - kubectl logs -n flux-system -l app=source-controller --tail=100 - echo "::endgroup::" - echo "::group::Recent events (warnings)" - kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -50 - echo "::endgroup::" + # Shared composite — single source of truth for Flux failure diagnostics + # (devantler-tech/actions#367). This prod path now inherits the richer + # diagnostics (nodes, all-pods, failing/CrashLooping pod logs, failing + # Jobs) that previously only the system-test path emitted. kubectl uses + # the Talos cert-based admin@prod kubeconfig, so no HCLOUD_TOKEN is + # needed by the diagnostics themselves. + uses: devantler-tech/actions/diagnose-flux@e182f2207fb386ef827f48b3cc9b7deac873046a # v7.1.0 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9e4405e..4e3cc88 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,80 +98,10 @@ jobs: - name: 🩺 Diagnose Flux on failure if: failure() - run: | - set +e - echo "::group::Nodes" - kubectl get nodes -o wide - echo "::endgroup::" - echo "::group::Flux Kustomizations" - kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Flux HelmReleases" - kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Flux OCIRepositories" - kubectl get ocirepositories.source.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Failing Kustomizations describe" - for k in infrastructure-controllers infrastructure apps; do - echo "--- $k ---" - kubectl describe kustomizations.kustomize.toolkit.fluxcd.io "$k" -n flux-system 2>&1 | tail -80 - done - echo "::endgroup::" - echo "::group::Flux controller pods" - kubectl get pods -n flux-system -o wide - echo "::endgroup::" - echo "::group::kustomize-controller logs" - kubectl logs -n flux-system -l app=kustomize-controller --tail=300 - echo "::endgroup::" - echo "::group::helm-controller logs" - kubectl logs -n flux-system -l app=helm-controller --tail=300 - echo "::endgroup::" - echo "::group::source-controller logs" - kubectl logs -n flux-system -l app=source-controller --tail=200 - echo "::endgroup::" - echo "::group::All pods" - kubectl get pods -A -o wide - echo "::endgroup::" - echo "::group::Recent events (warnings)" - kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -100 - echo "::endgroup::" - echo "::group::Failed/CrashLooping pod logs" - # Dump previous + current logs from any pod that's not Running OR - # has containers in a waiting/terminated-error state. CrashLoopBackOff - # pods stay in phase=Running with waiting.reason=CrashLoopBackOff, - # so we must inspect containerStatuses, not just phase. - pods_to_dump=$(kubectl get pods -A -o json 2>/dev/null | jq -r ' - .items[] - | select( - .status.phase != "Running" and .status.phase != "Succeeded" - or ((.status.containerStatuses // [])[]? | (.restartCount // 0) > 0) - or ((.status.containerStatuses // [])[]? | (.state.waiting.reason // "") | test("CrashLoopBackOff|Error|ImagePullBackOff|ErrImagePull|CreateContainerError")) - or ((.status.containerStatuses // [])[]? | (.state.terminated.reason // "") == "Error") - ) - | "\(.metadata.namespace)\t\(.metadata.name)" - ' | sort -u) - while IFS=$'\t' read -r ns name; do - [ -z "$ns" ] && continue - echo "--- $ns/$name (previous + current) ---" - kubectl logs -n "$ns" "$name" --all-containers --previous --tail=200 2>&1 | sed 's/^/ /' - echo " --- current ---" - kubectl logs -n "$ns" "$name" --all-containers --tail=200 2>&1 | sed 's/^/ /' - done <<< "$pods_to_dump" - # Also dump logs from any Job pods (these often complete then disappear from "not Running") - while IFS=$'\t' read -r ns name; do - [ -z "$ns" ] && continue - echo "--- Job pod $ns/$name ---" - kubectl logs -n "$ns" "$name" --all-containers --tail=200 2>&1 | sed 's/^/ /' - done < <(kubectl get pods -A -l job-name -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.name}{"\n"}{end}' 2>/dev/null) - echo "::endgroup::" - echo "::group::Failing Jobs describe" - kubectl get jobs -A -o json 2>/dev/null | jq -r '.items[] | select(.status.failed > 0 or (.status.conditions // [])[]?.type == "Failed") | "\(.metadata.namespace)/\(.metadata.name)"' | while read -r job; do - ns="${job%%/*}"; name="${job##*/}" - echo "--- $ns/$name ---" - kubectl describe job -n "$ns" "$name" 2>&1 | tail -50 - done - echo "::endgroup::" + # Shared composite — single source of truth for Flux failure diagnostics + # (devantler-tech/actions#367). Default kustomizations match this job's + # previous inline list (infrastructure-controllers infrastructure apps). + uses: devantler-tech/actions/diagnose-flux@e182f2207fb386ef827f48b3cc9b7deac873046a # v7.1.0 - name: 🔥 Delete cluster if: always() @@ -364,40 +294,13 @@ jobs: - name: 🩺 Diagnose Flux on failure if: failure() && steps.reconcile.outcome == 'failure' - env: - HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} - run: | - set +e - echo "::group::Flux Kustomizations" - kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Flux HelmReleases" - kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Flux OCIRepositories" - kubectl get ocirepositories.source.toolkit.fluxcd.io -A -o wide - echo "::endgroup::" - echo "::group::Failing Kustomizations describe" - for k in infrastructure-controllers infrastructure apps; do - echo "--- $k ---" - kubectl describe kustomizations.kustomize.toolkit.fluxcd.io "$k" -n flux-system 2>&1 | tail -60 - done - echo "::endgroup::" - echo "::group::Flux controller pods" - kubectl get pods -n flux-system -o wide - echo "::endgroup::" - echo "::group::kustomize-controller logs" - kubectl logs -n flux-system -l app=kustomize-controller --tail=200 - echo "::endgroup::" - echo "::group::helm-controller logs" - kubectl logs -n flux-system -l app=helm-controller --tail=200 - echo "::endgroup::" - echo "::group::source-controller logs" - kubectl logs -n flux-system -l app=source-controller --tail=100 - echo "::endgroup::" - echo "::group::Recent events (warnings)" - kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -50 - echo "::endgroup::" + # Shared composite — single source of truth for Flux failure diagnostics + # (devantler-tech/actions#367). This prod path now inherits the richer + # diagnostics (nodes, all-pods, failing/CrashLooping pod logs, failing + # Jobs) that previously only the system-test path emitted. kubectl uses + # the Talos cert-based admin@prod kubeconfig, so no HCLOUD_TOKEN is + # needed by the diagnostics themselves. + uses: devantler-tech/actions/diagnose-flux@e182f2207fb386ef827f48b3cc9b7deac873046a # v7.1.0 ci-required-checks: name: CI - Required Checks