Skip to content

ci: migrate Flux diagnostics to the shared diagnose-flux composite #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
devantler merged 1 commit into from
Jun 27, 2026
Merged

ci: migrate Flux diagnostics to the shared diagnose-flux composite #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 7 additions & 34 deletions .github/workflows/cd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,37 +272,10 @@ jobs:

- name: 🩺 Diagnose Flux on failure
if: failure() && steps.reconcile.conclusion == 'failure'
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
run: |
set +e
echo "::group::Flux Kustomizations"
kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Flux HelmReleases"
kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Flux OCIRepositories"
kubectl get ocirepositories.source.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Failing Kustomizations describe"
for k in infrastructure-controllers infrastructure apps; do
echo "--- $k ---"
kubectl describe kustomizations.kustomize.toolkit.fluxcd.io "$k" -n flux-system 2>&1 | tail -60
done
echo "::endgroup::"
echo "::group::Flux controller pods"
kubectl get pods -n flux-system -o wide
echo "::endgroup::"
echo "::group::kustomize-controller logs"
kubectl logs -n flux-system -l app=kustomize-controller --tail=200
echo "::endgroup::"
echo "::group::helm-controller logs"
kubectl logs -n flux-system -l app=helm-controller --tail=200
echo "::endgroup::"
echo "::group::source-controller logs"
kubectl logs -n flux-system -l app=source-controller --tail=100
echo "::endgroup::"
echo "::group::Recent events (warnings)"
kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -50
echo "::endgroup::"
# Shared composite — single source of truth for Flux failure diagnostics
# (devantler-tech/actions#367). This prod path now inherits the richer
# diagnostics (nodes, all-pods, failing/CrashLooping pod logs, failing
# Jobs) that previously only the system-test path emitted. kubectl uses
# the Talos cert-based admin@prod kubeconfig, so no HCLOUD_TOKEN is
# needed by the diagnostics themselves.
uses: devantler-tech/actions/diagnose-flux@e182f2207fb386ef827f48b3cc9b7deac873046a # v7.1.0
119 changes: 11 additions & 108 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,80 +98,10 @@ jobs:

- name: 🩺 Diagnose Flux on failure
if: failure()
run: |
set +e
echo "::group::Nodes"
kubectl get nodes -o wide
echo "::endgroup::"
echo "::group::Flux Kustomizations"
kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Flux HelmReleases"
kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Flux OCIRepositories"
kubectl get ocirepositories.source.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Failing Kustomizations describe"
for k in infrastructure-controllers infrastructure apps; do
echo "--- $k ---"
kubectl describe kustomizations.kustomize.toolkit.fluxcd.io "$k" -n flux-system 2>&1 | tail -80
done
echo "::endgroup::"
echo "::group::Flux controller pods"
kubectl get pods -n flux-system -o wide
echo "::endgroup::"
echo "::group::kustomize-controller logs"
kubectl logs -n flux-system -l app=kustomize-controller --tail=300
echo "::endgroup::"
echo "::group::helm-controller logs"
kubectl logs -n flux-system -l app=helm-controller --tail=300
echo "::endgroup::"
echo "::group::source-controller logs"
kubectl logs -n flux-system -l app=source-controller --tail=200
echo "::endgroup::"
echo "::group::All pods"
kubectl get pods -A -o wide
echo "::endgroup::"
echo "::group::Recent events (warnings)"
kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -100
echo "::endgroup::"
echo "::group::Failed/CrashLooping pod logs"
# Dump previous + current logs from any pod that's not Running OR
# has containers in a waiting/terminated-error state. CrashLoopBackOff
# pods stay in phase=Running with waiting.reason=CrashLoopBackOff,
# so we must inspect containerStatuses, not just phase.
pods_to_dump=$(kubectl get pods -A -o json 2>/dev/null | jq -r '
.items[]
| select(
.status.phase != "Running" and .status.phase != "Succeeded"
or ((.status.containerStatuses // [])[]? | (.restartCount // 0) > 0)
or ((.status.containerStatuses // [])[]? | (.state.waiting.reason // "") | test("CrashLoopBackOff|Error|ImagePullBackOff|ErrImagePull|CreateContainerError"))
or ((.status.containerStatuses // [])[]? | (.state.terminated.reason // "") == "Error")
)
| "\(.metadata.namespace)\t\(.metadata.name)"
' | sort -u)
while IFS=$'\t' read -r ns name; do
[ -z "$ns" ] && continue
echo "--- $ns/$name (previous + current) ---"
kubectl logs -n "$ns" "$name" --all-containers --previous --tail=200 2>&1 | sed 's/^/ /'
echo " --- current ---"
kubectl logs -n "$ns" "$name" --all-containers --tail=200 2>&1 | sed 's/^/ /'
done <<< "$pods_to_dump"
# Also dump logs from any Job pods (these often complete then disappear from "not Running")
while IFS=$'\t' read -r ns name; do
[ -z "$ns" ] && continue
echo "--- Job pod $ns/$name ---"
kubectl logs -n "$ns" "$name" --all-containers --tail=200 2>&1 | sed 's/^/ /'
done < <(kubectl get pods -A -l job-name -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.name}{"\n"}{end}' 2>/dev/null)
echo "::endgroup::"
echo "::group::Failing Jobs describe"
kubectl get jobs -A -o json 2>/dev/null | jq -r '.items[] | select(.status.failed > 0 or (.status.conditions // [])[]?.type == "Failed") | "\(.metadata.namespace)/\(.metadata.name)"' | while read -r job; do
ns="${job%%/*}"; name="${job##*/}"
echo "--- $ns/$name ---"
kubectl describe job -n "$ns" "$name" 2>&1 | tail -50
done
echo "::endgroup::"
# Shared composite — single source of truth for Flux failure diagnostics
# (devantler-tech/actions#367). Default kustomizations match this job's
# previous inline list (infrastructure-controllers infrastructure apps).
uses: devantler-tech/actions/diagnose-flux@e182f2207fb386ef827f48b3cc9b7deac873046a # v7.1.0

- name: 🔥 Delete cluster
if: always()
Expand Down Expand Up @@ -364,40 +294,13 @@ jobs:

- name: 🩺 Diagnose Flux on failure
if: failure() && steps.reconcile.outcome == 'failure'
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
run: |
set +e
echo "::group::Flux Kustomizations"
kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Flux HelmReleases"
kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Flux OCIRepositories"
kubectl get ocirepositories.source.toolkit.fluxcd.io -A -o wide
echo "::endgroup::"
echo "::group::Failing Kustomizations describe"
for k in infrastructure-controllers infrastructure apps; do
echo "--- $k ---"
kubectl describe kustomizations.kustomize.toolkit.fluxcd.io "$k" -n flux-system 2>&1 | tail -60
done
echo "::endgroup::"
echo "::group::Flux controller pods"
kubectl get pods -n flux-system -o wide
echo "::endgroup::"
echo "::group::kustomize-controller logs"
kubectl logs -n flux-system -l app=kustomize-controller --tail=200
echo "::endgroup::"
echo "::group::helm-controller logs"
kubectl logs -n flux-system -l app=helm-controller --tail=200
echo "::endgroup::"
echo "::group::source-controller logs"
kubectl logs -n flux-system -l app=source-controller --tail=100
echo "::endgroup::"
echo "::group::Recent events (warnings)"
kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp | tail -50
echo "::endgroup::"
# Shared composite — single source of truth for Flux failure diagnostics
# (devantler-tech/actions#367). This prod path now inherits the richer
# diagnostics (nodes, all-pods, failing/CrashLooping pod logs, failing
# Jobs) that previously only the system-test path emitted. kubectl uses
# the Talos cert-based admin@prod kubeconfig, so no HCLOUD_TOKEN is
# needed by the diagnostics themselves.
uses: devantler-tech/actions/diagnose-flux@e182f2207fb386ef827f48b3cc9b7deac873046a # v7.1.0

ci-required-checks:
name: CI - Required Checks
Expand Down