From 6e71c7896c593515a1971c6dacb29e5173a0957a Mon Sep 17 00:00:00 2001 From: Sam Day Date: Thu, 7 May 2026 13:30:19 +1000 Subject: [PATCH 1/2] headscale-node-cleaner: add image build --- .github/workflows/images.yaml | 1 + apps/headscale-node-cleaner/Dockerfile | 6 ++++++ hub/cluster/flux-system/image-policies.yaml | 4 ++++ hub/cluster/flux-system/image-repositories.yaml | 3 +++ 4 files changed, 14 insertions(+) create mode 100644 apps/headscale-node-cleaner/Dockerfile diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 70f4cd4..9ece269 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -27,6 +27,7 @@ jobs: name: - bootie - etcdetcetc + - headscale-node-cleaner - node-joiner steps: - name: Checkout repository diff --git a/apps/headscale-node-cleaner/Dockerfile b/apps/headscale-node-cleaner/Dockerfile new file mode 100644 index 0000000..28f5d09 --- /dev/null +++ b/apps/headscale-node-cleaner/Dockerfile @@ -0,0 +1,6 @@ +FROM ghcr.io/juanfont/headscale:v0.26.1 AS headscale + +FROM ghcr.io/flant/shell-operator:v1.4.16 + +COPY --from=headscale /ko-app/headscale /usr/local/bin/headscale +RUN chmod +x /usr/local/bin/headscale diff --git a/hub/cluster/flux-system/image-policies.yaml b/hub/cluster/flux-system/image-policies.yaml index 6f4ca7b..095ebfc 100644 --- a/hub/cluster/flux-system/image-policies.yaml +++ b/hub/cluster/flux-system/image-policies.yaml @@ -34,6 +34,10 @@ spec: spec: imageRepositoryRef: name: etcdetcetc + headscale-node-cleaner: + spec: + imageRepositoryRef: + name: headscale-node-cleaner node-joiner: spec: imageRepositoryRef: diff --git a/hub/cluster/flux-system/image-repositories.yaml b/hub/cluster/flux-system/image-repositories.yaml index 9f72934..5bef220 100644 --- a/hub/cluster/flux-system/image-repositories.yaml +++ b/hub/cluster/flux-system/image-repositories.yaml @@ -27,6 +27,9 @@ spec: etcdetcetc: spec: image: ghcr.io/samcday/infra-etcdetcetc + headscale-node-cleaner: + spec: + image: ghcr.io/samcday/infra-headscale-node-cleaner node-joiner: spec: image: ghcr.io/samcday/infra-node-joiner From 4ab85b3465a7cc62480189f463c1c3b3465f509f Mon Sep 17 00:00:00 2001 From: Sam Day Date: Thu, 7 May 2026 13:30:19 +1000 Subject: [PATCH 2/2] cloud-cluster: clean Headscale nodes on deletion --- .../cloud-cluster/headscale-node-cleaner.yaml | 283 ++++++++++++++++++ hub/cluster/cloud-cluster/kustomization.yaml | 1 + 2 files changed, 284 insertions(+) create mode 100644 hub/cluster/cloud-cluster/headscale-node-cleaner.yaml diff --git a/hub/cluster/cloud-cluster/headscale-node-cleaner.yaml b/hub/cluster/cloud-cluster/headscale-node-cleaner.yaml new file mode 100644 index 0000000..df28c78 --- /dev/null +++ b/hub/cluster/cloud-cluster/headscale-node-cleaner.yaml @@ -0,0 +1,283 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: headscale-node-cleaner + namespace: cloud-cluster +spec: + chart: + spec: + chart: charts/resources + reconcileStrategy: Revision + sourceRef: + kind: GitRepository + name: infra + namespace: flux-system + driftDetection: + mode: enabled + interval: 1h + values: + serviceAccount: + kind: ServiceAccount + metadata: + name: headscale-node-cleaner + adminApiKeyReader: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: headscale-node-cleaner + rules: + - apiGroups: [""] + resources: [secrets] + resourceNames: [admin-apikey] + verbs: [get] + adminApiKeyReaderBinding: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: headscale-node-cleaner + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: headscale-node-cleaner + subjects: + - kind: ServiceAccount + name: headscale-node-cleaner + namespace: cloud-cluster + hook: + kind: ConfigMap + metadata: + name: headscale-node-cleaner-hook + data: + hook.sh: | + #!/bin/bash + set -euo pipefail + + if [[ "${1:-}" == "--config" ]]; then + cat <<'YAML' + configVersion: v1 + kubernetes: + - name: cloud-nodes + apiVersion: v1 + kind: Node + labelSelector: + matchExpressions: + - key: instance.hetzner.cloud/provided-by + operator: In + values: [cloud] + executeHookOnEvent: [Added, Modified, Deleted] + executeHookOnSynchronization: true + jqFilter: | + { + name: .metadata.name, + deleting: (.metadata.deletionTimestamp != null), + finalizers: (.metadata.finalizers // []) + } + YAML + exit 0 + fi + + FINALIZER=${FINALIZER:-headscale.samcday.com/node-cleanup} + HEADSCALE_USER=${HEADSCALE_USER:-cloud} + HEADSCALE_TAG=${HEADSCALE_TAG:-tag:cloud-cluster-node} + HEADSCALE_CONFIG=${HEADSCALE_CONFIG:-/tmp/headscale/config.yaml} + HEADSCALE_CLI_ADDRESS=${HEADSCALE_CLI_ADDRESS:-headscale-generic.headscale.svc.cluster.hub.internal:50443} + HEADSCALE_API_KEY_SECRET_NAMESPACE=${HEADSCALE_API_KEY_SECRET_NAMESPACE:-headscale} + HEADSCALE_API_KEY_SECRET_NAME=${HEADSCALE_API_KEY_SECRET_NAME:-admin-apikey} + HEADSCALE_API_KEY_SECRET_KEY=${HEADSCALE_API_KEY_SECRET_KEY:-key} + CLOUD_KUBECONFIG=${KUBECONFIG:?KUBECONFIG must point at the child cluster kubeconfig} + + cloud_kubectl() { + kubectl --kubeconfig="$CLOUD_KUBECONFIG" "$@" + } + + hub_kubectl() { + env -u KUBECONFIG kubectl "$@" + } + + ensure_headscale_config() { + local api_key + api_key=$(hub_kubectl -n "$HEADSCALE_API_KEY_SECRET_NAMESPACE" \ + get secret "$HEADSCALE_API_KEY_SECRET_NAME" -o json | \ + jq -r --arg key "$HEADSCALE_API_KEY_SECRET_KEY" '.data[$key]' | base64 -d) + + mkdir -p "$(dirname "$HEADSCALE_CONFIG")" + cat >"$HEADSCALE_CONFIG" </dev/null || true) + if [[ -z "$current" ]] || [[ "$(jq -r '.metadata.deletionTimestamp != null' <<<"$current")" == "true" ]]; then + return + fi + + finalizers=$(jq -c '.metadata.finalizers // []' <<<"$current") + if jq -e --arg finalizer "$FINALIZER" 'index($finalizer)' <<<"$finalizers" >/dev/null; then + return + fi + + desired=$(jq -c --arg finalizer "$FINALIZER" '. + [$finalizer]' <<<"$finalizers") + echo "node $node_name: adding Headscale cleanup finalizer" + cloud_kubectl patch node "$node_name" --type=merge -p \ + "{\"metadata\":{\"finalizers\":$desired}}" + } + + cleanup_headscale() { + local node_name=$1 id + mapfile -t ids < <(matching_headscale_ids "$node_name") + if [[ ${#ids[@]} -eq 0 ]]; then + echo "node $node_name: no matching Headscale node found" + fi + + for id in "${ids[@]}"; do + echo "node $node_name: deleting Headscale node $id" + ensure_headscale_config + headscale -c "$HEADSCALE_CONFIG" nodes delete --force --identifier "$id" + done + + mapfile -t remaining < <(matching_headscale_ids "$node_name") + if [[ ${#remaining[@]} -ne 0 ]]; then + echo "node $node_name: Headscale nodes still present: ${remaining[*]}" >&2 + return 1 + fi + } + + remove_finalizer() { + local node_name=$1 current finalizers desired + current=$(cloud_kubectl get node "$node_name" -o json 2>/dev/null || true) + if [[ -z "$current" ]]; then + return + fi + + finalizers=$(jq -c '.metadata.finalizers // []' <<<"$current") + if ! jq -e --arg finalizer "$FINALIZER" 'index($finalizer)' <<<"$finalizers" >/dev/null; then + return + fi + + desired=$(jq -c --arg finalizer "$FINALIZER" 'map(select(. != $finalizer))' <<<"$finalizers") + echo "node $node_name: removing Headscale cleanup finalizer" + cloud_kubectl patch node "$node_name" --type=merge -p \ + "{\"metadata\":{\"finalizers\":$desired}}" + } + + reconcile() { + local op=$1 node_name=$2 deleting=$3 finalizers=$4 + if [[ -z "$node_name" || "$op" == "Deleted" ]]; then + return + fi + + if [[ "$deleting" == "true" ]]; then + if jq -e --arg finalizer "$FINALIZER" 'index($finalizer)' <<<"$finalizers" >/dev/null; then + cleanup_headscale "$node_name" + remove_finalizer "$node_name" + fi + return + fi + + ensure_finalizer "$node_name" + } + + jq -c '.[]' "$BINDING_CONTEXT_PATH" | while read -r row; do + type=$(jq -r '.type' <<<"$row") + case "$type" in + Synchronization) + jq -c '.objects[]? // empty' <<<"$row" | while read -r obj; do + reconcile "Synchronization" \ + "$(jq -r '.filterResult.name // ""' <<<"$obj")" \ + "$(jq -r '.filterResult.deleting // false' <<<"$obj")" \ + "$(jq -c '.filterResult.finalizers // []' <<<"$obj")" + done + ;; + Event) + reconcile \ + "$(jq -r '.watchEvent // ""' <<<"$row")" \ + "$(jq -r '.filterResult.name // ""' <<<"$row")" \ + "$(jq -r '.filterResult.deleting // false' <<<"$row")" \ + "$(jq -c '.filterResult.finalizers // []' <<<"$row")" + ;; + esac + done + deployment: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: headscale-node-cleaner + spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: headscale-node-cleaner + template: + metadata: + labels: + app.kubernetes.io/name: headscale-node-cleaner + spec: + serviceAccountName: headscale-node-cleaner + containers: + - name: shell-operator + image: ghcr.io/samcday/infra-headscale-node-cleaner:2026050700 # {"$imagepolicy": "flux-system:headscale-node-cleaner:tag"} + env: + - name: KUBECONFIG + value: /etc/cloud-kubeconfig/kubeconfig + - name: SHELL_OPERATOR_HOOKS_DIR + value: /hooks + - name: SHELL_OPERATOR_TMP_DIR + value: /tmp/shell-operator + - name: LOG_LEVEL + value: info + - name: HEADSCALE_CONFIG + value: /tmp/headscale/config.yaml + volumeMounts: + - name: cloud-kubeconfig + mountPath: /etc/cloud-kubeconfig + readOnly: true + - name: hooks + mountPath: /hooks + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: { cpu: 20m, memory: 128Mi } + limits: { memory: 256Mi } + volumes: + - name: cloud-kubeconfig + secret: + secretName: admin-kubeconfig + defaultMode: 256 + items: + - key: value + path: kubeconfig + - name: hooks + configMap: + name: headscale-node-cleaner-hook + defaultMode: 493 + - name: tmp + emptyDir: {} diff --git a/hub/cluster/cloud-cluster/kustomization.yaml b/hub/cluster/cloud-cluster/kustomization.yaml index 8d5d407..23349ad 100644 --- a/hub/cluster/cloud-cluster/kustomization.yaml +++ b/hub/cluster/cloud-cluster/kustomization.yaml @@ -25,6 +25,7 @@ resources: - flux-webhook-token.yaml - flux-webhook.yaml - hccm.yaml + - headscale-node-cleaner.yaml - hcloud-token.yaml - keda.yaml - keda-vpas.yaml