From 68265d61322d7d53910323455c01a8309e856335 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 16 Jun 2026 23:03:37 +0000
Subject: [PATCH 01/12] Initial plan


From 3aad1705f8ba4ce7631097acd9a39910da48b759 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 16 Jun 2026 23:08:20 +0000
Subject: [PATCH 02/12] Validate NPD in e2e flow

---
 hack/e2e/README.md       |  4 +--
 hack/e2e/lib/validate.sh | 65 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/hack/e2e/README.md b/hack/e2e/README.md
index c328a0cb..bbcfa15c 100644
--- a/hack/e2e/README.md
+++ b/hack/e2e/README.md
@@ -29,7 +29,7 @@ The default `all` command runs:
 1. Build the local `aks-flex-node` binary unless `--binary` or `--skip-build` is used.
 2. Deploy AKS and three VMs with Bicep.
 3. Join all three VMs.
-4. Validate node readiness and run smoke workloads.
+4. Validate node readiness, node-problem-detector status, and run smoke workloads.
 5. Unjoin all Flex Nodes and verify they are absent.
 6. Rejoin all Flex Nodes and validate again.
 7. Run local-machine-driven repave validation.
@@ -51,7 +51,7 @@ The default `all` command runs:
 | `unjoin-msi` | Unjoin only the managed-identity node. |
 | `unjoin-token` | Unjoin only the bootstrap-token node. |
 | `unjoin-kubeadm` | Unjoin only the kubeadm-style node. |
-| `validate` | Verify joined nodes and run smoke tests. |
+| `validate` | Verify joined nodes, node-problem-detector status, and run smoke tests. |
 | `validate-absent` | Verify Flex Node objects are absent after unjoin. |
 | `smoke` | Run smoke workloads only. |
 | `upgrade-drift` | Validate local-machine-driven repave to the alternate nspawn side. |
diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh
index 3bf60097..c53ba2b8 100755
--- a/hack/e2e/lib/validate.sh
+++ b/hack/e2e/lib/validate.sh
@@ -5,6 +5,7 @@
 # Functions:
 #   validate_node_joined  <vm_name>  - Wait for a specific node to appear in kubectl
 #   validate_all_nodes                - Verify MSI, token, and kubeadm nodes joined
+#   validate_npd_status   <vm_name> <vm_ip> - Verify node-problem-detector is active
 #   validate_node_absent  <vm_name>  - Wait for a node to disappear from kubectl
 #   validate_all_nodes_absent         - Verify all flex nodes are gone after unjoin
 #   smoke_test            <vm_name> <label>  - Schedule an nginx pod on a node
@@ -73,6 +74,63 @@ validate_node_ip() {
   return 1
 }
 
+# ---------------------------------------------------------------------------
+# validate_npd_status - Ensure node-problem-detector is active and reporting
+# ---------------------------------------------------------------------------
+validate_npd_status() {
+  local vm_name="$1"
+  local vm_ip="$2"
+  local timeout="${E2E_NODE_JOIN_TIMEOUT}"
+  local elapsed=0
+
+  log_info "Validating node-problem-detector on '${vm_name}'..."
+
+  remote_exec "${vm_ip}" "VM_NAME=${vm_name} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE'
+set -euo pipefail
+
+deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT))
+while true; do
+  active_machine="$(sudo sed -n 's/.*"activeMachine"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' /etc/aks-flex-node/daemon-state.json 2>/dev/null || true)"
+  if [[ -n "${active_machine}" ]] && machinectl show "${active_machine}" &>/dev/null; then
+    status="$(sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl is-active node-problem-detector.service 2>/dev/null || true)"
+    if [[ "${status}" == "active" ]]; then
+      echo "node-problem-detector.service is active in ${active_machine}"
+      exit 0
+    fi
+  fi
+
+  if (( SECONDS >= deadline )); then
+    echo "node-problem-detector.service did not become active on ${VM_NAME}"
+    machinectl list --no-pager || true
+    if [[ -n "${active_machine:-}" ]]; then
+      sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl status node-problem-detector.service --no-pager -l || true
+      sudo systemd-run --machine="${active_machine}" --quiet --pipe journalctl -u node-problem-detector.service -n 50 --no-pager || true
+    fi
+    exit 1
+  fi
+
+  sleep 5
+done
+REMOTE
+
+  while [[ "${elapsed}" -lt "${timeout}" ]]; do
+    local kernel_deadlock
+    kernel_deadlock="$(kubectl get node "${vm_name}" -o jsonpath='{.status.conditions[?(@.type=="KernelDeadlock")].status}' 2>/dev/null || true)"
+    if [[ "${kernel_deadlock}" == "False" ]]; then
+      log_success "node-problem-detector is active and reporting on '${vm_name}'"
+      return 0
+    fi
+
+    sleep 10
+    elapsed=$((elapsed + 10))
+    log_debug "Waiting for node-problem-detector condition on ${vm_name}... (${elapsed}/${timeout}s)"
+  done
+
+  log_error "node-problem-detector did not report KernelDeadlock=False on '${vm_name}' within ${timeout}s"
+  kubectl describe node "${vm_name}" 2>&1 || true
+  return 1
+}
+
 # ---------------------------------------------------------------------------
 # validate_all_nodes - Check all MSI, token, and kubeadm VMs joined
 # ---------------------------------------------------------------------------
@@ -91,10 +149,14 @@ validate_all_nodes() {
     --admin
 
   local msi_vm_name token_vm_name kubeadm_vm_name
+  local msi_vm_ip token_vm_ip kubeadm_vm_ip
   local token_vm_private_ip
   msi_vm_name="$(state_get msi_vm_name)"
   token_vm_name="$(state_get token_vm_name)"
   kubeadm_vm_name="$(state_get kubeadm_vm_name)"
+  msi_vm_ip="$(state_get msi_vm_ip)"
+  token_vm_ip="$(state_get token_vm_ip)"
+  kubeadm_vm_ip="$(state_get kubeadm_vm_ip)"
   token_vm_private_ip="$(state_get token_vm_private_ip)"
 
   local failed=0
@@ -102,6 +164,9 @@ validate_all_nodes() {
   validate_node_joined "${token_vm_name}" || failed=1
   validate_node_joined "${kubeadm_vm_name}" || failed=1
   validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1
+  validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1
+  validate_npd_status "${token_vm_name}" "${token_vm_ip}" || failed=1
+  validate_npd_status "${kubeadm_vm_name}" "${kubeadm_vm_ip}" || failed=1
 
   if [[ "${failed}" -eq 1 ]]; then
     log_error "One or more nodes failed to join"

From 58bfa492c689f962ad5c8c4b18a0ab3ea422c73c Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 16 Jun 2026 23:09:04 +0000
Subject: [PATCH 03/12] Address NPD validation review feedback

---
 hack/e2e/lib/validate.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh
index c53ba2b8..82d1fdff 100755
--- a/hack/e2e/lib/validate.sh
+++ b/hack/e2e/lib/validate.sh
@@ -85,12 +85,17 @@ validate_npd_status() {
 
   log_info "Validating node-problem-detector on '${vm_name}'..."
 
-  remote_exec "${vm_ip}" "VM_NAME=${vm_name} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE'
+  remote_exec "${vm_ip}" "E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE'
 set -euo pipefail
 
 deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT))
 while true; do
-  active_machine="$(sudo sed -n 's/.*"activeMachine"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' /etc/aks-flex-node/daemon-state.json 2>/dev/null || true)"
+  active_machine="$(sudo python3 - <<'PY' 2>/dev/null || true
+import json
+with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
+    print(json.load(state).get("activeMachine", ""))
+PY
+)"
   if [[ -n "${active_machine}" ]] && machinectl show "${active_machine}" &>/dev/null; then
     status="$(sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl is-active node-problem-detector.service 2>/dev/null || true)"
     if [[ "${status}" == "active" ]]; then
@@ -100,7 +105,7 @@ while true; do
   fi
 
   if (( SECONDS >= deadline )); then
-    echo "node-problem-detector.service did not become active on ${VM_NAME}"
+    echo "node-problem-detector.service did not become active"
     machinectl list --no-pager || true
     if [[ -n "${active_machine:-}" ]]; then
       sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl status node-problem-detector.service --no-pager -l || true
@@ -113,8 +118,8 @@ while true; do
 done
 REMOTE
 
+  local kernel_deadlock
   while [[ "${elapsed}" -lt "${timeout}" ]]; do
-    local kernel_deadlock
     kernel_deadlock="$(kubectl get node "${vm_name}" -o jsonpath='{.status.conditions[?(@.type=="KernelDeadlock")].status}' 2>/dev/null || true)"
     if [[ "${kernel_deadlock}" == "False" ]]; then
       log_success "node-problem-detector is active and reporting on '${vm_name}'"

From c4ef0d41c53a5c73691df5f3a585d47978e13f79 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 16 Jun 2026 23:09:55 +0000
Subject: [PATCH 04/12] Refine NPD validation checks

---
 hack/e2e/lib/validate.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh
index 82d1fdff..8785a9fd 100755
--- a/hack/e2e/lib/validate.sh
+++ b/hack/e2e/lib/validate.sh
@@ -82,10 +82,16 @@ validate_npd_status() {
   local vm_ip="$2"
   local timeout="${E2E_NODE_JOIN_TIMEOUT}"
   local elapsed=0
+  local npd_condition_jsonpath='{.status.conditions[?(@.type=="KernelDeadlock")].status}'
 
   log_info "Validating node-problem-detector on '${vm_name}'..."
 
-  remote_exec "${vm_ip}" "E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} bash -s" <<'REMOTE'
+  if ! [[ "${timeout}" =~ ^[0-9]+$ ]]; then
+    log_error "E2E_NODE_JOIN_TIMEOUT must be numeric, got '${timeout}'"
+    return 1
+  fi
+
+  remote_exec "${vm_ip}" "E2E_NODE_JOIN_TIMEOUT=${timeout} bash -s" <<'REMOTE'
 set -euo pipefail
 
 deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT))
@@ -120,7 +126,7 @@ REMOTE
 
   local kernel_deadlock
   while [[ "${elapsed}" -lt "${timeout}" ]]; do
-    kernel_deadlock="$(kubectl get node "${vm_name}" -o jsonpath='{.status.conditions[?(@.type=="KernelDeadlock")].status}' 2>/dev/null || true)"
+    kernel_deadlock="$(kubectl get node "${vm_name}" -o jsonpath="${npd_condition_jsonpath}" 2>/dev/null || true)"
     if [[ "${kernel_deadlock}" == "False" ]]; then
       log_success "node-problem-detector is active and reporting on '${vm_name}'"
       return 0

From 92f521c2a2f634faf37b8a58f4cb7e50b1c5048c Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 16 Jun 2026 23:10:54 +0000
Subject: [PATCH 05/12] Harden NPD validation script

---
 hack/e2e/lib/validate.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh
index 8785a9fd..bb46072e 100755
--- a/hack/e2e/lib/validate.sh
+++ b/hack/e2e/lib/validate.sh
@@ -83,6 +83,7 @@ validate_npd_status() {
   local timeout="${E2E_NODE_JOIN_TIMEOUT}"
   local elapsed=0
   local npd_condition_jsonpath='{.status.conditions[?(@.type=="KernelDeadlock")].status}'
+  local quoted_timeout
 
   log_info "Validating node-problem-detector on '${vm_name}'..."
 
@@ -90,13 +91,15 @@ validate_npd_status() {
     log_error "E2E_NODE_JOIN_TIMEOUT must be numeric, got '${timeout}'"
     return 1
   fi
+  printf -v quoted_timeout "%q" "${timeout}"
 
-  remote_exec "${vm_ip}" "E2E_NODE_JOIN_TIMEOUT=${timeout} bash -s" <<'REMOTE'
+  remote_exec "${vm_ip}" "E2E_NODE_JOIN_TIMEOUT=${quoted_timeout} bash -s" <<'REMOTE'
 set -euo pipefail
 
 deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT))
+active_machine_error="/tmp/aks-flex-node-e2e-active-machine.err"
 while true; do
-  active_machine="$(sudo python3 - <<'PY' 2>/dev/null || true
+  active_machine="$(sudo python3 - <<'PY' 2>"${active_machine_error}" || true
 import json
 with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
     print(json.load(state).get("activeMachine", ""))
@@ -112,6 +115,9 @@ PY
 
   if (( SECONDS >= deadline )); then
     echo "node-problem-detector.service did not become active"
+    if [[ -s "${active_machine_error}" ]]; then
+      cat "${active_machine_error}"
+    fi
     machinectl list --no-pager || true
     if [[ -n "${active_machine:-}" ]]; then
       sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl status node-problem-detector.service --no-pager -l || true

From 0af12f89e6797d301ed7fef8d375c3930558d1f9 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 16 Jun 2026 23:16:47 +0000
Subject: [PATCH 06/12] Improve NPD validation diagnostics

---
 hack/e2e/lib/validate.sh | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh
index bb46072e..12bd8868 100755
--- a/hack/e2e/lib/validate.sh
+++ b/hack/e2e/lib/validate.sh
@@ -83,6 +83,7 @@ validate_npd_status() {
   local timeout="${E2E_NODE_JOIN_TIMEOUT}"
   local elapsed=0
   local npd_condition_jsonpath='{.status.conditions[?(@.type=="KernelDeadlock")].status}'
+  local condition_error="${E2E_WORK_DIR}/npd-condition-${vm_name}.err"
   local quoted_timeout
 
   log_info "Validating node-problem-detector on '${vm_name}'..."
@@ -97,16 +98,22 @@ validate_npd_status() {
 set -euo pipefail
 
 deadline=$((SECONDS + E2E_NODE_JOIN_TIMEOUT))
-active_machine_error="/tmp/aks-flex-node-e2e-active-machine.err"
+active_machine_error="/tmp/aks-flex-node-e2e-active-machine-$$.err"
+status_error="/tmp/aks-flex-node-e2e-npd-status-$$.err"
 while true; do
-  active_machine="$(sudo python3 - <<'PY' 2>"${active_machine_error}" || true
+  if [[ ! -f /etc/aks-flex-node/daemon-state.json ]]; then
+    active_machine=""
+    echo "/etc/aks-flex-node/daemon-state.json is missing" > "${active_machine_error}"
+  else
+    active_machine="$(sudo python3 - <<'PY' 2>"${active_machine_error}" || true
 import json
 with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
     print(json.load(state).get("activeMachine", ""))
 PY
 )"
+  fi
   if [[ -n "${active_machine}" ]] && machinectl show "${active_machine}" &>/dev/null; then
-    status="$(sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl is-active node-problem-detector.service 2>/dev/null || true)"
+    status="$(sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl is-active node-problem-detector.service 2>"${status_error}" || true)"
     if [[ "${status}" == "active" ]]; then
       echo "node-problem-detector.service is active in ${active_machine}"
       exit 0
@@ -118,6 +125,9 @@ PY
     if [[ -s "${active_machine_error}" ]]; then
       cat "${active_machine_error}"
     fi
+    if [[ -s "${status_error}" ]]; then
+      cat "${status_error}"
+    fi
     machinectl list --no-pager || true
     if [[ -n "${active_machine:-}" ]]; then
       sudo systemd-run --machine="${active_machine}" --quiet --pipe systemctl status node-problem-detector.service --no-pager -l || true
@@ -132,7 +142,7 @@ REMOTE
 
   local kernel_deadlock
   while [[ "${elapsed}" -lt "${timeout}" ]]; do
-    kernel_deadlock="$(kubectl get node "${vm_name}" -o jsonpath="${npd_condition_jsonpath}" 2>/dev/null || true)"
+    kernel_deadlock="$(kubectl get node "${vm_name}" -o jsonpath="${npd_condition_jsonpath}" 2>"${condition_error}" || true)"
     if [[ "${kernel_deadlock}" == "False" ]]; then
       log_success "node-problem-detector is active and reporting on '${vm_name}'"
       return 0
@@ -144,6 +154,9 @@ REMOTE
   done
 
   log_error "node-problem-detector did not report KernelDeadlock=False on '${vm_name}' within ${timeout}s"
+  if [[ -s "${condition_error}" ]]; then
+    cat "${condition_error}" >&2
+  fi
   kubectl describe node "${vm_name}" 2>&1 || true
   return 1
 }

From 6831aead32ef038f4a438f8a54516759c1be9a90 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 17 Jun 2026 00:18:24 +0000
Subject: [PATCH 07/12] Collect NPD logs in E2E

---
 hack/e2e/README.md      |  2 +-
 hack/e2e/lib/cleanup.sh | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/hack/e2e/README.md b/hack/e2e/README.md
index bbcfa15c..b7e7725b 100644
--- a/hack/e2e/README.md
+++ b/hack/e2e/README.md
@@ -197,6 +197,6 @@ Logs are collected under `$E2E_WORK_DIR/logs/`.
 - **Missing prerequisites:** run `./hack/e2e/run.sh --help` and confirm `az`, `jq`, `kubectl`, `ssh`, `scp`, and `openssl` are available.
 - **Azure auth failures:** run `az account show` and `az login` if needed.
 - **SSH failures:** inspect `state.json` for VM public IPs and confirm the SSH key configured by `E2E_SSH_KEY_FILE` is available.
-- **Node join failures:** run `./hack/e2e/run.sh logs` and inspect agent, bootstrap unit, kubelet, and containerd logs.
+- **Node join failures:** run `./hack/e2e/run.sh logs` and inspect agent, bootstrap unit, kubelet, containerd, and node-problem-detector logs.
 - **Repave failures:** check `aks-flex-node-agent` logs, `machinectl list`, and kubelet versions inside `kube1` and `kube2`.
 - **Leftover resources:** run `E2E_RESOURCE_GROUP=<rg> ./hack/e2e/run.sh cleanup`.
diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh
index fcc98f9a..835be7da 100755
--- a/hack/e2e/lib/cleanup.sh
+++ b/hack/e2e/lib/cleanup.sh
@@ -52,6 +52,20 @@ _collect_vm_logs() {
      fi" \
     > "${E2E_LOG_DIR}/${prefix}-containerd.log" 2>/dev/null || true
 
+  remote_exec "${vm_ip}" "bash -s" <<'REMOTE' > "${E2E_LOG_DIR}/${prefix}-npd.log" 2>/dev/null || true
+machine="$(sudo python3 - <<'PY' 2>/dev/null || true
+import json
+with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
+    print(json.load(state).get("activeMachine", ""))
+PY
+)"
+if [ -n "${machine}" ]; then
+  sudo systemd-run --machine="${machine}" --quiet --pipe journalctl -u node-problem-detector.service -n 500 --no-pager 2>/dev/null
+else
+  sudo journalctl -u node-problem-detector.service -n 500 --no-pager 2>/dev/null
+fi
+REMOTE
+
   # Collect CNI config and nspawn machine state for networking diagnostics.
   # Read directly from the nspawn rootfs at /var/lib/machines/kube1/.
   local kube1_root="/var/lib/machines/kube1"

From be31c15b37cef242d52a9ee3af902fd775a009f1 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 17 Jun 2026 00:19:16 +0000
Subject: [PATCH 08/12] Surface NPD log collection errors

---
 hack/e2e/lib/cleanup.sh | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh
index 835be7da..52d687fe 100755
--- a/hack/e2e/lib/cleanup.sh
+++ b/hack/e2e/lib/cleanup.sh
@@ -52,17 +52,27 @@ _collect_vm_logs() {
      fi" \
     > "${E2E_LOG_DIR}/${prefix}-containerd.log" 2>/dev/null || true
 
-  remote_exec "${vm_ip}" "bash -s" <<'REMOTE' > "${E2E_LOG_DIR}/${prefix}-npd.log" 2>/dev/null || true
-machine="$(sudo python3 - <<'PY' 2>/dev/null || true
+  remote_exec "${vm_ip}" "bash -s" <<'REMOTE' > "${E2E_LOG_DIR}/${prefix}-npd.log" 2>&1 || true
+machine_error="$(mktemp)"
+machine="$(sudo python3 - <<'PY' 2>"${machine_error}" || true
 import json
 with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
     print(json.load(state).get("activeMachine", ""))
 PY
 )"
+if [ -s "${machine_error}" ]; then
+  echo "warning: unable to read active machine from daemon state"
+  cat "${machine_error}"
+fi
+rm -f "${machine_error}"
 if [ -n "${machine}" ]; then
-  sudo systemd-run --machine="${machine}" --quiet --pipe journalctl -u node-problem-detector.service -n 500 --no-pager 2>/dev/null
+  echo "=== node-problem-detector.service logs (${machine}) ==="
+  sudo systemd-run --machine="${machine}" --quiet --pipe journalctl -u node-problem-detector.service -n 500 --no-pager || \
+    echo "warning: failed to collect node-problem-detector logs from ${machine}"
 else
-  sudo journalctl -u node-problem-detector.service -n 500 --no-pager 2>/dev/null
+  echo "warning: active machine unknown; falling back to host journal"
+  sudo journalctl -u node-problem-detector.service -n 500 --no-pager || \
+    echo "warning: failed to collect host node-problem-detector logs"
 fi
 REMOTE
 

From cf2a5add0b625653c4dbc30e1e4e52ab05e42235 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 17 Jun 2026 00:20:12 +0000
Subject: [PATCH 09/12] Harden NPD log diagnostics

---
 hack/e2e/lib/cleanup.sh | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh
index 52d687fe..a5fe73dd 100755
--- a/hack/e2e/lib/cleanup.sh
+++ b/hack/e2e/lib/cleanup.sh
@@ -54,19 +54,29 @@ _collect_vm_logs() {
 
   remote_exec "${vm_ip}" "bash -s" <<'REMOTE' > "${E2E_LOG_DIR}/${prefix}-npd.log" 2>&1 || true
 machine_error="$(mktemp)"
-machine="$(sudo python3 - <<'PY' 2>"${machine_error}" || true
+trap 'rm -f "${machine_error}"' EXIT
+machine="$(sudo python3 - <<'PY' 2>"${machine_error}"
 import json
-with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
-    print(json.load(state).get("activeMachine", ""))
+import sys
+
+try:
+    with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
+        print(json.load(state).get("activeMachine", ""))
+except FileNotFoundError as exc:
+    print(f"daemon state not found: {exc}", file=sys.stderr)
+except json.JSONDecodeError as exc:
+    print(f"daemon state is not valid JSON: {exc}", file=sys.stderr)
+except PermissionError as exc:
+    print(f"daemon state permission denied: {exc}", file=sys.stderr)
 PY
 )"
 if [ -s "${machine_error}" ]; then
   echo "warning: unable to read active machine from daemon state"
   cat "${machine_error}"
 fi
-rm -f "${machine_error}"
 if [ -n "${machine}" ]; then
   echo "=== node-problem-detector.service logs (${machine}) ==="
+  # Match the agent and kubelet log depth; NPD entries are sparse but useful across node lifecycle phases.
   sudo systemd-run --machine="${machine}" --quiet --pipe journalctl -u node-problem-detector.service -n 500 --no-pager || \
     echo "warning: failed to collect node-problem-detector logs from ${machine}"
 else

From 4db154d3255b1a3e603aada31c82189b16639855 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 17 Jun 2026 00:20:52 +0000
Subject: [PATCH 10/12] Tighten NPD log cleanup

---
 hack/e2e/lib/cleanup.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh
index a5fe73dd..b19d6679 100755
--- a/hack/e2e/lib/cleanup.sh
+++ b/hack/e2e/lib/cleanup.sh
@@ -74,6 +74,8 @@ if [ -s "${machine_error}" ]; then
   echo "warning: unable to read active machine from daemon state"
   cat "${machine_error}"
 fi
+rm -f "${machine_error}"
+trap - EXIT
 if [ -n "${machine}" ]; then
   echo "=== node-problem-detector.service logs (${machine}) ==="
   # Match the agent and kubelet log depth; NPD entries are sparse but useful across node lifecycle phases.
@@ -82,7 +84,7 @@ if [ -n "${machine}" ]; then
 else
   echo "warning: active machine unknown; falling back to host journal"
   sudo journalctl -u node-problem-detector.service -n 500 --no-pager || \
-    echo "warning: failed to collect host node-problem-detector logs"
+    echo "warning: failed to collect node-problem-detector logs from host"
 fi
 REMOTE
 

From aed10a0be3b04c72ff926cb052aa05c6fd35451f Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 17 Jun 2026 00:21:40 +0000
Subject: [PATCH 11/12] Simplify NPD log diagnostics

---
 hack/e2e/lib/cleanup.sh | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh
index b19d6679..2de190cf 100755
--- a/hack/e2e/lib/cleanup.sh
+++ b/hack/e2e/lib/cleanup.sh
@@ -53,15 +53,18 @@ _collect_vm_logs() {
     > "${E2E_LOG_DIR}/${prefix}-containerd.log" 2>/dev/null || true
 
   remote_exec "${vm_ip}" "bash -s" <<'REMOTE' > "${E2E_LOG_DIR}/${prefix}-npd.log" 2>&1 || true
-machine_error="$(mktemp)"
-trap 'rm -f "${machine_error}"' EXIT
-machine="$(sudo python3 - <<'PY' 2>"${machine_error}"
+service="node-problem-detector.service"
+machine="$(sudo python3 - <<'PY'
 import json
 import sys
 
 try:
     with open("/etc/aks-flex-node/daemon-state.json", encoding="utf-8") as state:
-        print(json.load(state).get("activeMachine", ""))
+        active_machine = json.load(state).get("activeMachine", "")
+    if active_machine:
+        print(active_machine)
+    else:
+        print("daemon state does not include activeMachine", file=sys.stderr)
 except FileNotFoundError as exc:
     print(f"daemon state not found: {exc}", file=sys.stderr)
 except json.JSONDecodeError as exc:
@@ -70,21 +73,15 @@ except PermissionError as exc:
     print(f"daemon state permission denied: {exc}", file=sys.stderr)
 PY
 )"
-if [ -s "${machine_error}" ]; then
-  echo "warning: unable to read active machine from daemon state"
-  cat "${machine_error}"
-fi
-rm -f "${machine_error}"
-trap - EXIT
 if [ -n "${machine}" ]; then
-  echo "=== node-problem-detector.service logs (${machine}) ==="
+  echo "=== ${service} logs (${machine}) ==="
   # Match the agent and kubelet log depth; NPD entries are sparse but useful across node lifecycle phases.
-  sudo systemd-run --machine="${machine}" --quiet --pipe journalctl -u node-problem-detector.service -n 500 --no-pager || \
-    echo "warning: failed to collect node-problem-detector logs from ${machine}"
+  sudo systemd-run --machine="${machine}" --quiet --pipe journalctl -u "${service}" -n 500 --no-pager || \
+    echo "warning: failed to collect ${service} logs from ${machine}"
 else
   echo "warning: active machine unknown; falling back to host journal"
-  sudo journalctl -u node-problem-detector.service -n 500 --no-pager || \
-    echo "warning: failed to collect node-problem-detector logs from host"
+  sudo journalctl -u "${service}" -n 500 --no-pager || \
+    echo "warning: failed to collect ${service} logs from host"
 fi
 REMOTE
 

From d89a6317d2f3500615f4c8e5f9139c826bece1d0 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 17 Jun 2026 00:22:18 +0000
Subject: [PATCH 12/12] Clarify NPD log variables

---
 hack/e2e/lib/cleanup.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh
index 2de190cf..f4957e60 100755
--- a/hack/e2e/lib/cleanup.sh
+++ b/hack/e2e/lib/cleanup.sh
@@ -53,8 +53,8 @@ _collect_vm_logs() {
     > "${E2E_LOG_DIR}/${prefix}-containerd.log" 2>/dev/null || true
 
   remote_exec "${vm_ip}" "bash -s" <<'REMOTE' > "${E2E_LOG_DIR}/${prefix}-npd.log" 2>&1 || true
-service="node-problem-detector.service"
-machine="$(sudo python3 - <<'PY'
+npd_service="node-problem-detector.service"
+active_machine="$(sudo python3 - <<'PY'
 import json
 import sys
 
@@ -73,15 +73,15 @@ except PermissionError as exc:
     print(f"daemon state permission denied: {exc}", file=sys.stderr)
 PY
 )"
-if [ -n "${machine}" ]; then
-  echo "=== ${service} logs (${machine}) ==="
+if [ -n "${active_machine}" ]; then
+  echo "=== ${npd_service} logs (${active_machine}) ==="
   # Match the agent and kubelet log depth; NPD entries are sparse but useful across node lifecycle phases.
-  sudo systemd-run --machine="${machine}" --quiet --pipe journalctl -u "${service}" -n 500 --no-pager || \
-    echo "warning: failed to collect ${service} logs from ${machine}"
+  sudo systemd-run --machine="${active_machine}" --quiet --pipe journalctl -u "${npd_service}" -n 500 --no-pager || \
+    echo "warning: failed to collect ${npd_service} logs from ${active_machine}"
 else
   echo "warning: active machine unknown; falling back to host journal"
-  sudo journalctl -u "${service}" -n 500 --no-pager || \
-    echo "warning: failed to collect ${service} logs from host"
+  sudo journalctl -u "${npd_service}" -n 500 --no-pager || \
+    echo "warning: failed to collect ${npd_service} logs from host"
 fi
 REMOTE