From f5bed7ddbb00ae148b559b2747a97951fdb8bf9c Mon Sep 17 00:00:00 2001 From: Ganeshkumar Ashokavardhanan Date: Mon, 13 Jul 2026 14:04:33 -0700 Subject: [PATCH 1/3] fix(gpu): tear down cuda-lts VHD prebake on GRID nodes before managed install The shared Ubuntu VHD prebakes only the cuda(-lts) driver plus a DKMS marker. A GRID / converged (A10, NVv5) node then installs the grid driver on top, and the stale prebaked cuda module + its /usr/bin/lib64 userspace libs collide with the grid driver, so nvidia-smi fails with "Failed to initialize NVML: Driver/library version mismatch". The existing cleanUpPrebakedGPUDriver only runs on nodes that do NOT install a managed driver (GPU_NODE != true OR skip), so managed GRID nodes keep the stale prebake. This adds cleanUpGridNodeCudaPrebake, run from ensureGPUDrivers before the configGPUDrivers/validateGPUDrivers dispatch (so both paths are covered), which tears down the prebake when this node installs grid and the marker is not grid (a cuda or legacy kind-less marker). Pure driver-kind mismatch, so no version comparison is needed. CUDA nodes are intentionally untouched. Scoped to the GRID mismatch only; the NAP cuda same-kind/different-version case is addressed separately by aligning the driver version NAP requests with the baked image. --- .../linux/cloud-init/artifacts/cse_config.sh | 37 ++++++++++ .../cloud-init/artifacts/cse_config_spec.sh | 74 +++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index f77dc1a1843..b57bf150eac 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -1321,11 +1321,48 @@ logGPUDriverPrebakeReadiness() { echo "AKS_GPU_PREBAKE event=managed_gpu driver_type=${NVIDIA_GPU_DRIVER_TYPE:-} marker_present=${marker_present} driver_kind_match=${driver_kind_match}" } +# cleanUpGridNodeCudaPrebake tears down a cuda-lts driver pre-baked into the shared Ubuntu VHD when +# THIS node installs a GRID driver. The shared VHD bakes only the cuda(-lts) driver + a DKMS marker; +# a GRID/converged (A10, NVv5) node then installs the grid driver on top, and the stale prebaked +# cuda module + its /usr/bin/lib64 userspace libs collide with the grid driver -> nvidia-smi fails +# with "Failed to initialize NVML: Driver/library version mismatch". This is a pure driver-KIND +# mismatch (grid vs cuda), so no version comparison is needed. A legacy marker with no driver_kind= +# line is treated as a cuda prebake (the only kind the VHD bakes today). No-op unless the node is +# GRID and a prebake marker exists. Reuses cleanUpPrebakedGPUDriver (from cse_install_ubuntu.sh) for +# the actual removal. NAP/agentpool cuda nodes are intentionally untouched here. +cleanUpGridNodeCudaPrebake() { + [ "$OS" = "$UBUNTU_OS_NAME" ] || return 0 + local marker="${GPU_DKMS_MARKER_FILE:-/opt/azure/aks-gpu/dkms-marker}" + [ -f "${marker}" ] || return 0 + + local node_kind m_kind + case "${NVIDIA_GPU_DRIVER_TYPE:-}" in + grid*) node_kind=grid ;; + *) return 0 ;; + esac + m_kind="$(sed -n 's/^driver_kind=//p' "${marker}" | head -n1)" + + # Keep only when the prebake is explicitly grid (matches this grid node). An empty marker kind is + # a legacy cuda prebake; a "cuda" marker is a cuda prebake -- both mismatch a grid node, tear down. + if [ "${m_kind}" = "grid" ]; then + return 0 + fi + echo "AKS_GPU_PREBAKE event=grid_cuda_prebake_teardown driver_type=${NVIDIA_GPU_DRIVER_TYPE:-} marker_kind=${m_kind:-none} node_kind=${node_kind} action=teardown" + cleanUpPrebakedGPUDriver +} + ensureGPUDrivers() { if [ "$(isARM64)" -eq 1 ]; then return fi + # Tear down a mismatched cuda-lts VHD prebake before a GRID node installs its own driver, or the + # stale module/libs collide with the grid driver (NVML version mismatch). Runs before the dispatch + # below so it covers both the configGPUDrivers and validateGPUDrivers paths. + if [ "$OS" = "$UBUNTU_OS_NAME" ]; then + logs_to_events "AKS.CSE.ensureGPUDrivers.cleanUpGridNodeCudaPrebake" cleanUpGridNodeCudaPrebake || exit $ERR_GPU_DRIVERS_START_FAIL + fi + if [ "${CONFIG_GPU_DRIVER_IF_NEEDED}" = true ]; then logs_to_events "AKS.CSE.ensureGPUDrivers.configGPUDrivers" configGPUDrivers else diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index c671b911f92..d016ac74920 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -74,6 +74,80 @@ Describe 'cse_config.sh' End End + Describe 'cleanUpGridNodeCudaPrebake' + # Stub the actual removal so tests assert the keep-vs-teardown DECISION without touching the + # real filesystem. OS defaults to Ubuntu (the only path this function acts on). + OS="$UBUNTU_OS_NAME" + # shellcheck disable=SC2329 # invoked dynamically by cleanUpGridNodeCudaPrebake. + cleanUpPrebakedGPUDriver() { echo "STUB_TEARDOWN_CALLED"; } + + It 'tears down a cuda prebake before installing a GRID driver (A10/GRID outage path)' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should include "action=teardown" + The output should include "marker_kind=cuda" + The output should include "node_kind=grid" + The output should include "STUB_TEARDOWN_CALLED" + rm -f "$marker" + End + + It 'tears down for a grid-v20 node whose driver-type maps to grid' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid-v20" + When call cleanUpGridNodeCudaPrebake + The output should include "action=teardown" + The output should include "STUB_TEARDOWN_CALLED" + rm -f "$marker" + End + + It 'treats a legacy marker without driver_kind as a cuda prebake and tears down on a GRID node' + marker="$(mktemp)" + printf 'kernel=5.15.0-1114-azure\n' > "$marker" # no driver_kind= line + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should include "action=teardown" + The output should include "marker_kind=none" + The output should include "STUB_TEARDOWN_CALLED" + rm -f "$marker" + End + + It 'is a no-op on a CUDA node (leaves the cuda prebake for the version-match/library-bump path)' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="cuda-lts" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + rm -f "$marker" + End + + It 'is a no-op when the prebake is already grid (matches the grid node)' + marker="$(mktemp)" + printf 'driver_kind=grid\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + rm -f "$marker" + End + + It 'is a no-op when no prebake marker exists' + GPU_DKMS_MARKER_FILE="$(mktemp)"; rm -f "${GPU_DKMS_MARKER_FILE}" + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + End + End + Describe 'configureAzureJson' AZURE_JSON_PATH="azure.json" AKS_CUSTOM_CLOUD_JSON_PATH="customcloud.json" From 102bc35c92ec812831d1e598f7390b0e2d5ec3e2 Mon Sep 17 00:00:00 2001 From: "aks-node-assistant[bot]" <190555641+aks-node-assistant[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 21:06:21 +0000 Subject: [PATCH 2/3] chore: auto-generate hotfix template entries --- parts/linux/cloud-init/nodecustomdata.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 93ad6ecc202..d0f89bb5416 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -34,6 +34,16 @@ write_files: {{GetVariableProperty "cloudInitData" "initAKSCustomCloud"}} {{end}} + +# ---- hotfix: auto-generated by hotfix-generate GH Action ---- +- path: {{GetCSEConfigScriptFilepath}} + permissions: "0744" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "provisionConfigs"}} + +# ---- end hotfix ---- {{- else }} - path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" From 995e244aa2444d7d55d4829dbba47409a3001579 Mon Sep 17 00:00:00 2001 From: Ganeshkumar Ashokavardhanan Date: Mon, 13 Jul 2026 14:42:54 -0700 Subject: [PATCH 3/3] test(gpu): add non-Ubuntu no-op case for cleanUpGridNodeCudaPrebake Addresses Copilot review on #8919: cover the OS != UBUNTU_OS_NAME early return so a regression that removes or inverts the Ubuntu gate is caught. --- .../linux/cloud-init/artifacts/cse_config_spec.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index d016ac74920..5f55dbd1500 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -146,6 +146,19 @@ Describe 'cse_config.sh' The output should not include "STUB_TEARDOWN_CALLED" The status should be success End + + It 'is a no-op on a non-Ubuntu OS even when a mismatched marker is present' + marker="$(mktemp)" + printf 'driver_kind=cuda\n' > "$marker" + GPU_DKMS_MARKER_FILE="$marker" + OS="MARINER" # override the Ubuntu default set at the Describe level + NVIDIA_GPU_DRIVER_TYPE="grid" + When call cleanUpGridNodeCudaPrebake + The output should not include "STUB_TEARDOWN_CALLED" + The status should be success + OS="$UBUNTU_OS_NAME" # restore for any subsequent examples + rm -f "$marker" + End End Describe 'configureAzureJson'