From 77952d11753fdf8cdf8116659c34f969a56bba3d Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Wed, 8 Jul 2026 15:16:04 -0700 Subject: [PATCH] Label GKE GPU node pools so the autoscaler can cold-start from zero A GPU node pool with minNodeCount 0 never scaled up. A model's pod claims its GPU through a DRA ResourceClaim, which binds only against a ResourceSlice the NVIDIA DRA driver publishes per running GPU node. At zero nodes there are no slices, and GKE's cluster autoscaler couldn't tell that a new node would satisfy the claim, so it refused to create the first one. The pod stayed pending with "cannot allocate all claims". GKE's autoscaler recognises a DRA-capable node pool by the cloud.google.com/gke-nvidia-gpu-dra-driver=true node label, which lets it model the ResourceSlices such a node would publish when simulating a scale-up. compose-gke-cluster set only modelplane.ai/gpu and modelplane.ai/pool, so the pools were never recognised. This sets the label on every GPU node pool. The getting-started GKE manifests pinned minNodeCount 1 to work around the stall; they now allow minNodeCount 0. Fixes #306. Signed-off-by: Nic Cope --- docs/manifests/getting-started/gke/platform-scale.yaml | 4 ++-- docs/manifests/getting-started/gke/platform.yaml | 2 +- functions/compose-gke-cluster/function/fn.py | 10 ++++++++++ functions/compose-gke-cluster/tests/test_fn.py | 2 ++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/manifests/getting-started/gke/platform-scale.yaml b/docs/manifests/getting-started/gke/platform-scale.yaml index 2a2ec741..6861ed08 100644 --- a/docs/manifests/getting-started/gke/platform-scale.yaml +++ b/docs/manifests/getting-started/gke/platform-scale.yaml @@ -42,7 +42,7 @@ spec: - name: gpu-a100 className: gke-a100-40-1x nodeCount: 1 - minNodeCount: 1 # keep >=1; the autoscaler can't scale a GPU pool up from 0 for DRA pods + minNodeCount: 0 maxNodeCount: 2 zones: - us-west1-b @@ -63,7 +63,7 @@ spec: - name: gpu-a100 className: gke-a100-40-1x nodeCount: 1 - minNodeCount: 1 # keep >=1; the autoscaler can't scale a GPU pool up from 0 for DRA pods + minNodeCount: 0 maxNodeCount: 2 zones: - us-east1-b diff --git a/docs/manifests/getting-started/gke/platform.yaml b/docs/manifests/getting-started/gke/platform.yaml index 79b4ef3e..5993eca4 100644 --- a/docs/manifests/getting-started/gke/platform.yaml +++ b/docs/manifests/getting-started/gke/platform.yaml @@ -39,7 +39,7 @@ spec: - name: gpu-l4 className: gke-l4-1x-g2 nodeCount: 1 - minNodeCount: 1 # keep >=1; the autoscaler can't scale a GPU pool up from 0 for DRA pods + minNodeCount: 0 maxNodeCount: 2 zones: - us-central1-a diff --git a/functions/compose-gke-cluster/function/fn.py b/functions/compose-gke-cluster/function/fn.py index 6e8621c0..e1c15355 100644 --- a/functions/compose-gke-cluster/function/fn.py +++ b/functions/compose-gke-cluster/function/fn.py @@ -63,6 +63,15 @@ _LABEL_GPU = "modelplane.ai/gpu" _LABEL_POOL = "modelplane.ai/pool" +# GKE's cluster autoscaler uses this label to recognise a node pool as one whose +# nodes run the NVIDIA GPU DRA driver, and so models the DRA ResourceSlices such +# a node would publish. Without it the autoscaler can't tell that a new node +# would satisfy a pod's GPU ResourceClaim, so a pool sitting at zero nodes never +# scales up: the claim binds only against a ResourceSlice, and no slice exists +# until a node is already running. Setting it lets a GPU pool cold-start from +# minNodeCount 0. +_LABEL_GPU_DRA_DRIVER = "cloud.google.com/gke-nvidia-gpu-dra-driver" + # Secret types written to XR status. compose-inference-cluster reads # these to wire the kubeconfig and SA key into ProviderConfigs. The SA key's # type is the provider identity it authenticates as (see _IDENTITY_TYPE_GCP). @@ -293,6 +302,7 @@ def compose_node_pools(self) -> None: node_config.labels = { _LABEL_GPU: pool.gpu.acceleratorType, _LABEL_POOL: pool.name, + _LABEL_GPU_DRA_DRIVER: "true", } else: node_config.labels = { diff --git a/functions/compose-gke-cluster/tests/test_fn.py b/functions/compose-gke-cluster/tests/test_fn.py index 7283328b..012a6f00 100644 --- a/functions/compose-gke-cluster/tests/test_fn.py +++ b/functions/compose-gke-cluster/tests/test_fn.py @@ -245,6 +245,7 @@ async def test_compose(self) -> None: "labels": { "modelplane.ai/gpu": "nvidia-tesla-a100", "modelplane.ai/pool": "gpu-pool", + "cloud.google.com/gke-nvidia-gpu-dra-driver": "true", }, }, }, @@ -636,6 +637,7 @@ async def test_compose(self) -> None: "labels": { "modelplane.ai/gpu": "nvidia-tesla-a100", "modelplane.ai/pool": "gpu-pool", + "cloud.google.com/gke-nvidia-gpu-dra-driver": "true", }, }, },