From a5652e14b5caf9c87e97491cd94aeecfe1c5da73 Mon Sep 17 00:00:00 2001 From: Derek Pitman Date: Wed, 15 Jul 2026 14:26:53 -0400 Subject: [PATCH 1/4] [TRTLLMINF-81][test] Stop retrying a SLURM stage on its dead dispatcher pod When the K8s dispatcher pod dies mid-run (kubelet eviction, container termination, agent offline), the inner SLURM retry classified the wrapped "marked offline" as a transient infra failure and started another attempt -- which then executes on the same dead pod and fails immediately, spamming AgentOfflineException and burning the retry budget for nothing. Detect a dispatcher-pod death from the flattened cause chain (robust to the cleanup AbortException that wraps it) and fail closed instead of retrying in place. This is the first half of dispatcher-pod-death handling; off-pod reconciliation of the orphaned SLURM job / Jenkins node follows. Signed-off-by: Derek Pitman --- jenkins/L0_Test.groovy | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 1978dcf5bb68..e392ef1afbbb 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1843,6 +1843,25 @@ def cbtsResizeSplits(configs) { return resized } +// True when an exception indicates the K8s dispatcher pod this SLURM stage runs +// inside died mid-run -- kubelet eviction, container termination, or the JNLP +// agent otherwise going offline. Retrying inside such a pod is futile (every +// step runs on the dead agent and fails immediately) and its in-pod cleanup can +// no longer reach the SLURM controller, so callers stop retrying in place and +// reconcile the orphaned SLURM job / Jenkins node off-pod. Matches the flattened +// cause chain so the signal is still recognized when wrapped by the cleanup's +// AbortException (e.g. "Error during clean up SLURM resources: ... marked +// offline: Pod failed (Reason: Evicted ...)"). +boolean isDispatcherPodFailure(Throwable e) { + def text = FailureClassifier.flattenThrowable(e).collect { it.toString() }.join(" ").toLowerCase() + return [ + "pod failed (reason:", + "pod just failed", + "pod failed because container terminated", + "unable to create live filepath", + ].any { text.contains(it) } +} + def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, cpver="cp312", String outerAttemptTag="", boolean useClusterDurations=false) { echo "Run Slurm job with native sbatch: $runWithSbatch" @@ -1900,6 +1919,15 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p // User abort / pipeline timeout -- never retry throw e } catch (Exception e) { + // If the K8s dispatcher pod this stage runs inside died mid-run, every + // retry attempt would execute on that same dead pod and fail immediately, + // and the in-pod cleanup can no longer reach the SLURM controller. Stop + // retrying in place and propagate so the pod-level wrapper reconciles the + // orphaned SLURM job / Jenkins node off-pod (fail closed). + if (isDispatcherPodFailure(e)) { + echo "[INFRA-RETRY] ${stageName}: dispatcher pod died mid-run; not retrying on the dead pod (${e.toString()}). Failing closed for off-pod reconciliation." + throw e + } // classify() handles FlowInterruptedException + exit-code-143 + // typed throws + cause-chain pattern matching, returning one of // PipelineInterruption / InfraFailure / UserFailure. Scope=SLURM From 8419d0ba51604817f3bdf7e0aa8e3a1fb69df01a Mon Sep 17 00:00:00 2001 From: Derek Pitman Date: Wed, 15 Jul 2026 14:46:50 -0400 Subject: [PATCH 2/4] [TRTLLMINF-81][test] Reconcile orphaned SLURM resources off-pod on pod death When a SLURM dispatcher pod dies mid-run, its in-pod cleanup can no longer reach the login node, so the SLURM job and any Jenkins agent node leak (every "Clean Up Slurm Resource" retry fails with AgentOfflineException against the dead pod, right up to End of Pipeline). Add a build-scoped registry (keyed by stageName) of live SLURM resources: the pod wrapper records the dispatcher pod spec, the stage body records the SLURM job / Jenkins node identity, and the normal cleanup path deregisters once the resources are actually torn down. Only serializable primitives are stored (the SlurmCluster is rebuilt from clusterName) so pipeline persistence is unaffected. On a dispatcher-pod death (runnerStarted, isDispatcherPodFailure) the pod wrapper -- now back on the parent context -- launches a fresh cleanup pod and runs the normal scancel/workspace-clean/destroyNode off-pod. A post-build sweep backstops anything the in-catch finalize missed or failed on. Best-effort throughout: a finalizer failure is logged and never masks the stage failure. Together with the fail-closed change, this stops the leak the retry-on-a-dead- pod behavior left behind. Runtime paths (fresh-pod launch inside the catch / post-build finally, @Field registry) need CI validation. Signed-off-by: Derek Pitman --- jenkins/L0_Test.groovy | 145 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index e392ef1afbbb..6e6dba840c18 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -634,6 +634,102 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName, } } +// ---- Off-pod SLURM resource reconciliation -------------------------------- +// A SLURM stage runs inside a K8s dispatcher pod that ssh-drives the job on the +// login node. If that pod dies mid-run (eviction, container error, agent +// offline), the in-pod cleanup can no longer reach the controller, so the SLURM +// job and any Jenkins agent node leak. slurmResourceRegistry records the live +// resources per stage -- the dispatcher pod spec (from the pod wrapper) and the +// SLURM job / Jenkins node identity (from the stage body). The normal cleanup +// path and the finalizer remove an entry once its resources are actually torn +// down; whatever is left is reconciled by running the cleanup from a *fresh* +// pod. Only serializable primitives are stored (no SlurmCluster/Throwable) so +// pipeline persistence is unaffected; the cluster is rebuilt from clusterName. +// Keyed by stageName: dispatcher-pod deaths are no longer retried in place (see +// isDispatcherPodFailure), so a stage has at most one orphaned attempt. +@Field def slurmResourceRegistry = new java.util.concurrent.ConcurrentHashMap() + +void registerSlurmResource(String stageName, Map fields) { + if (!stageName) { + return + } + def entry = slurmResourceRegistry.get(stageName) + if (entry == null) { + entry = new java.util.concurrent.ConcurrentHashMap() + slurmResourceRegistry.put(stageName, entry) + } + // ConcurrentHashMap rejects null values; skip absent fields. Writers for a + // given stage run sequentially (pod wrapper, then stage body), so no merge race. + fields.each { k, v -> if (v != null) { entry.put(k, v) } } +} + +void deregisterSlurmResource(String stageName) { + if (stageName) { + slurmResourceRegistry.remove(stageName) + } +} + +// Reconcile one orphaned SLURM entry off the (dead) dispatcher pod: launch a +// fresh short-lived pod and run the normal cleanup from there (scancel the job, +// clean the workspace, and -- agent path -- drop the leaked Jenkins node), then +// deregister. Best-effort: a finalizer failure is logged and the entry is left +// for the post-build sweep to retry; it never masks the stage's own failure. +def finalizeSlurmResourceEntry(pipeline, String stageName, def entry, def podSpecOverride = null) { + if (entry == null) { + return + } + // Pod died before any job/node was provisioned: nothing to reconcile. + if (!entry.jobUID && !entry.nodeName) { + deregisterSlurmResource(stageName) + return + } + def podSpec = podSpecOverride ?: entry.podSpec + def cluster = entry.clusterName ? SlurmConfig.clusterConfig[entry.clusterName] : null + if (!podSpec || !cluster) { + echo "[SLURM-FINALIZER] ${stageName}: cannot reconcile off-pod (missing pod spec or unknown cluster " + + "'${entry.clusterName}'); SLURM job=${entry.slurmJobId ?: entry.jobUID ?: 'unknown'} " + + "node=${entry.nodeName ?: 'n/a'} may need manual cleanup." + return + } + try { + echo "[SLURM-FINALIZER] ${stageName}: reconciling orphaned SLURM resources off-pod " + + "(job=${entry.slurmJobId ?: entry.jobUID}, node=${entry.nodeName ?: 'n/a'})." + trtllm_utils.launchKubernetesPod(pipeline, podSpec, entry.containerName ?: "trt-llm", { + if (entry.usedSbatch) { + cleanUpSlurmResources(pipeline, cluster, entry.clusterName, entry.jobUID) + } else { + cleanUpNodeResources(pipeline, cluster, entry.clusterName, entry.nodeName, entry.slurmJobId) + } + }) + deregisterSlurmResource(stageName) + echo "[SLURM-FINALIZER] ${stageName}: off-pod reconciliation complete." + } catch (Exception e) { + echo "[SLURM-FINALIZER] ${stageName}: off-pod reconciliation failed (${e.toString()}); leaving entry for post-build sweep." + } +} + +def finalizeOrphanedSlurmResource(pipeline, String stageName, def podSpecOverride = null) { + finalizeSlurmResourceEntry(pipeline, stageName, slurmResourceRegistry.get(stageName), podSpecOverride) +} + +// Post-build backstop: reconcile any SLURM resources still registered at the end +// of the build (a dispatcher-pod death whose in-catch finalize also failed, or a +// failure mode the catch never saw). Runs off-pod from a fresh cleanup pod. +def sweepOrphanedSlurmResources(pipeline) { + def orphans = slurmResourceRegistry.keySet().collect { it }.findAll { + def e = slurmResourceRegistry.get(it) + e != null && (e.jobUID || e.nodeName) + } + if (orphans.isEmpty()) { + return + } + echo "[SLURM-FINALIZER] post-build sweep: ${orphans.size()} SLURM resource(s) were not cleaned up " + + "in-stage; reconciling off-pod: ${orphans.join(', ')}" + orphans.each { sName -> + finalizeSlurmResourceEntry(pipeline, sName, slurmResourceRegistry.get(sName)) + } +} + // Authoritative timeout signal: ask the SLURM controller (via sacct on the // cluster login node) for a job's terminal state. Returns the uppercased // primary state token -- e.g. "TIMEOUT", "COMPLETED", "FAILED", "NODE_FAIL", @@ -745,6 +841,11 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, slurmJobID = jobIDs ? jobIDs[-1] : null + // Record the live SLURM job + Jenkins node so a dispatcher-pod death + // can be reconciled off-pod (the in-pod cleanup can't reach the login + // node once the pod is gone). Deregistered when cleanup actually runs. + registerSlurmResource(stageName, [clusterName: partition.clusterName, nodeName: nodeName, slurmJobId: slurmJobID, usedSbatch: false]) + if (!slurmJobID || !slurmJobID.isNumber()) { echo "Slurm job did not submit successfully. No job ID found.\nSubmission output:\n${slurmSubmitOutput}" } @@ -986,6 +1087,9 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, } } } + // Cleanup ran on the live pod; drop the registry entry so the off-pod + // finalizer/sweep does not reconcile already-freed resources. + deregisterSlurmResource(stageName) } } } @@ -1668,6 +1772,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG recordSlurmPlacementContext(placementContext, slurmJobId, null, stageName) } Utils.exec(pipeline, script: "echo Slurm job ID: ${slurmJobId}") + // Record the live SLURM job so a dispatcher-pod death can be reconciled + // off-pod (the in-pod cleanup can't reach the login node once the pod is + // gone). Deregistered when cleanup actually runs. + registerSlurmResource(stageName, [clusterName: partition.clusterName, jobUID: jobUID, slurmJobId: slurmJobId, usedSbatch: true]) def scriptTrack = """#!/bin/bash set -xEeuo pipefail @@ -1810,6 +1918,9 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG } } } + // Cleanup ran on the live pod; drop the registry entry so the off-pod + // finalizer/sweep does not reconcile already-freed resources. + deregisterSlurmResource(stageName) } } } @@ -4456,6 +4567,9 @@ def runInKubernetes(pipeline, podSpec, containerName) def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerName, String stageName, Closure runner) { boolean singleAttempt = opts.singleAttempt ?: false + // SLURM dispatcher pods opt in to off-pod resource reconciliation: on a + // mid-run pod death their SLURM job / Jenkins node would otherwise leak. + boolean slurmDispatcher = opts.slurmDispatcher ?: false // DEBUG_MODE preserves the existing 2-hour-input human-inspection workflow // inside runLLMTestlistOnPlatform's finallyRunner: a single attempt only. @@ -4483,6 +4597,11 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN echo "[INFRA-RETRY] ${stageName}: relaunching pod (attempt ${launchAttempt}), avoiding prior host node(s): ${avoidedKubernetesHostNodes.join(', ')}" } def attemptPodSpec = trtllm_utils.withKubernetesHostNodeExclusion(podSpec, avoidedKubernetesHostNodes) + if (slurmDispatcher) { + // Record the dispatcher pod spec so the off-pod finalizer/sweep can + // launch a fresh cleanup pod if this pod dies mid-run. + registerSlurmResource(stageName, [podSpec: attemptPodSpec, containerName: containerName]) + } trtllm_utils.launchKubernetesPodWithPlacement(pipeline, attemptPodSpec, containerName, attemptPlacementContext, { attemptPlacementContext.runnerStarted = true runner("", true, null) @@ -4492,8 +4611,15 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN throw e } catch (Exception e) { // Once the runner has started, this is an execution failure the - // inner retry owns -- honor singleAttempt and do not re-run. + // inner retry owns -- honor singleAttempt and do not re-run. But if + // the dispatcher pod itself died mid-run, its in-pod cleanup could not + // reach the login node, leaking the SLURM job / Jenkins node. We are + // back on the parent context here, so reconcile them from a fresh + // cleanup pod before failing closed. if (attemptPlacementContext.runnerStarted) { + if (slurmDispatcher && isDispatcherPodFailure(e)) { + finalizeOrphanedSlurmResource(pipeline, stageName) + } throw e } def c = FailureClassifier.classify(e, InfraFailure.K8S) @@ -4826,7 +4952,7 @@ def launchTestJobs(pipeline, testFilter) config = LLVM_CONFIG } runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag) - }, [singleAttempt: true]]]} + }, [singleAttempt: true, slurmDispatcher: true]]]} // SLURM dispatcher pods run their own inner retry loop // (runLLMTestlistOnSlurm with SLURM_INFRA_RETRY_MAX). Disabling the outer // K8s pod retry (singleAttempt:true) here caps total attempts at @@ -5097,7 +5223,7 @@ def launchTestJobs(pipeline, testFilter) config = LLVM_CONFIG } runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false) - }, [singleAttempt: true]]]} + }, [singleAttempt: true, slurmDispatcher: true]]]} parallelJobs += parallelSlurmJobs // Add SBSA multi node Slurm jobs @@ -5111,7 +5237,7 @@ def launchTestJobs(pipeline, testFilter) config = LLVM_CONFIG } runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 2, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false) - }, [singleAttempt: true]]]} + }, [singleAttempt: true, slurmDispatcher: true]]]} parallelJobs += parallelMultiNodesSBSAJobs } @@ -5694,6 +5820,7 @@ pipeline { stage("Test") { steps { script { + try { if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) { parallelJobs = launchTestJobsForImagesSanityCheck(this, globalVars) } else { @@ -5754,6 +5881,16 @@ pipeline { } } } + } finally { + // Backstop: reclaim any SLURM job / Jenkins node left orphaned + // by a dispatcher-pod death whose in-catch finalize did not run + // or failed. Best-effort; never fails the build. + try { + sweepOrphanedSlurmResources(this) + } catch (Exception sweepErr) { + echo "[SLURM-FINALIZER] post-build sweep error: ${sweepErr}" + } + } } } } // Test stage From 5c41011c22072b8c83c4864c4d427f23bd6e7169 Mon Sep 17 00:00:00 2001 From: Derek Pitman Date: Wed, 15 Jul 2026 15:43:05 -0400 Subject: [PATCH 3/4] [TRTLLMINF-81][test] Fix SLURM registry sandbox rejection and podSpec wipe Two fixes to the off-pod reconciliation registry: - The Jenkins script sandbox forbids `new java.util.concurrent.ConcurrentHashMap` ("Scripts not permitted to use ..."), which failed the pipeline at load. Use plain map literals: pipeline Groovy runs single-threaded under CPS (parallel branches interleave at step boundaries, never execute Groovy concurrently) and each stage writes its own key, so there is no concurrent map corruption to guard against. - deregisterSlurmResource removed the whole entry, but podSpec is registered only once (before the inner SLURM retry loop). After the first attempt's cleanup deregistered, a later attempt's dispatcher-pod death had no podSpec to launch a cleanup pod with, and the post-build sweep could not recover it either. Deregister now clears only the per-attempt job/node identity and keeps the stage-level podSpec, so both the in-catch finalize and the sweep can always reconcile. An entry left with only a podSpec is inert (finalize and the sweep skip entries with no job/node). Signed-off-by: Derek Pitman --- jenkins/L0_Test.groovy | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 6e6dba840c18..ab775cdced76 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -640,14 +640,18 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName, // offline), the in-pod cleanup can no longer reach the controller, so the SLURM // job and any Jenkins agent node leak. slurmResourceRegistry records the live // resources per stage -- the dispatcher pod spec (from the pod wrapper) and the -// SLURM job / Jenkins node identity (from the stage body). The normal cleanup -// path and the finalizer remove an entry once its resources are actually torn -// down; whatever is left is reconciled by running the cleanup from a *fresh* -// pod. Only serializable primitives are stored (no SlurmCluster/Throwable) so -// pipeline persistence is unaffected; the cluster is rebuilt from clusterName. -// Keyed by stageName: dispatcher-pod deaths are no longer retried in place (see -// isDispatcherPodFailure), so a stage has at most one orphaned attempt. -@Field def slurmResourceRegistry = new java.util.concurrent.ConcurrentHashMap() +// SLURM job / Jenkins node identity (from the stage body, re-registered per +// inner attempt). Deregistering after a successful cleanup clears only the +// per-attempt job/node identity and keeps the stage-level podSpec, so a *later* +// attempt's dispatcher-pod death can still be reconciled off-pod (the pod spec +// is registered once, before the inner retry loop). Only serializable +// primitives are stored (no SlurmCluster/Throwable) so pipeline persistence is +// unaffected; the cluster is rebuilt from clusterName. Plain maps: pipeline +// Groovy runs single-threaded under CPS (parallel branches interleave at step +// boundaries, never execute Groovy concurrently) and each stage writes its own +// key, so no concurrent map corruption -- and the Jenkins script sandbox forbids +// `new ConcurrentHashMap`. Keyed by stageName. +@Field def slurmResourceRegistry = [:] void registerSlurmResource(String stageName, Map fields) { if (!stageName) { @@ -655,17 +659,25 @@ void registerSlurmResource(String stageName, Map fields) { } def entry = slurmResourceRegistry.get(stageName) if (entry == null) { - entry = new java.util.concurrent.ConcurrentHashMap() + entry = [:] slurmResourceRegistry.put(stageName, entry) } - // ConcurrentHashMap rejects null values; skip absent fields. Writers for a - // given stage run sequentially (pod wrapper, then stage body), so no merge race. + // Skip absent fields. Writers for a given stage run sequentially (pod + // wrapper, then stage body), so no merge race. fields.each { k, v -> if (v != null) { entry.put(k, v) } } } +// Called once an attempt's resources are actually torn down: drop the per-attempt +// job/node identity but keep the stage's podSpec (needed to launch a cleanup pod +// for a later attempt). An entry left with only a podSpec is inert -- finalize +// and the post-build sweep both skip entries with no job/node. void deregisterSlurmResource(String stageName) { - if (stageName) { - slurmResourceRegistry.remove(stageName) + if (!stageName) { + return + } + def entry = slurmResourceRegistry.get(stageName) + if (entry != null) { + ["clusterName", "jobUID", "nodeName", "slurmJobId", "usedSbatch"].each { entry.remove(it) } } } From b675f873072bdc3a715a0e6d19eaf50a6229d81d Mon Sep 17 00:00:00 2001 From: Derek Pitman Date: Tue, 21 Jul 2026 14:42:55 -0400 Subject: [PATCH 4/4] [TRTLLMINF-81][test] Address review: pass podSpec to finalizer, fix indentation - runLLMTestlistOnSlurm dispatcher-pod-failure branch now passes the in-scope attemptPodSpec to finalizeOrphanedSlurmResource, so the off-pod finalizer never depends on the registry entry's podSpec surviving a prior attempt's deregister (CodeRabbit). - Re-indent the Test-stage body wrapped by the post-build sweep try/finally (chzblych). Signed-off-by: Derek Pitman --- jenkins/L0_Test.groovy | 107 +++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 6404da9f9a8a..c3dc42a89197 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4789,7 +4789,10 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN // cleanup pod before failing closed. if (attemptPlacementContext.runnerStarted) { if (slurmDispatcher && isDispatcherPodFailure(e)) { - finalizeOrphanedSlurmResource(pipeline, stageName) + // Pass this attempt's pod spec explicitly so the finalizer + // never depends on the registry entry's podSpec surviving a + // prior attempt's deregister. + finalizeOrphanedSlurmResource(pipeline, stageName, attemptPodSpec) } throw e } @@ -6026,66 +6029,66 @@ pipeline { steps { script { try { - if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) { - parallelJobs = launchTestJobsForImagesSanityCheck(this, globalVars) - } else { - parallelJobs = launchTestJobs(this, testFilter) - } + if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) { + parallelJobs = launchTestJobsForImagesSanityCheck(this, globalVars) + } else { + parallelJobs = launchTestJobs(this, testFilter) + } - singleGpuJobs = parallelJobs - dgxJobs = [:] + singleGpuJobs = parallelJobs + dgxJobs = [:] - def testPhase2StageName = env.testPhase2StageName - if (testPhase2StageName) { - def multiGpuPattern = /\d+_GPUs/ - singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} - dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} - } + def testPhase2StageName = env.testPhase2StageName + if (testPhase2StageName) { + def multiGpuPattern = /\d+_GPUs/ + singleGpuJobs = parallelJobs.findAll{!(it.key =~ multiGpuPattern)} + dgxJobs = parallelJobs.findAll{it.key =~ multiGpuPattern} + } - if (env.JOB_NAME ==~ /.*Single-GPU.*/) { - echo "Only run single-GPU tests." - if (dgxJobs.size() > 0) { - if (globalVars[ACTION_INFO]['parents'].size() > 0) { - // We add a special marker to the parent job's description. - // This will be used to decide whether to run multi-GPU test stage. - def parentJob = globalVars[ACTION_INFO]['parents'][-2] - def archStr = (env.targetArch == X86_64_TRIPLE) ? "x86_64" : (env.targetArch == AARCH64_TRIPLE ? "SBSA" : "Unknown") - trtllm_utils.appendBuildDescription(this, parentJob['name'], parentJob['build_number'], "====Require ${archStr} Multi-GPU Testing====
") + if (env.JOB_NAME ==~ /.*Single-GPU.*/) { + echo "Only run single-GPU tests." + if (dgxJobs.size() > 0) { + if (globalVars[ACTION_INFO]['parents'].size() > 0) { + // We add a special marker to the parent job's description. + // This will be used to decide whether to run multi-GPU test stage. + def parentJob = globalVars[ACTION_INFO]['parents'][-2] + def archStr = (env.targetArch == X86_64_TRIPLE) ? "x86_64" : (env.targetArch == AARCH64_TRIPLE ? "SBSA" : "Unknown") + trtllm_utils.appendBuildDescription(this, parentJob['name'], parentJob['build_number'], "====Require ${archStr} Multi-GPU Testing====
") + } else { + echo "No parent job found to add the special marker for executing multi-GPU test stage." + } } else { - echo "No parent job found to add the special marker for executing multi-GPU test stage." + echo "Skip multi-GPU testing. No test to run." } - } else { - echo "Skip multi-GPU testing. No test to run." - } - if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs - } else { - echo "Skip single-GPU testing. No test to run." - } - } else if (env.JOB_NAME ==~ /.*Multi-GPU.*/) { - echo "Only run multi-GPU tests." - if (dgxJobs.size() > 0) { - dgxJobs.failFast = params.enableFailFast - parallel dgxJobs - } else { - error "Skip multi-GPU testing. No test to run." - } - } else { - if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs - } else { - echo "Skip single-GPU testing. No test to run." - } - - if (dgxJobs.size() > 0) { - stage(testPhase2StageName) { + if (singleGpuJobs.size() > 0) { + singleGpuJobs.failFast = params.enableFailFast + parallel singleGpuJobs + } else { + echo "Skip single-GPU testing. No test to run." + } + } else if (env.JOB_NAME ==~ /.*Multi-GPU.*/) { + echo "Only run multi-GPU tests." + if (dgxJobs.size() > 0) { dgxJobs.failFast = params.enableFailFast parallel dgxJobs + } else { + error "Skip multi-GPU testing. No test to run." + } + } else { + if (singleGpuJobs.size() > 0) { + singleGpuJobs.failFast = params.enableFailFast + parallel singleGpuJobs + } else { + echo "Skip single-GPU testing. No test to run." + } + + if (dgxJobs.size() > 0) { + stage(testPhase2StageName) { + dgxJobs.failFast = params.enableFailFast + parallel dgxJobs + } } } - } } finally { // Backstop: reclaim any SLURM job / Jenkins node left orphaned // by a dispatcher-pod death whose in-catch finalize did not run