From 27fb5cbf080f5e0cf59d727bbf1b43d34d09f0d9 Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Wed, 26 Mar 2025 04:15:43 -0700 Subject: [PATCH 1/8] Support get file change for github PR Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 56 ++++++++++++++++++++++++++++++---- jenkins/L0_Test.groovy | 1 + 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 7c2ae4b72fcf..027dc45f24e7 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -107,6 +107,8 @@ def testFilter = [ String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) +String githubPrApiUrl = gitlabParamsFromBot.get('github_pr_api_url', null) + // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. boolean enableUpdateGitlabStatus = !testFilter[ENABLE_SKIP_TEST] && @@ -276,7 +278,7 @@ def echoNodeAndGpuInfo(pipeline, stageName) pipeline.echo "HOST_NODE_NAME = ${hostNodeName} ; GPU_UUIDS = ${gpuUuids} ; STAGE_NAME = ${stageName}" } -def setupPipelineEnvironment(pipeline, testFilter) +def setupPipelineEnvironment(pipeline, testFilter, githubPrApiUrl) { setupPipelineSpec = createKubernetesPodConfig(LLM_DOCKER_IMAGE, "build") trtllm_utils.launchKubernetesPod(pipeline, setupPipelineSpec, "trt-llm", { @@ -294,7 +296,7 @@ def setupPipelineEnvironment(pipeline, testFilter) } echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter) + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) }) } @@ -364,7 +366,7 @@ def launchReleaseCheck(pipeline) }) } -def getMergeRequestChangedFileList(pipeline) { +def getMergeRequestChangedFileListGitlab(pipeline) { def changedFileList = [] def pageId = 0 withCredentials([ @@ -393,7 +395,49 @@ def getMergeRequestChangedFileList(pipeline) { return changedFileList } -def getMultiGpuFileChanged(pipeline, testFilter) +def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { + def changedFileList = [] + def pageId = 0 + withCredentials([ + string( + credentialsId: 'github-token-trtllm-ci', + variable: 'GITHUB_API_TOKEN' + ), + ]) { + while(true) { + pageId += 1 + def rawDataJson = pipeline.sh( + script: "curl --header \"Authorization: Bearer $GITHUB_API_TOKEN\" --url \"${githubPrApiUrl}/files?page=${pageId}&per_page=20\"", + returnStdout: true + ) + echo "rawDataJson: ${rawDataJson}" + def rawDataList = readJSON text: rawDataJson, returnPojo: true + rawDataList.each { rawData -> + changedFileList += [rawData.get("filename"), rawData.get("previous_filename")].findAll { it } + } + if (!rawDataList) { break } + } + } + def changedFileListStr = changedFileList.join(",\n") + pipeline.echo("The changeset of this PR is: ${changedFileListStr}.") + return changedFileList +} + +def getMergeRequestChangedFileList(pipeline, githubPrApiUrl) { + try { + if (githubPrApiUrl != null) { + return getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) + } + return getMergeRequestChangedFileListGitlab(pipeline) + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("Get merge request changed file list failed.") + return [] + } +} + +def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) { if (testFilter[(DISABLE_MULTI_GPU_TEST)]) { pipeline.echo("Force not run multi-GPU testing.") @@ -459,7 +503,7 @@ def getMultiGpuFileChanged(pipeline, testFilter) def changedFileList = "," def relatedFileChanged = false try { - changedFileList = getMergeRequestChangedFileList(pipeline).join(", ") + changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl).join(", ") relatedFileChanged = relatedFileList.any { it -> if (changedFileList.contains(it)) { return true @@ -829,7 +873,7 @@ pipeline { steps { script { - setupPipelineEnvironment(this, testFilter) + setupPipelineEnvironment(this, testFilter, githubPrApiUrl) echo "enableFailFast is: ${enableFailFast}" echo "env.gitlabTriggerPhrase is: ${env.gitlabTriggerPhrase}" println testFilter diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 324715b5a2df..aca2ac082f64 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -91,6 +91,7 @@ def trimForStageList(stageNameList) return trimedList } +// Test filter flags @Field def REUSE_STAGE_LIST = "reuse_stage_list" @Field From 019e9a34e3ac1334b26b419f66e8ee591fbc87f9 Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:22:39 +0000 Subject: [PATCH 2/8] Fix reviews Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 027dc45f24e7..792f0cb9a36d 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -380,7 +380,10 @@ def getMergeRequestChangedFileListGitlab(pipeline) { while(true) { pageId += 1 def rawDataJson = pipeline.sh( - script: "curl --header \"PRIVATE-TOKEN: $GITLAB_API_TOKEN\" --url \"https://${DEFAULT_GIT_URL}/api/v4/projects/${env.gitlabMergeRequestTargetProjectId}/merge_requests/${env.gitlabMergeRequestIid}/diffs?page=${pageId}&per_page=20\"", + script: """ + curl --header "PRIVATE-TOKEN: $GITLAB_API_TOKEN" \ + --url "https://${DEFAULT_GIT_URL}/api/v4/projects/${env.gitlabMergeRequestTargetProjectId}/merge_requests/${env.gitlabMergeRequestIid}/diffs?page=${pageId}&per_page=20" + """, returnStdout: true ) def rawDataList = readJSON text: rawDataJson, returnPojo: true @@ -407,7 +410,10 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { while(true) { pageId += 1 def rawDataJson = pipeline.sh( - script: "curl --header \"Authorization: Bearer $GITHUB_API_TOKEN\" --url \"${githubPrApiUrl}/files?page=${pageId}&per_page=20\"", + script: """ + curl --header "Authorization: Bearer $GITHUB_API_TOKEN" \ + --url "${githubPrApiUrl}/files?page=${pageId}&per_page=20" + """, returnStdout: true ) echo "rawDataJson: ${rawDataJson}" @@ -432,7 +438,7 @@ def getMergeRequestChangedFileList(pipeline, githubPrApiUrl) { } catch (InterruptedException e) { throw e } catch (Exception e) { - pipeline.echo("Get merge request changed file list failed.") + pipeline.echo("Get merge request changed file list failed. Error: ${e.toString()}") return [] } } @@ -516,7 +522,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) } catch (Exception e) { - pipeline.echo("getMultiGpuFileChanged failed execution.") + pipeline.echo("getMultiGpuFileChanged failed execution. Error: ${e.toString()}") } if (relatedFileChanged) { pipeline.echo("Detect multi-GPU related files changed.") From f132b17d409d76aa5e6352e7f6fae85205f406db Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Sun, 30 Mar 2025 23:44:24 -0700 Subject: [PATCH 3/8] Check if only pytorch related file changed Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 55 ++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 792f0cb9a36d..16781f3e5f54 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -92,6 +92,9 @@ def DISABLE_MULTI_GPU_TEST = "disable_multi_gpu_test" def EXTRA_STAGE_LIST = "extra_stage" @Field def MULTI_GPU_FILE_CHANGED = "multi_gpu_file_changed" +@Field +def ONLY_PYTORCH_FILE_CHANGED = "only_pytorch_file_changed" + def testFilter = [ (REUSE_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get(REUSE_STAGE_LIST, null)?.tokenize(',')), (ENABLE_SKIP_TEST): gitlabParamsFromBot.get((ENABLE_SKIP_TEST), false), @@ -103,6 +106,7 @@ def testFilter = [ (DISABLE_MULTI_GPU_TEST): gitlabParamsFromBot.get((DISABLE_MULTI_GPU_TEST), false), (EXTRA_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get((EXTRA_STAGE_LIST), null)?.tokenize(',')), (MULTI_GPU_FILE_CHANGED): false, + (ONLY_PYTORCH_FILE_CHANGED): false, ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @@ -297,6 +301,7 @@ def setupPipelineEnvironment(pipeline, testFilter, githubPrApiUrl) echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) + testFilter[(ONLY_PYTORCH_FILE_CHANGED)] = getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl) }) } @@ -429,17 +434,25 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { return changedFileList } +def cachedChangedFileList = null + def getMergeRequestChangedFileList(pipeline, githubPrApiUrl) { + if (cachedChangedFileList != null) { + return cachedChangedFileList + } try { if (githubPrApiUrl != null) { - return getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) + cachedChangedFileList = getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) + } else { + cachedChangedFileList = getMergeRequestChangedFileListGitlab(pipeline) } - return getMergeRequestChangedFileListGitlab(pipeline) + return cachedChangedFileList } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("Get merge request changed file list failed. Error: ${e.toString()}") - return [] + cachedChangedFileList = [] + return cachedChangedFileList } } @@ -506,12 +519,16 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) "jenkins/L0_Test.groovy", ] - def changedFileList = "," + if (!changedFileList) { + return false + } + + def changedFileListStr = "," def relatedFileChanged = false try { - changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl).join(", ") + changedFileListStr = getMergeRequestChangedFileList(pipeline, githubPrApiUrl).join(", ") relatedFileChanged = relatedFileList.any { it -> - if (changedFileList.contains(it)) { + if (changedFileListStr.contains(it)) { return true } } @@ -530,6 +547,32 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) return relatedFileChanged } +def getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl) { + def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) + if (env.alternativeTRT || isOfficialPostMergeJob) { + pipeline.echo("Force set ONLY_PYTORCH_FILE_CHANGED false.") + return false + } + def pytorchOnlyPattern = ~/^tensorrt_llm\/_torch\/.*/ + + def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl) + + if (!changedFileList || changedFileList.isEmpty()) { + return false + } + + def result = changedFileList.every { file -> + def isPytorchFile = file =~ pytorchOnlyPattern + if (!isPytorchFile) { + pipeline.echo("Found non-pytorch file: ${file}") + } + return isPytorchFile + } + + pipeline.echo("Only PyTorch files changed: ${result}") + return result +} + def collectTestResults(pipeline, testFilter) { collectResultPodSpec = createKubernetesPodConfig("", "agent") From 67565f6e81638ecbeaaa92ef7f8ae382bb420d78 Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 31 Mar 2025 18:47:29 -0700 Subject: [PATCH 4/8] Fix Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 16781f3e5f54..072c70998859 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -519,6 +519,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) "jenkins/L0_Test.groovy", ] + def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl) if (!changedFileList) { return false } @@ -526,7 +527,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) def changedFileListStr = "," def relatedFileChanged = false try { - changedFileListStr = getMergeRequestChangedFileList(pipeline, githubPrApiUrl).join(", ") + changedFileListStr = changedFileList.join(", ") relatedFileChanged = relatedFileList.any { it -> if (changedFileListStr.contains(it)) { return true From ef2f3ac86bbbcc3cc3926ce73c8c3e56954250c0 Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 31 Mar 2025 19:06:41 -0700 Subject: [PATCH 5/8] Fix global var Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 072c70998859..22e0832b5efb 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -113,6 +113,8 @@ String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) String githubPrApiUrl = gitlabParamsFromBot.get('github_pr_api_url', null) +def cachedChangedFileList = null + // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. boolean enableUpdateGitlabStatus = !testFilter[ENABLE_SKIP_TEST] && @@ -282,7 +284,7 @@ def echoNodeAndGpuInfo(pipeline, stageName) pipeline.echo "HOST_NODE_NAME = ${hostNodeName} ; GPU_UUIDS = ${gpuUuids} ; STAGE_NAME = ${stageName}" } -def setupPipelineEnvironment(pipeline, testFilter, githubPrApiUrl) +def setupPipelineEnvironment(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) { setupPipelineSpec = createKubernetesPodConfig(LLM_DOCKER_IMAGE, "build") trtllm_utils.launchKubernetesPod(pipeline, setupPipelineSpec, "trt-llm", { @@ -300,8 +302,8 @@ def setupPipelineEnvironment(pipeline, testFilter, githubPrApiUrl) } echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) - testFilter[(ONLY_PYTORCH_FILE_CHANGED)] = getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl) + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) + testFilter[(ONLY_PYTORCH_FILE_CHANGED)] = getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) }) } @@ -434,9 +436,7 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { return changedFileList } -def cachedChangedFileList = null - -def getMergeRequestChangedFileList(pipeline, githubPrApiUrl) { +def getMergeRequestChangedFileList(pipeline, githubPrApiUrl, cachedChangedFileList) { if (cachedChangedFileList != null) { return cachedChangedFileList } @@ -456,7 +456,7 @@ def getMergeRequestChangedFileList(pipeline, githubPrApiUrl) { } } -def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) +def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) { if (testFilter[(DISABLE_MULTI_GPU_TEST)]) { pipeline.echo("Force not run multi-GPU testing.") @@ -519,7 +519,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) "jenkins/L0_Test.groovy", ] - def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl) + def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl, cachedChangedFileList) if (!changedFileList) { return false } @@ -548,7 +548,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl) return relatedFileChanged } -def getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl) { +def getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if (env.alternativeTRT || isOfficialPostMergeJob) { pipeline.echo("Force set ONLY_PYTORCH_FILE_CHANGED false.") @@ -556,7 +556,7 @@ def getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl) { } def pytorchOnlyPattern = ~/^tensorrt_llm\/_torch\/.*/ - def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl) + def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl, cachedChangedFileList) if (!changedFileList || changedFileList.isEmpty()) { return false @@ -923,7 +923,7 @@ pipeline { steps { script { - setupPipelineEnvironment(this, testFilter, githubPrApiUrl) + setupPipelineEnvironment(this, testFilter, githubPrApiUrl, cachedChangedFileList) echo "enableFailFast is: ${enableFailFast}" echo "env.gitlabTriggerPhrase is: ${env.gitlabTriggerPhrase}" println testFilter From 56314a5f76d34a1a534ed8f6d91a6746ca7d5ecb Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 31 Mar 2025 19:57:12 -0700 Subject: [PATCH 6/8] Use globalVars for global values Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 22e0832b5efb..fc42df82db64 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -111,9 +111,14 @@ def testFilter = [ String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) -String githubPrApiUrl = gitlabParamsFromBot.get('github_pr_api_url', null) - -def cachedChangedFileList = null +@Field +def GITLAB_PR_API_URL = "gitlab_pr_api_url" +@Field +def CACHED_CHANGED_FILE_LIST = "cached_changed_file_list" +def globalVars = [ + (GITLAB_PR_API_URL): gitlabParamsFromBot.get('github_pr_api_url', null), + (CACHED_CHANGED_FILE_LIST): null, +] // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. boolean enableUpdateGitlabStatus = @@ -284,7 +289,7 @@ def echoNodeAndGpuInfo(pipeline, stageName) pipeline.echo "HOST_NODE_NAME = ${hostNodeName} ; GPU_UUIDS = ${gpuUuids} ; STAGE_NAME = ${stageName}" } -def setupPipelineEnvironment(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) +def setupPipelineEnvironment(pipeline, testFilter, globalVars) { setupPipelineSpec = createKubernetesPodConfig(LLM_DOCKER_IMAGE, "build") trtllm_utils.launchKubernetesPod(pipeline, setupPipelineSpec, "trt-llm", { @@ -302,8 +307,8 @@ def setupPipelineEnvironment(pipeline, testFilter, githubPrApiUrl, cachedChanged } echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) - testFilter[(ONLY_PYTORCH_FILE_CHANGED)] = getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) + testFilter[(ONLY_PYTORCH_FILE_CHANGED)] = getOnlyPytorchFileChanged(pipeline, testFilter, globalVars) }) } @@ -436,7 +441,10 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { return changedFileList } -def getMergeRequestChangedFileList(pipeline, githubPrApiUrl, cachedChangedFileList) { +def getMergeRequestChangedFileList(pipeline, globalVars) { + def githubPrApiUrl = globalVars[GITLAB_PR_API_URL] + def cachedChangedFileList = globalVars[CACHED_CHANGED_FILE_LIST] + if (cachedChangedFileList != null) { return cachedChangedFileList } @@ -456,7 +464,7 @@ def getMergeRequestChangedFileList(pipeline, githubPrApiUrl, cachedChangedFileLi } } -def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) +def getMultiGpuFileChanged(pipeline, testFilter, globalVars) { if (testFilter[(DISABLE_MULTI_GPU_TEST)]) { pipeline.echo("Force not run multi-GPU testing.") @@ -519,7 +527,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFi "jenkins/L0_Test.groovy", ] - def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl, cachedChangedFileList) + def changedFileList = getMergeRequestChangedFileList(pipeline, globalVars) if (!changedFileList) { return false } @@ -548,7 +556,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFi return relatedFileChanged } -def getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChangedFileList) { +def getOnlyPytorchFileChanged(pipeline, testFilter, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) if (env.alternativeTRT || isOfficialPostMergeJob) { pipeline.echo("Force set ONLY_PYTORCH_FILE_CHANGED false.") @@ -556,7 +564,7 @@ def getOnlyPytorchFileChanged(pipeline, testFilter, githubPrApiUrl, cachedChange } def pytorchOnlyPattern = ~/^tensorrt_llm\/_torch\/.*/ - def changedFileList = getMergeRequestChangedFileList(pipeline, githubPrApiUrl, cachedChangedFileList) + def changedFileList = getMergeRequestChangedFileList(pipeline, globalVars) if (!changedFileList || changedFileList.isEmpty()) { return false @@ -923,7 +931,7 @@ pipeline { steps { script { - setupPipelineEnvironment(this, testFilter, githubPrApiUrl, cachedChangedFileList) + setupPipelineEnvironment(this, testFilter, globalVars) echo "enableFailFast is: ${enableFailFast}" echo "env.gitlabTriggerPhrase is: ${env.gitlabTriggerPhrase}" println testFilter From 379b2e5567874f09f6cb1af78bd0a524c195fab9 Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Mon, 31 Mar 2025 20:17:27 -0700 Subject: [PATCH 7/8] Fix Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index fc42df82db64..859dd6cf2477 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -443,24 +443,23 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { def getMergeRequestChangedFileList(pipeline, globalVars) { def githubPrApiUrl = globalVars[GITLAB_PR_API_URL] - def cachedChangedFileList = globalVars[CACHED_CHANGED_FILE_LIST] - if (cachedChangedFileList != null) { - return cachedChangedFileList + if (globalVars[CACHED_CHANGED_FILE_LIST] != null) { + return globalVars[CACHED_CHANGED_FILE_LIST] } try { if (githubPrApiUrl != null) { - cachedChangedFileList = getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) + globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) } else { - cachedChangedFileList = getMergeRequestChangedFileListGitlab(pipeline) + globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGitlab(pipeline) } - return cachedChangedFileList + return globalVars[CACHED_CHANGED_FILE_LIST] } catch (InterruptedException e) { throw e } catch (Exception e) { pipeline.echo("Get merge request changed file list failed. Error: ${e.toString()}") - cachedChangedFileList = [] - return cachedChangedFileList + globalVars[CACHED_CHANGED_FILE_LIST] = [] + return globalVars[CACHED_CHANGED_FILE_LIST] } } From dd5a254138f9c8d91dc3f56bd27c72b16cada885 Mon Sep 17 00:00:00 2001 From: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> Date: Tue, 1 Apr 2025 19:12:57 -0700 Subject: [PATCH 8/8] Fix review Signed-off-by: ZhanruiSunCh <184402041+ZhanruiSunCh@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 859dd6cf2477..7f5b25d1d4b7 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -112,11 +112,11 @@ def testFilter = [ String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @Field -def GITLAB_PR_API_URL = "gitlab_pr_api_url" +def GITHUB_PR_API_URL = "github_pr_api_url" @Field def CACHED_CHANGED_FILE_LIST = "cached_changed_file_list" def globalVars = [ - (GITLAB_PR_API_URL): gitlabParamsFromBot.get('github_pr_api_url', null), + (GITHUB_PR_API_URL): gitlabParamsFromBot.get('github_pr_api_url', null), (CACHED_CHANGED_FILE_LIST): null, ] @@ -442,7 +442,7 @@ def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { } def getMergeRequestChangedFileList(pipeline, globalVars) { - def githubPrApiUrl = globalVars[GITLAB_PR_API_URL] + def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] if (globalVars[CACHED_CHANGED_FILE_LIST] != null) { return globalVars[CACHED_CHANGED_FILE_LIST] @@ -527,7 +527,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) ] def changedFileList = getMergeRequestChangedFileList(pipeline, globalVars) - if (!changedFileList) { + if (!changedFileList || changedFileList.isEmpty()) { return false } @@ -572,7 +572,7 @@ def getOnlyPytorchFileChanged(pipeline, testFilter, globalVars) { def result = changedFileList.every { file -> def isPytorchFile = file =~ pytorchOnlyPattern if (!isPytorchFile) { - pipeline.echo("Found non-pytorch file: ${file}") + pipeline.echo("Found non-PyTorch file: ${file}") } return isPytorchFile }