diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 7c2ae4b72fcf..7f5b25d1d4b7 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -92,6 +92,9 @@ def DISABLE_MULTI_GPU_TEST = "disable_multi_gpu_test" def EXTRA_STAGE_LIST = "extra_stage" @Field def MULTI_GPU_FILE_CHANGED = "multi_gpu_file_changed" +@Field +def ONLY_PYTORCH_FILE_CHANGED = "only_pytorch_file_changed" + def testFilter = [ (REUSE_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get(REUSE_STAGE_LIST, null)?.tokenize(',')), (ENABLE_SKIP_TEST): gitlabParamsFromBot.get((ENABLE_SKIP_TEST), false), @@ -103,10 +106,20 @@ def testFilter = [ (DISABLE_MULTI_GPU_TEST): gitlabParamsFromBot.get((DISABLE_MULTI_GPU_TEST), false), (EXTRA_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get((EXTRA_STAGE_LIST), null)?.tokenize(',')), (MULTI_GPU_FILE_CHANGED): false, + (ONLY_PYTORCH_FILE_CHANGED): false, ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) +@Field +def GITHUB_PR_API_URL = "github_pr_api_url" +@Field +def CACHED_CHANGED_FILE_LIST = "cached_changed_file_list" +def globalVars = [ + (GITHUB_PR_API_URL): gitlabParamsFromBot.get('github_pr_api_url', null), + (CACHED_CHANGED_FILE_LIST): null, +] + // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. boolean enableUpdateGitlabStatus = !testFilter[ENABLE_SKIP_TEST] && @@ -276,7 +289,7 @@ def echoNodeAndGpuInfo(pipeline, stageName) pipeline.echo "HOST_NODE_NAME = ${hostNodeName} ; GPU_UUIDS = ${gpuUuids} ; STAGE_NAME = ${stageName}" } -def setupPipelineEnvironment(pipeline, testFilter) +def setupPipelineEnvironment(pipeline, testFilter, globalVars) { setupPipelineSpec = createKubernetesPodConfig(LLM_DOCKER_IMAGE, "build") trtllm_utils.launchKubernetesPod(pipeline, setupPipelineSpec, "trt-llm", { @@ -294,7 +307,8 @@ def setupPipelineEnvironment(pipeline, testFilter) } echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter) + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) + testFilter[(ONLY_PYTORCH_FILE_CHANGED)] = getOnlyPytorchFileChanged(pipeline, testFilter, globalVars) }) } @@ -364,7 +378,7 @@ def launchReleaseCheck(pipeline) }) } -def getMergeRequestChangedFileList(pipeline) { +def getMergeRequestChangedFileListGitlab(pipeline) { def changedFileList = [] def pageId = 0 withCredentials([ @@ -378,7 +392,10 @@ def getMergeRequestChangedFileList(pipeline) { while(true) { pageId += 1 def rawDataJson = pipeline.sh( - script: "curl --header \"PRIVATE-TOKEN: $GITLAB_API_TOKEN\" --url \"https://${DEFAULT_GIT_URL}/api/v4/projects/${env.gitlabMergeRequestTargetProjectId}/merge_requests/${env.gitlabMergeRequestIid}/diffs?page=${pageId}&per_page=20\"", + script: """ + curl --header "PRIVATE-TOKEN: $GITLAB_API_TOKEN" \ + --url "https://${DEFAULT_GIT_URL}/api/v4/projects/${env.gitlabMergeRequestTargetProjectId}/merge_requests/${env.gitlabMergeRequestIid}/diffs?page=${pageId}&per_page=20" + """, returnStdout: true ) def rawDataList = readJSON text: rawDataJson, returnPojo: true @@ -393,7 +410,60 @@ def getMergeRequestChangedFileList(pipeline) { return changedFileList } -def getMultiGpuFileChanged(pipeline, testFilter) +def getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) { + def changedFileList = [] + def pageId = 0 + withCredentials([ + string( + credentialsId: 'github-token-trtllm-ci', + variable: 'GITHUB_API_TOKEN' + ), + ]) { + while(true) { + pageId += 1 + def rawDataJson = pipeline.sh( + script: """ + curl --header "Authorization: Bearer $GITHUB_API_TOKEN" \ + --url "${githubPrApiUrl}/files?page=${pageId}&per_page=20" + """, + returnStdout: true + ) + echo "rawDataJson: ${rawDataJson}" + def rawDataList = readJSON text: rawDataJson, returnPojo: true + rawDataList.each { rawData -> + changedFileList += [rawData.get("filename"), rawData.get("previous_filename")].findAll { it } + } + if (!rawDataList) { break } + } + } + def changedFileListStr = changedFileList.join(",\n") + pipeline.echo("The changeset of this PR is: ${changedFileListStr}.") + return changedFileList +} + +def getMergeRequestChangedFileList(pipeline, globalVars) { + def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] + + if (globalVars[CACHED_CHANGED_FILE_LIST] != null) { + return globalVars[CACHED_CHANGED_FILE_LIST] + } + try { + if (githubPrApiUrl != null) { + globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGithub(pipeline, githubPrApiUrl) + } else { + globalVars[CACHED_CHANGED_FILE_LIST] = getMergeRequestChangedFileListGitlab(pipeline) + } + return globalVars[CACHED_CHANGED_FILE_LIST] + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("Get merge request changed file list failed. Error: ${e.toString()}") + globalVars[CACHED_CHANGED_FILE_LIST] = [] + return globalVars[CACHED_CHANGED_FILE_LIST] + } +} + +def getMultiGpuFileChanged(pipeline, testFilter, globalVars) { if (testFilter[(DISABLE_MULTI_GPU_TEST)]) { pipeline.echo("Force not run multi-GPU testing.") @@ -456,12 +526,17 @@ def getMultiGpuFileChanged(pipeline, testFilter) "jenkins/L0_Test.groovy", ] - def changedFileList = "," + def changedFileList = getMergeRequestChangedFileList(pipeline, globalVars) + if (!changedFileList || changedFileList.isEmpty()) { + return false + } + + def changedFileListStr = "," def relatedFileChanged = false try { - changedFileList = getMergeRequestChangedFileList(pipeline).join(", ") + changedFileListStr = changedFileList.join(", ") relatedFileChanged = relatedFileList.any { it -> - if (changedFileList.contains(it)) { + if (changedFileListStr.contains(it)) { return true } } @@ -472,7 +547,7 @@ def getMultiGpuFileChanged(pipeline, testFilter) } catch (Exception e) { - pipeline.echo("getMultiGpuFileChanged failed execution.") + pipeline.echo("getMultiGpuFileChanged failed execution. Error: ${e.toString()}") } if (relatedFileChanged) { pipeline.echo("Detect multi-GPU related files changed.") @@ -480,6 +555,32 @@ def getMultiGpuFileChanged(pipeline, testFilter) return relatedFileChanged } +def getOnlyPytorchFileChanged(pipeline, testFilter, globalVars) { + def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) + if (env.alternativeTRT || isOfficialPostMergeJob) { + pipeline.echo("Force set ONLY_PYTORCH_FILE_CHANGED false.") + return false + } + def pytorchOnlyPattern = ~/^tensorrt_llm\/_torch\/.*/ + + def changedFileList = getMergeRequestChangedFileList(pipeline, globalVars) + + if (!changedFileList || changedFileList.isEmpty()) { + return false + } + + def result = changedFileList.every { file -> + def isPytorchFile = file =~ pytorchOnlyPattern + if (!isPytorchFile) { + pipeline.echo("Found non-PyTorch file: ${file}") + } + return isPytorchFile + } + + pipeline.echo("Only PyTorch files changed: ${result}") + return result +} + def collectTestResults(pipeline, testFilter) { collectResultPodSpec = createKubernetesPodConfig("", "agent") @@ -829,7 +930,7 @@ pipeline { steps { script { - setupPipelineEnvironment(this, testFilter) + setupPipelineEnvironment(this, testFilter, globalVars) echo "enableFailFast is: ${enableFailFast}" echo "env.gitlabTriggerPhrase is: ${env.gitlabTriggerPhrase}" println testFilter diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 324715b5a2df..aca2ac082f64 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -91,6 +91,7 @@ def trimForStageList(stageNameList) return trimedList } +// Test filter flags @Field def REUSE_STAGE_LIST = "reuse_stage_list" @Field