diff --git a/.github/workflows/precommit-check.yml b/.github/workflows/precommit-check.yml new file mode 100644 index 000000000000..55ffa6f5e108 --- /dev/null +++ b/.github/workflows/precommit-check.yml @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Release Checks +on: + pull_request: + workflow_dispatch: + inputs: + ref: + description: 'commit sha to check' + required: true +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +jobs: + precommit-check: + name: Pre-commit Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || github.ref }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Run pre-commit checks + run: | + python3 -u scripts/release_check.py diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 49d72e7f25e4..b304f63c9eb4 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -309,13 +309,7 @@ def launchReleaseCheck(pipeline) // Step 1: cloning tekit source code trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) sh "cd ${LLM_ROOT} && git config --unset-all core.hooksPath" - trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${LLM_ROOT} && pip3 install `grep pre-commit requirements-dev.txt`") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${LLM_ROOT} && pip3 install `grep bandit requirements-dev.txt`") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${LLM_ROOT} && pre-commit install") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${LLM_ROOT} && pre-commit run -a --show-diff-on-failure || (git restore . && false)") - sh "cd ${LLM_ROOT} && bandit --configfile scripts/bandit.yaml -r tensorrt_llm | tee /tmp/bandit.log" - sh "cat /tmp/bandit.log | grep -q 'Total lines skipped (#nosec): 0' && exit 0 || exit 1" - sh "cat /tmp/bandit.log | grep -q 'Issue:' && exit 1 || exit 0" + trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${LLM_ROOT} && python3 -u scripts/release_check.py") // Step 2: build tools withEnv(['GONOSUMDB=*.nvidia.com']) { diff --git a/scripts/package_trt_llm.py b/scripts/package_trt_llm.py index fcb4cde494d1..e67e24347352 100644 --- a/scripts/package_trt_llm.py +++ b/scripts/package_trt_llm.py @@ -6,7 +6,7 @@ from collections import namedtuple from os import PathLike from pathlib import Path -from typing import Iterable, Tuple +from typing import Iterable, List, Tuple def _clean_files(src_dir: PathLike, extend_files: str) -> None: @@ -24,15 +24,18 @@ def _clean_files(src_dir: PathLike, extend_files: str) -> None: "tests/README.md", ] #yapf: disable + files_to_remove.extend(extend_files) for file in files_to_remove: file_path = src_dir / file if file_path.is_dir(): shutil.rmtree(file_path) - logging.debug(f"Removed directory: {file_path}") + logging.debug(f"Removed directory: {file_path}" + ) else: - file_path.unlink() + file_path.unlink( + ) logging.debug(f"Removed file: {file_path}") @@ -91,7 +94,8 @@ def compress(tgt_pkg_name: Path, src_dir: Path) -> None: LibInfo = namedtuple( 'LibInfo', - ('name', 'skip_windows', 'is_static', 'path', 'cleanfiles', 'cleantrees')) + ('name', 'skip_windows', 'is_static', 'path', + 'cleanfiles', 'cleantrees')) LibListConfig = namedtuple('LibListConfig', ('libs', 'cleanfiles')) _builtin_liblist = { diff --git a/scripts/release_check.py b/scripts/release_check.py new file mode 100644 index 000000000000..a4fc719be869 --- /dev/null +++ b/scripts/release_check.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess as sp +import sys + + +def run_cmd(cmd): + """Run a command and return output, raising error by default if command fails""" + + print(f"Running command: {cmd}") + result = sp.run(cmd, + shell=True, + stdout=sp.PIPE, + text=True, + stderr=sp.STDOUT) + if result.returncode != 0: + print(f"Failing command output:\n{result.stdout}") + sys.exit(1) + return result + + +def main(): + # Install pre-commit and bandit from requirements-dev.txt + with open("requirements-dev.txt") as f: + reqs = f.readlines() + pre_commit_req = next(line for line in reqs if "pre-commit" in line) + bandit_req = next(line for line in reqs if "bandit" in line) + + run_cmd(f"pip3 install {pre_commit_req.strip()}") + run_cmd(f"pip3 install {bandit_req.strip()}") + + # Install pre-commit hooks + run_cmd("pre-commit install") + + # Run pre-commit on all files + run_cmd("pre-commit run -a --show-diff-on-failure") + + # Run bandit security checks + bandit_output = run_cmd( + "bandit --configfile scripts/bandit.yaml -r tensorrt_llm").stdout + print(f"Bandit output:\n{bandit_output}") + + # Check bandit results + if "Total lines skipped (#nosec): 0" not in bandit_output: + print("Error: Found #nosec annotations in code") + sys.exit(1) + + if "Issue:" in bandit_output: + print("Error: Bandit found security issues") + sys.exit(1) + + print("pre-commit and bandit checks passed") + + +if __name__ == "__main__": + main()