diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..f1c416fbf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Enforce LF line endings for line-ending-sensitive files on every platform, +# so editors on Windows can't introduce CRLF (which breaks shell scripts and +# trips the codestyle / pre-commit checks). +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.py text eol=lf diff --git a/.github/actions/rerun-workflow/action.yml b/.github/actions/rerun-workflow/action.yml new file mode 100644 index 000000000..2cb4a9c85 --- /dev/null +++ b/.github/actions/rerun-workflow/action.yml @@ -0,0 +1,31 @@ +name: 'Rerun workflow' +description: 'Rerun a failed jobs or a specific named job for a pull request' + +inputs: + GITHUB_TOKEN: + description: 'Token with actions:write scope (pass secrets.GITHUB_TOKEN).' + required: true + OWNER: + description: 'Repository owner' + required: true + REPO: + description: 'Repository name' + required: true + PR_ID: + description: 'Pull request ID' + required: true + JOB_NAME: + description: 'Name of the job to rerun' + required: true + +runs: + using: 'composite' + steps: + - run: bash ./.github/actions/rerun-workflow/rerun.sh + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} + OWNER: ${{ inputs.OWNER }} + REPO: ${{ inputs.REPO }} + PR_ID: ${{ inputs.PR_ID }} + JOB_NAME: ${{ inputs.JOB_NAME }} diff --git a/.github/actions/rerun-workflow/rerun.sh b/.github/actions/rerun-workflow/rerun.sh new file mode 100644 index 000000000..6473de10c --- /dev/null +++ b/.github/actions/rerun-workflow/rerun.sh @@ -0,0 +1,77 @@ +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +COMMIT_SHA=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/pulls/$PR_ID" | jq -r '.head.sha') + +echo "Commit SHA: $COMMIT_SHA" + +response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/actions/runs?head_sha=$COMMIT_SHA&per_page=100") + +echo "Response: $response" + +run_ids=$(echo "$response" | jq -r '.workflow_runs[].id') + +if [ -n "$run_ids" ]; then + echo "Found run_ids for commit $COMMIT_SHA: $run_ids" + + for run_id in $run_ids; do + if [ "$JOB_NAME" = "all-failed" ]; then + echo "Rerunning all failed jobs for run_id: $run_id" + + rerun_response=$(curl -X POST -s -w "%{http_code}" -o /dev/null \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/rerun-failed-jobs") + if [ "$rerun_response" -eq 201 ]; then + echo "Successfully requested rerun for all blocked jobs in run_id: $run_id" + else + echo "Failed to request rerun for run_id: $run_id with status code $rerun_response" + fi + + else + jobs_response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/jobs") + + echo "Jobs Response for run_id $run_id: $jobs_response" + + # if [[ "$JOB_NAME" == *"bypass"* ]]; then + block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \ + '.jobs[] | select(.name == $job_name) | .id') + # else + # block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \ + # '.jobs[] | select(.name == $job_name and .conclusion != "success") | .id') + # fi + + if [ -n "$block_jobs" ]; then + echo "Found block jobs for run_id $run_id: $block_jobs" + + for job_id in $block_jobs; do + echo "Rerunning job_id: $job_id" + curl -X POST -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$OWNER/$REPO/actions/jobs/$job_id/rerun" + done + else + echo "No block jobs found for run_id $run_id with name $JOB_NAME." + fi + fi + done +else + echo "No matching workflow runs found for commit $COMMIT_SHA." + exit 1 +fi diff --git a/.github/workflows/Install_check.yml b/.github/workflows/Install_check.yml new file mode 100644 index 000000000..a49aef111 --- /dev/null +++ b/.github/workflows/Install_check.yml @@ -0,0 +1,93 @@ +name: Install Check + +on: + # Nightly: re-validate master against the freshly built nightly paddle + # Cron is UTC: '0 21 * * *' => 05:00 UTC+8 + # schedule: + # - cron: '0 21 * * *' + pull_request: + branches: [master, develop, "release/**"] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + + push: + branches: [master, develop] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + install-check: + name: Install Check + # TODO: replace with the actual self-hosted CPU runner group name + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: | + requirements.txt + tests/requirements.txt + + - name: Install Latest Release CPU Version Torch + run: | + python -m pip install --upgrade pip + python -m pip install -U torch torchvision --index-url https://download.pytorch.org/whl/cpu + python -c "import torch; print('torch version information:', torch.__version__)" + + - name: Install Latest develop CPU Version Paddle + run: | + python -m pip uninstall -y paddlepaddle + python -m pip uninstall -y paddlepaddle-gpu + python -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle \ + -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ \ + --extra-index-url https://pypi.org/simple \ + --timeout 120 --retries 3 + python -c "import paddle; print('paddle version information:', paddle.__version__); print('paddle commit information:', paddle.__git_commit__)" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + if [ -f tests/requirements.txt ]; then + python -m pip install -r tests/requirements.txt + fi + + - name: Install Check + run: | + python setup.py sdist bdist_wheel + python -m pip install dist/*.whl --force-reinstall + paconvert -V + paconvert --run_check + + - name: Upload dist on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: install-dist-${{ github.run_id }} + path: dist/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/Modeltest_check.yml b/.github/workflows/Modeltest_check.yml new file mode 100644 index 000000000..568d11764 --- /dev/null +++ b/.github/workflows/Modeltest_check.yml @@ -0,0 +1,181 @@ +name: Modeltest Check + +on: + # Nightly: re-validate master against the freshly built nightly paddle + # Cron is UTC: '0 21 * * *' => 05:00 UTC+8 + # schedule: + # - cron: '0 21 * * *' + pull_request: + branches: [master, develop, 'release/**'] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + + push: + branches: [master, develop] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + modeltest: + name: Modeltest Check + runs-on: + group: PaConvert + timeout-minutes: 120 + env: + TORCH_PROJECT_PATH: /workspace/torch_project + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Ensure CI image Exists + run: | + docker image inspect paconvert-ci:cu118 >/dev/null 2>&1 || \ + docker build --pull -t paconvert-ci:cu118 -f tools/docker/Dockerfile tools/docker + + - name: Start Container + run: | + container_name="paconvert-modeltest-${{ github.run_id }}-${{ github.run_attempt }}" + echo "container_name=${container_name}" >> "$GITHUB_ENV" + docker run -d --name "${container_name}" \ + -e http_proxy=$http_proxy \ + -e https_proxy=$https_proxy \ + -e no_proxy="localhost,127.0.0.1" \ + -e NVIDIA_VISIBLE_DEVICES=void \ + -e TORCH_PROJECT_PATH="${TORCH_PROJECT_PATH}" \ + --network host \ + -v /dev/shm:/dev/shm \ + -v "${{ github.workspace }}:/ws" \ + -v "${TORCH_PROJECT_PATH}:${TORCH_PROJECT_PATH}:ro" \ + -w /ws \ + paconvert-ci:cu118 sleep infinity + + - name: Run Modeltest + run: docker exec "${container_name}" bash scripts/ci/run_modeltest.sh + + - name: Upload modeltest logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: modeltest-log-${{ github.run_id }} + path: | + failed_projects.txt + tests/code_library/model_case/**/convert_paddle_code* + if-no-files-found: ignore + retention-days: 14 + + - name: Cleanup Container + if: always() + run: | + if [ -n "${container_name:-}" ]; then + docker exec "${container_name}" chown -R "$(id -u):$(id -g)" /ws 2>/dev/null || true + docker rm -f "${container_name}" || true + fi + + + # - name: Setup Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.10' + # cache: pip + # cache-dependency-path: | + # requirements.txt + # tests/requirements.txt + + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # python -m pip install -U torch --index-url https://download.pytorch.org/whl/cpu + # python -m pip uninstall -y paddlepaddle paddlepaddle-gpu + + # # Detect current Python ABI tag (e.g. cp310) to avoid installing a mismatched wheel + # PY_ABI=$(python -c "import sys; print('cp{}{}'.format(*sys.version_info[:2]))") + # echo "Current Python ABI: $PY_ABI" + + # if ls paddlepaddle-0.0.0-*.whl >/dev/null 2>&1; then + # # Validate the local wheel matches the current Python version before installing + # MATCHED_WHL=$(ls paddlepaddle-0.0.0-*${PY_ABI}*.whl 2>/dev/null | head -1) + # if [ -z "$MATCHED_WHL" ]; then + # echo "WARNING: local wheel found but none matches ABI $PY_ABI; falling back to nightly." + # python -m pip install --force-reinstall --no-cache-dir --pre paddlepaddle \ + # -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ \ + # --extra-index-url https://pypi.org/simple \ + # --timeout 120 --retries 3 + # else + # echo "Installing local wheel: $MATCHED_WHL" + # python -m pip install "$MATCHED_WHL" + # fi + # else + # # No local wheel — install from nightly with retry to reduce transient failures + # python -m pip install --force-reinstall --no-cache-dir --pre paddlepaddle \ + # -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ \ + # --timeout 120 --retries 3 + # fi + # python -m pip install -r requirements.txt + # python -m pip install pandas openpyxl + # python -c "import torch; print('torch version information:', torch.__version__)" + # python -c "import paddle; print('paddle version information:', paddle.__version__); print('paddle commit information:', paddle.__git_commit__)" + + # - name: Run code set convert check + # shell: bash + # run: | + # if [ ! -d "$TORCH_PROJECT_PATH" ]; then + # echo "$TORCH_PROJECT_PATH does not exist. Please prepare the model code set on the self-hosted runner." + # exit 1 + # fi + + # shopt -s nullglob + # projects=("$TORCH_PROJECT_PATH"/*) + # if [ ${#projects[@]} -eq 0 ]; then + # echo "$TORCH_PROJECT_PATH is empty. Please prepare the model code set on the self-hosted runner." + # exit 1 + # fi + + # failed_projects=() + # for project in "${projects[@]}"; do + # if [ -d "$project" ]; then + # project_name=$(basename "$project") + # echo "[code-set-convert] Converting project: $project_name" + # if ! python paconvert/main.py --in_dir "$project" --show_unsupport_api --calculate_speed; then + # failed_projects+=("$project_name") + # fi + # fi + # done + + # if [ ${#failed_projects[@]} -ne 0 ]; then + # printf '%s\n' "${failed_projects[@]}" > failed_projects.txt + # echo "[code-set-convert] The following projects failed to convert:" + # cat failed_projects.txt + # exit 1 + # fi + + # - name: Run modeltest + # run: python tools/modeltest/modeltest_check.py + + # - name: Upload modeltest logs on failure + # if: failure() + # uses: actions/upload-artifact@v4 + # with: + # name: modeltest-log-${{ github.run_id }} + # path: | + # failed_projects.txt + # tests/code_library/model_case/**/convert_paddle_code* + # if-no-files-found: ignore + # retention-days: 14 diff --git a/.github/workflows/Unittest_check.yml b/.github/workflows/Unittest_check.yml new file mode 100644 index 000000000..ec3394cb6 --- /dev/null +++ b/.github/workflows/Unittest_check.yml @@ -0,0 +1,174 @@ +name: CI Unittest + +on: + # Nightly: re-validate master against the freshly built nightly paddle + # Cron is UTC: '0 21 * * *' => 05:00 UTC+8 + # schedule: + # - cron: '0 21 * * *' + pull_request: + branches: [master, develop, "release/**"] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + push: + branches: [master, develop] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + ci-unittest: + name: CI Unittest + runs-on: + group: PaConvert + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Ensure CI image Exists + run: | + docker image inspect paconvert-ci:cu118 >/dev/null 2>&1 || \ + docker build --pull --network=host \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + --build-arg no_proxy="localhost,127.0.0.1" \ + -t paconvert-ci:cu118 -f tools/docker/Dockerfile tools/docker + + - name: Start Container + run: | + container_name="paconvert-cpu-unittest-${{ github.run_id }}-${{ github.run_attempt }}" + echo "container_name=${container_name}" >> "$GITHUB_ENV" + docker run -d --name "${container_name}" \ + -e NVIDIA_VISIBLE_DEVICES=void \ + -e http_proxy=$http_proxy \ + -e https_proxy=$https_proxy \ + -e no_proxy="localhost,127.0.0.1" \ + --network host \ + -v /dev/shm:/dev/shm \ + -v "${{ github.workspace }}:/ws" \ + -w /ws \ + paconvert-ci:cu118 sleep infinity + + - name: Run CI Unittest + shell: bash + run: docker exec "$container_name" bash scripts/ci/run_cpu_unittest.sh + + - name: Upload logs on failing + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ci-unittest-log-${{ github.run_id }} + path: | + pytest.log + tests/**/pytest.log + if-no-files-found: ignore + retention-days: 14 + + - name: Cleanup Container + if: always() + run: | + if [ -n "${container_name:-}" ]; then + docker exec "${container_name}" chown -R "$(id -u):$(id -g)" /ws 2>/dev/null || true + docker rm -f "${container_name}" || true + fi + + # - name: Setup Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.10' + # cache: pip + # cache-dependency-path: | + # requirements.txt + # tests/requirements.txt + + # - name: Check Env + # run: | + # python --version + # pwd + + # - name: Install Latest Release CPU Version Torch + # run: | + # python -m pip install --upgrade pip + # python -m pip install -U torch torchvision --index-url https://download.pytorch.org/whl/cpu + # python -c "import torch; print('torch version information:', torch.__version__)" + + # - name: Install Latest develop CPU Version Paddle + # run: | + # python -m pip uninstall -y paddlepaddle + # python -m pip uninstall -y paddlepaddle-gpu + # python -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle \ + # -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ \ + # --extra-index-url https://pypi.org/simple \ + # --timeout 120 --retries 3 + # python -c "import paddle; print('paddle version information:', paddle.__version__); print('paddle commit information:', paddle.__git_commit__)" + + # - name: Install paconvert requirements + # run: | + # python -m pip install -r requirements.txt + # if [ -f tests/requirements.txt ]; then + # python -m pip install -r tests/requirements.txt + # fi + + # - name: Run CI Unittest + # shell: bash + # run: | + # python -m pip install pytest-timeout pytest-xdist pytest-rerunfailures + + # # Disable errexit so a failing run is captured instead of aborting immediately + # # A passing retry can clear the error. + # set +e + + # echo "Checking code cpu unit test by pytest ..." + # # tee output to pytest.log so it can be uploaded as an artifact on failure; + # # PIPESTATUS[0] preserves pytest's exit code instead of tee's. + # python -m pytest -v -s -p no:warnings --reruns=3 ./tests 2>&1 | tee pytest.log + # check_error=${PIPESTATUS[0]} + # if [ ${check_error} -ne 0 ]; then + # echo "Rerun cpu unit test check." + # python -m pytest -v -s -p no:warnings --lf ./tests 2>&1 | tee -a pytest.log + # check_error=${PIPESTATUS[0]} + # fi + + # echo '************************************************************************************************************' + # if [ ${check_error} -ne 0 ]; then + # echo "Your PR code cpu unit test check failed." + # echo "Please run the following command:" + # echo "" + # echo " python -m pytest tests" + # echo "" + # echo "For more information, please refer to our check guide:" + # echo "https://github.com/PaddlePaddle/PaConvert#readme." + # else + # echo "Your PR code cpu unit test check passed." + # fi + # echo '************************************************************************************************************' + + # exit ${check_error} + + # - name: Upload logs on failure + # if: failure() + # uses: actions/upload-artifact@v4 + # with: + # name: ci-unittest-log-${{ github.run_id }} + # path: | + # pytest.log + # tests/**/pytest.log + # if-no-files-found: ignore + # retention-days: 14 diff --git a/.github/workflows/Unittest_check_distribute.yml b/.github/workflows/Unittest_check_distribute.yml new file mode 100644 index 000000000..3e594db50 --- /dev/null +++ b/.github/workflows/Unittest_check_distribute.yml @@ -0,0 +1,208 @@ +name: Distributed Unittest Check + +on: + # Nightly: re-validate master against the freshly built nightly paddle + # Cron is UTC: '0 21 * * *' => 05:00 UTC+8 + # schedule: + # - cron: '0 21 * * *' + pull_request: + branches: [master, develop, "release/**"] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + push: + branches: [master, develop] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + distributed-unittest: + name: Distributed Unittest Check + runs-on: + group: PaConvert + timeout-minutes: 120 + concurrency: + group: paconvert-gpu-runner + cancel-in-progress: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Ensure CI image Exists + run: | + docker image inspect paconvert-ci:cu118 >/dev/null 2>&1 || \ + docker build --pull -t paconvert-ci:cu118 -f tools/docker/Dockerfile tools/docker + + - name: Free port + run: | + ss -ltnp 2>/dev/null | grep ':29500' | grep -oP 'pid=\K[0-9]+' | xargs -r kill -9 2>/dev/null || true + + - name: Start Container + run: | + container_name="paconvert-distribute-${{ github.run_id }}-${{ github.run_attempt }}" + echo "container_name=${container_name}" >> "$GITHUB_ENV" + docker run -d --name "${container_name}" \ + -e http_proxy=$http_proxy \ + -e https_proxy=$https_proxy \ + -e no_proxy="localhost,127.0.0.1" \ + --gpus all \ + --network host \ + --ipc host \ + -e CUDA_VISIBLE_DEVICES=0,1 \ + -v /dev/shm:/dev/shm \ + -v "${{ github.workspace }}:/ws" \ + -w /ws \ + paconvert-ci:cu118 sleep infinity + + - name: Run Distributed Unittest + run: | + docker exec "${container_name}" bash scripts/ci/run_distributed.sh + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: distributed-unittest-log-${{ github.run_id }} + path: | + tests/distributed/failed_tests.txt + tests/distributed/*.log + paddle_dist/** + if-no-files-found: ignore + retention-days: 14 + + - name: Cleanup Container + if: always() + run: | + if [ -n "${container_name:-}" ]; then + docker exec "${container_name}" chown -R "$(id -u):$(id -g)" /ws 2>/dev/null || true + docker rm -f "${container_name}" || true + fi + + # - name: Setup Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.10' + # cache: pip + # cache-dependency-path: | + # requirements.txt + # tests/requirements.txt + + # - name: Check Env + # run: | + # python3 --version + # pwd + + # - name: Download GPU dependencies + # working-directory: tests/distributed + # run: | + # echo "Download torch wheel..." + # wget --no-verbose https://paddle-paconvert.bj.bcebos.com/torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl + # ls -lh torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl + # echo "Download torch wheel successful!" + + # - name: Install Latest Release GPU Version Torch + # working-directory: tests/distributed + # run: | + # python3 -m pip install --upgrade pip + # python3 -m pip uninstall -y torchaudio + # python3 -m pip uninstall -y torchvision + # NO_PROXY='*' no_proxy='*' python3 -m pip install \ + # torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl \ + # -i https://pypi.tuna.tsinghua.edu.cn/simple \ + # --timeout 120 --retries 3 + # python3 -c "import torch; print('torch version information:', torch.__version__)" + + # - name: Install Latest develop GPU Version Paddle + # run: | + # python3 -m pip uninstall -y paddlepaddle + # python3 -m pip uninstall -y paddlepaddle-gpu + # python3 -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle-gpu \ + # -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ \ + # --extra-index-url https://pypi.org/simple \ + # --timeout 120 --retries 3 + # python3 -m pip install safetensors==0.6.2 + # python3 -c "import paddle; print('paddle version information:', paddle.__version__); print('paddle commit information:', paddle.__git_commit__)" + + # - name: Install paconvert requirements + # run: | + # python3 -m pip install -r requirements.txt + + # - name: Convert torch code to paddle + # working-directory: tests/distributed + # run: | + # python3 ../../paconvert/main.py -i . -o /tmp/paddle_dist --log_level "DEBUG" + + # - name: Run distributed unit tests + # working-directory: tests/distributed + # shell: bash + # env: + # CUDA_VISIBLE_DEVICES: "0,1" + # run: | + # # Free port 29500 if any leftover process is occupying it + # netstat -tulnp 2>/dev/null | grep ':29500' | awk '{print $7}' | cut -d/ -f1 | xargs -r kill -9 2>/dev/null || true + + # # Disable errexit so every test runs and all failures are collected + # # Instead of aborting on the first failing test. + # set +e + + # check_error=0 + # failed_tests=() + # test_list=$(ls *.py | grep -v run_and_compare.py) + # for item in $test_list; do + # cmd1="torchrun --nproc_per_node=2 ${item}" + # cmd2="python3 -m paddle.distributed.launch /tmp/paddle_dist/${item}" + # python3 run_and_compare.py "$cmd1" "$cmd2" + # tmp_check_error=$? + # if [ $tmp_check_error -ne 0 ]; then + # check_error=1 + # failed_tests+=("$item") + # fi + # done + + # echo '************************************************************************************************************' + # if [ ${#failed_tests[@]} -ne 0 ]; then + # printf '%s\n' "${failed_tests[@]}" > failed_tests.txt + # echo "Your PR code distributed unittest check FAILED" + # echo "The following distributed tests failed:" + # cat failed_tests.txt + # echo "Please run the following command:" + # echo "" + # echo " cd tests/distributed && bash unittest_check_distribute.sh" + # echo "" + # echo "For more information, please refer to our check guide:" + # echo "https://github.com/PaddlePaddle/PaConvert#readme" + # else + # echo "Your PR code distributed unit test check passed." + # fi + # echo '************************************************************************************************************' + + # exit ${check_error} + + # - name: Upload logs on failure + # if: failure() + # uses: actions/upload-artifact@v4 + # with: + # name: distributed-unittest-log-${{ github.run_id }} + # path: | + # tests/distributed/failed_tests.txt + # tests/distributed/*.log + # /tmp/paddle_dist/** + # if-no-files-found: ignore + # retention-days: 14 diff --git a/.github/workflows/Unittest_check_gpu.yml b/.github/workflows/Unittest_check_gpu.yml new file mode 100644 index 000000000..f0c9ebbc7 --- /dev/null +++ b/.github/workflows/Unittest_check_gpu.yml @@ -0,0 +1,181 @@ +name: GPU Unittest Check + +on: + # Nightly: re-validate master against the freshly built nightly paddle + # Cron is UTC: '0 21 * * *' => 05:00 UTC+8 + # schedule: + # - cron: '0 21 * * *' + pull_request: + branches: [master, develop, "release/**"] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + push: + branches: [master, develop] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + gpu-unittest: + name: GPU Unittest Check + runs-on: + group: PaConvert + timeout-minutes: 120 + concurrency: + group: paconvert-gpu-runner + cancel-in-progress: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Ensure CI image Exists + run: | + docker image inspect paconvert-ci:cu118 >/dev/null 2>&1 || \ + docker build --pull -t paconvert-ci:cu118 -f tools/docker/Dockerfile tools/docker + + - name: Start Container + run: | + container_name="paconvert-gpu-unittest-${{ github.run_id }}-${{ github.run_attempt }}" + echo "container_name=${container_name}" >> "$GITHUB_ENV" + docker run -d --name "${container_name}" \ + -e http_proxy=$http_proxy \ + -e https_proxy=$https_proxy \ + -e no_proxy="localhost,127.0.0.1" \ + --gpus all \ + --network host \ + -v /dev/shm:/dev/shm \ + -v "${{ github.workspace }}:/ws" \ + -w /ws \ + paconvert-ci:cu118 sleep infinity + + - name: Run GPU Unittest + run: | + docker exec "$container_name" bash scripts/ci/run_gpu_unittest.sh + + - name: Cleanup Container + if: always() + run: | + if [ -n "${container_name:-}" ]; then + docker exec "${container_name}" chown -R "$(id -u):$(id -g)" /ws 2>/dev/null || true + docker rm -f "${container_name}" || true + fi + + # - name: Setup Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.10' + # cache: pip + # cache-dependency-path: | + # requirements.txt + # tests/requirements.txt + + # - name: Check Env + # run: | + # python3 --version + # pwd + + # - name: Download GPU dependencies + # run: | + # echo "Downloading torch wheel..." + # wget --no-verbose https://paddle-paconvert.bj.bcebos.com/torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl + # ls -lh torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl + + # echo "Downloading torchvision wheel..." + # wget --no-verbose https://paddle-paconvert.bj.bcebos.com/torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl + # ls -lh torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl + + # echo "Downloaded GPU dependencies successfully!" + + # - name: Install Latest Release GPU Version Torch + # run: | + # python3 -m pip install --upgrade pip + # python3 -m pip uninstall -y torchaudio + # NO_PROXY='*' no_proxy='*' python3 -m pip install \ + # torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl \ + # torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl \ + # -i https://pypi.tuna.tsinghua.edu.cn/simple \ + # --timeout 120 --retries 3 + # python3 -c "import torch; print('torch version information:', torch.__version__)" + + # - name: Install Latest develop GPU Version Paddle + # run: | + # python3 -m pip uninstall -y paddlepaddle + # python3 -m pip uninstall -y paddlepaddle-gpu + # NO_PROXY='*' no_proxy='*' python3 -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle-gpu \ + # -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ \ + # --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + # --timeout 120 --retries 3 + # NO_PROXY='*' no_proxy='*' python3 -m pip install safetensors==0.6.2 \ + # -i https://pypi.tuna.tsinghua.edu.cn/simple + # python3 -c "import paddle; print('paddle version information:', paddle.__version__); print('paddle commit information:', paddle.__git_commit__)" + + # - name: Install paconvert requirements + # run: | + # python3 -m pip install -r requirements.txt + # if [ -f tests/requirements.txt ]; then + # python3 -m pip install -r tests/requirements.txt + # fi + + # - name: Run GPU Unittest + # shell: bash + # run: | + # python3 -m pip install pytest-timeout pytest-xdist pytest-rerunfailures + + # # Disable errexit so a failing run is captured instead of aborting immediately + # # A passing retry can clear the error. + # set +e + + # echo "Checking code gpu unit test by pytest ..." + # # tee output to pytest.log so it can be uploaded as an artifact on failure; + # # PIPESTATUS[0] preserves pytest's exit code instead of tee's. + # python3 -m pytest -v -s -p no:warnings --reruns=3 ./tests 2>&1 | tee pytest.log + # check_error=${PIPESTATUS[0]} + # if [ ${check_error} -ne 0 ]; then + # echo "Rerun gpu unit test check." + # python3 -m pytest -v -s -p no:warnings --lf ./tests 2>&1 | tee -a pytest.log + # check_error=${PIPESTATUS[0]} + # fi + + # echo '************************************************************************************************************' + # if [ ${check_error} -ne 0 ]; then + # echo "Your PR code gpu unit test check failed." + # echo "Please run the following command:" + # echo "" + # echo " python3 -m pytest tests" + # echo "" + # echo "For more information, please refer to our check guide:" + # echo "https://github.com/PaddlePaddle/PaConvert#readme." + # else + # echo "Your PR code gpu unit test check passed." + # fi + # echo '************************************************************************************************************' + + # exit ${check_error} + + # - name: Upload logs on failure + # if: failure() + # uses: actions/upload-artifact@v4 + # with: + # name: gpu-unittest-log-${{ github.run_id }} + # path: | + # pytest.log + # tests/**/pytest.log + # if-no-files-found: ignore + # retention-days: 14 diff --git a/.github/workflows/build_ci_images.yml b/.github/workflows/build_ci_images.yml new file mode 100644 index 000000000..ca03ba0b9 --- /dev/null +++ b/.github/workflows/build_ci_images.yml @@ -0,0 +1,46 @@ +name : Build CI Image + +on: + push: + branches: [master, develop, migrate_workflow] + paths: + - 'tools/docker/**' + - '.github/workflows/build_ci_image.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-ci-image: + name: Build CI Image + runs-on: + group: PaConvert + timeout-minutes: 100 + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Build Image + run: | + docker build --pull --network=host \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + --build-arg no_proxy=".baidu.com,.bcebos.com" \ + -t paconvert-ci:cu118 -f tools/docker/Dockerfile tools/docker + + - name: GPU Test + run: | + docker run --rm --gpus all paconvert-ci:cu118 nvidia-smi + + - name: Image Build Summary + run: | + docker image inspect paconvert-ci:cu118 \ + --format 'paconvert-ci:cu118 size={{.Size}} created={{.Created}}' diff --git a/.github/workflows/codestyle.yml b/.github/workflows/codestyle.yml new file mode 100644 index 000000000..48feb08ea --- /dev/null +++ b/.github/workflows/codestyle.yml @@ -0,0 +1,52 @@ +name: CodeStyle Check + +on: + # Nightly: re-validate master against the freshly built nightly paddle + # Cron is UTC: '0 21 * * *' => 05:00 UTC+8 + # schedule: + # - cron: '0 21 * * *' + pull_request: + branches: [master, develop, "release/**"] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + push: + branches: [master, develop] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + pre-commit: + # TODO: replace with the actual self-hosted CPU runner group name + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pre-commit==2.17.0 + + - name: Run pre-commit + run: pre-commit run --all-files --show-diff-on-failure diff --git a/.github/workflows/consistency.yml b/.github/workflows/consistency.yml new file mode 100644 index 000000000..accd5d3fa --- /dev/null +++ b/.github/workflows/consistency.yml @@ -0,0 +1,78 @@ +name: Consistency Check + +on: + # Nightly: re-validate master against the freshly built nightly paddle + # Cron is UTC: '0 21 * * *' => 05:00 UTC+8 + # schedule: + # - cron: '0 21 * * *' + pull_request: + branches: [master, develop, "release/**"] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + + push: + branches: [master, develop] + paths-ignore: + - '**/*.md' + - 'docs/**' + - 'LICENSE' + - '.gitignore' + - '.pre-commit-config.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + common-api-consistency: + name: Common API Consistency Check + # TODO: replace with the actual self-hosted CPU runner group name + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + cache-dependency-path: | + requirements.txt + tests/requirements.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + if [ -f tests/requirements.txt ]; then + python -m pip install -r tests/requirements.txt + fi + + - name: Run common API consistency check + run: | + set -o pipefail + python tools/consistency/consistency_check.py 2>&1 | tee consistency_check.log + + - name : Upload log on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: consistency-log-${{ github.run_id }} + path: | + consistency_check.log + tools/consistency/*.log + tools/consistency/*.txt + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/rerun.yml b/.github/workflows/rerun.yml new file mode 100644 index 000000000..78ed4a978 --- /dev/null +++ b/.github/workflows/rerun.yml @@ -0,0 +1,107 @@ +name: Rerun Workflows + +on: + issue_comment: + types: [created] + +permissions: + actions: write + contents: read + pull-requests: read + +concurrency: + group: rerun-${{ github.event.issue.number }} + cancel-in-progress: true + +jobs: + re-run: + name: rerun workflows + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/re-run') && (github.event.comment.user.login == github.event.issue.user.login || contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)) }} + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Rerun all failed jobs + if: ${{ contains(github.event.comment.body, 'all-failed') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'all-failed' + + - name: Rerun Codestyle + if: ${{ contains(github.event.comment.body, 'codestyle') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'pre-commit' + + - name: Rerun Common API Consistency Check + if: ${{ contains(github.event.comment.body, 'consistency') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'Common API Consistency Check' + + - name: Rerun Install Check + if: ${{ contains(github.event.comment.body, 'install-check') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'Install Check' + + - name: Rerun Modeltest Check + if: ${{ contains(github.event.comment.body, 'model-test') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'Modeltest Check' + + - name: Rerun Distributed Unittest Check + if: ${{ contains(github.event.comment.body, 'distributed-unittest') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'Distributed Unittest Check' + + - name: Rerun CI Unittest + if: ${{ contains(github.event.comment.body, 'cpu-unittest') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'CI Unittest' + + - name: Rerun GPU Unittest Check + if: ${{ contains(github.event.comment.body, 'gpu-unittest') }} + uses: ./.github/actions/rerun-workflow + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO: ${{ github.event.repository.name }} + PR_ID: ${{ github.event.issue.number }} + JOB_NAME: 'GPU Unittest Check' diff --git a/scripts/ci/run_cpu_unittest.sh b/scripts/ci/run_cpu_unittest.sh new file mode 100644 index 000000000..88d2555a8 --- /dev/null +++ b/scripts/ci/run_cpu_unittest.sh @@ -0,0 +1,75 @@ +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eo pipefail + +echo '******************************************************************************' +echo "Installing develop CPU version paddle" +python -m pip uninstall -y paddlepaddle paddlepaddle-gpu || true +python -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle \ + -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ \ + --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + --timeout 120 --retries 3 +python -c "import paddle; print('paddle version: ', paddle.__version__); print('paddle commit info: ', paddle.__git_commit__)" + +echo '******************************************************************************' +echo "Installing paconvert requirements" +python -m pip install -r requirements.txt +if [ -f tests/requirements.txt ]; then + python -m pip install -r tests/requirements.txt +fi + +echo '******************************************************************************' +python -c "import torch; print('torch version: ', torch.__version__, '| cuda available: ', torch.cuda.is_available())" + +echo '******************************************************************************' +echo "Checking code cpu unit test by pytest ..." +set +e + +PYTEST_IGNORE="\ +--ignore=tests/test_backends_cuda_is_built.py \ +--ignore=tests/test_cuda_is_bf16_supported.py \ +--ignore=tests/test_backends_cudnn_is_available.py \ +--ignore=tests/test_distributed_is_nccl_available.py \ +--ignore=tests/test_hub_download_url_to_file.py \ +--ignore=tests/test_hub_help.py \ +--ignore=tests/test_hub_list.py \ +--ignore=tests/test_hub_load.py \ +--ignore=tests/test_hub_load_state_dict_from_url.py \ +" + +python -m pytest -v -s -p no:warnings $PYTEST_IGNORE --reruns=3 ./tests 2>&1 | tee pytest.log +check_errors=${PIPESTATUS[0]} +if [ ${check_errors} -ne 0 ]; then + echo "Rerun CPU unit test" + python -m pytest -v -s -p no:warnings $PYTEST_IGNORE --lf ./tests 2>&1 | tee -a pytest.log + check_errors=${PIPESTATUS[0]} +fi + +echo '******************************************************************************' +if [ ${check_errors} -ne 0 ]; then + echo "Your PR code CPU unit test check FAILED" + echo "Please run the following command:" + echo "" + echo " pytest -m pytest tests" + echo "" + echo "For more information, please refer to our check guides:" + echo "https://github.com/paddlepaddle/paconvert#readme" +else + echo "All tests PASSED!" +fi +echo '******************************************************************************' + +exit ${check_errors} diff --git a/scripts/ci/run_distributed.sh b/scripts/ci/run_distributed.sh new file mode 100644 index 000000000..f4de5ea6c --- /dev/null +++ b/scripts/ci/run_distributed.sh @@ -0,0 +1,75 @@ +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eo pipefail + +DIST_OUT="$(pwd)/paddle_dist" + +echo '******************************************************************************' +echo "Installing develop GPU version paddle" +python -m pip uninstall -y paddlepaddle paddlepaddle-gpu || true +python -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle-gpu \ + -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ \ + --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + --timeout 120 --retries 3 +python -m pip install safetensors==0.6.2 +python -c "import paddle; print('paddle version: ', paddle.__version__); print('paddle commit info: ', paddle.__git_commit__)" + +echo '******************************************************************************' +echo "Installing paconvert requirements" +python -m pip install -r requirements.txt + +cd tests/distributed + +echo '******************************************************************************' +echo 'Converting torch code to paddle -> ${DIST_OUT}' +rm -rf "${DIST_OUT}" +python ../../paconvert/main.py -i . -o "${DIST_OUT}" --log_level "DEBUG" + +echo '******************************************************************************' +echo "Running Distribute Unit Tests" +set +e + +check_errors=0 +failed_tests=() +test_list=$(ls *.py | grep -v run_and_compare.py) +for item in $test_list; do + cmd1="torchrun --nproc_per_node=2 ${item}" + cmd2="python -m paddle.distributed.launch ${DIST_OUT}/${item}" + python run_and_compare.py "$cmd1" "$cmd2" + if [ $? -ne 0 ]; then + failed_tests+=("${item}") + check_errors=1 + fi +done + +echo '******************************************************************************' +if [ ${#failed_tests[@]} -ne 0 ]; then + printf '%s\n' "${failed_tests[@]}" > failed_tests.txt + echo "Your PR code Distributed unit test check FAILED" + echo "The following distributed tests failed:" + cat failed_tests.txt + echo "Please run the following command:" + echo "" + echo " cd tests/distributed && bash unittest_check_distribute.sh" + echo "" + echo "For more information, please refer to our check guides:" + echo "https://github.com/paddlepaddle/paconvert#readme" +else + echo "All tests PASSED!" +fi +echo '******************************************************************************' + +exit ${check_errors} diff --git a/scripts/ci/run_gpu_unittest.sh b/scripts/ci/run_gpu_unittest.sh new file mode 100644 index 000000000..18cec854c --- /dev/null +++ b/scripts/ci/run_gpu_unittest.sh @@ -0,0 +1,64 @@ +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eo pipefail + +echo '******************************************************************************' +echo "Installing develop GPU version paddle" +python -m pip uninstall -y paddlepaddle paddlepaddle-gpu || true +python -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle-gpu \ + -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ \ + --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + --timeout 120 --retries 3 +python -m pip install safetensors==0.6.2 +python -c "import paddle; print('paddle version: ', paddle.__version__); print('paddle commit info: ', paddle.__git_commit__)" + +echo '******************************************************************************' +echo "Installing paconvert requirements" +python -m pip install -r requirements.txt +if [ -f tests/requirements.txt ]; then + python -m pip install -r tests/requirements.txt +fi + +echo '******************************************************************************' +python -c "import torch; print('torch version: ', torch.__version__, '| cuda available: ', torch.cuda.is_available())" + +echo '******************************************************************************' +echo "Checking code gpu unit test by pytest ..." +set +e + +python -m pytest -v -s -p no:warnings -n 1 --reruns=3 ./tests 2>&1 | tee pytest.log +check_errors=${PIPESTATUS[0]} +if [ ${check_errors} -ne 0 ]; then + echo "Rerun GPU unit test" + python -m pytest -v -s -p no:warnings -n 1 --lf ./tests 2>&1 | tee -a pytest.log + check_errors=${PIPESTATUS[0]} +fi + +echo '******************************************************************************' +if [ ${check_errors} -ne 0 ]; then + echo "Your PR code GPU unit test check FAILED" + echo "Please run the following command:" + echo "" + echo " pytest -m pytest tests" + echo "" + echo "For more information, please refer to our check guides:" + echo "https://github.com/paddlepaddle/paconvert#readme" +else + echo "All tests PASSED!" +fi +echo '******************************************************************************' + +exit ${check_errors} diff --git a/scripts/ci/run_modeltest.sh b/scripts/ci/run_modeltest.sh new file mode 100644 index 000000000..619d892a1 --- /dev/null +++ b/scripts/ci/run_modeltest.sh @@ -0,0 +1,81 @@ +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eo pipefail + +TORCH_PROJECT_PATH="${TORCH_PROJECT_PATH:-/workspace/torch_project}" + +echo '******************************************************************************' +echo "Installing develop CPU version paddle" +python -m pip uninstall -y paddlepaddle paddlepaddle-gpu || true +python -m pip install --force-reinstall --no-cache-dir -U --pre paddlepaddle \ + -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ \ + --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + --timeout 120 --retries 3 +python -c "import paddle; print('paddle version: ', paddle.__version__); print('paddle commit info: ', paddle.__git_commit__)" + +echo '******************************************************************************' +echo "Installing paconvert requirements" +python -m pip install -r requirements.txt +python -m pip install pandas openpyxl || true + +set +e + +echo '******************************************************************************' +echo "[code-set-convert] Start converting code set under ${TORCH_PROJECT_PATH}" +if [ ! -d "${TORCH_PROJECT_PATH}" ]; then + echo "${TORCH_PROJECT_PATH} is not a valid directory. Please stage the model code set on the runner host." + exit 1 +fi + +shopt -s nullglob +projects=("${TORCH_PROJECT_PATH}"/*) +if [ ${#projects[@]} -eq 0 ]; then + echo "${TORCH_PROJECT_PATH} is empty. Please stage the model code set on the runner host." + exit 1 +fi + +failed_project=() +for project in "${projects[@]}"; do + if [ -d "$project" ]; then + project_name=$(basename "$project") + echo "[code-set-convert] Converting project: $project_name" + if ! python paconvert/main.py --in_dir "$project" --show_unsupport_api --calculate_speed; then + failed_project+=("$project_name") + fi + fi +done + +if [ ${#failed_project[@]} -ne 0 ]; then + printf '%s\n' "${failed_project[@]}" > failed_projects.txt + echo "[code-set-convert] The following projects fail to convert:" + cat failed_projects.txt + exit 1 +fi + +echo '******************************************************************************' +echo "[modeltest] Start modeltest" +python tools/modeltest/modeltest_check.py +check_errors=$? + +echo '******************************************************************************' +if [ ${check_errors} -ne 0 ]; then + echo "Your PR code modeltest check FAILED" +else + echo "All Modeltest PASSED!" +fi +echo '******************************************************************************' + +exit ${check_errors} diff --git a/tests/flash_attn_tests/__init__.py b/tests/flash_attn_tests/__init__.py new file mode 100644 index 000000000..8766bb689 --- /dev/null +++ b/tests/flash_attn_tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/test_block_diag.py b/tests/test_block_diag.py index c38b5d9cc..2e2952c8f 100644 --- a/tests/test_block_diag.py +++ b/tests/test_block_diag.py @@ -1,108 +1,108 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import textwrap - -from apibase import APIBase - -obj = APIBase("torch.block_diag") - - -def test_case_1(): - pytorch_code = textwrap.dedent( - """ - import torch - A = torch.tensor([[0, 1], [1, 0]]) - B = torch.tensor([[3, 4, 5], [6, 7, 8]]) - C = torch.tensor(7) - D = torch.tensor([1, 2, 3]) - E = torch.tensor([[4], [5], [6]]) - result = torch.block_diag(A, B, C, D, E) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_2(): - pytorch_code = textwrap.dedent( - """ - import torch - A = torch.tensor([[4], [3], [2]]) - B = torch.tensor([7, 6, 5]) - C = torch.tensor(1) - result = torch.block_diag(torch.tensor([[4], [3], [2]]), - torch.tensor([7, 6, 5]), - torch.tensor(1)) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_3(): - pytorch_code = textwrap.dedent( - """ - import torch - A = torch.tensor([[4], [3], [2]]) - B = torch.tensor([[5, 6], [9, 1]]) - C = torch.tensor([1, 2, 3]) - result = torch.block_diag(A) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_4(): - pytorch_code = textwrap.dedent( - """ - import torch - A = torch.tensor([[4], [3], [2]]) - B = torch.tensor([[5], [6]]) - result = torch.block_diag(A, B, torch.tensor([1, 2, 3])) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_5(): - pytorch_code = textwrap.dedent( - """ - import torch - tensors = torch.tensor([[0,1,2]]), torch.tensor([[0],[1]]), torch.tensor([[20]]) - result = torch.block_diag(*tensors) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_6(): - pytorch_code = textwrap.dedent( - """ - import torch - result = torch.block_diag(torch.tensor([[4], [3], [2]])) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_7(): - """Test with expression arguments""" - pytorch_code = textwrap.dedent( - """ - import torch - A = torch.tensor([[1, 2], [3, 4]]) - B = torch.tensor([[5, 6], [7, 8]]) - result = torch.block_diag(A + 1, B * 2) - """ - ) +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import textwrap + +from apibase import APIBase + +obj = APIBase("torch.block_diag") + + +def test_case_1(): + pytorch_code = textwrap.dedent( + """ + import torch + A = torch.tensor([[0, 1], [1, 0]]) + B = torch.tensor([[3, 4, 5], [6, 7, 8]]) + C = torch.tensor(7) + D = torch.tensor([1, 2, 3]) + E = torch.tensor([[4], [5], [6]]) + result = torch.block_diag(A, B, C, D, E) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_2(): + pytorch_code = textwrap.dedent( + """ + import torch + A = torch.tensor([[4], [3], [2]]) + B = torch.tensor([7, 6, 5]) + C = torch.tensor(1) + result = torch.block_diag(torch.tensor([[4], [3], [2]]), + torch.tensor([7, 6, 5]), + torch.tensor(1)) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_3(): + pytorch_code = textwrap.dedent( + """ + import torch + A = torch.tensor([[4], [3], [2]]) + B = torch.tensor([[5, 6], [9, 1]]) + C = torch.tensor([1, 2, 3]) + result = torch.block_diag(A) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_4(): + pytorch_code = textwrap.dedent( + """ + import torch + A = torch.tensor([[4], [3], [2]]) + B = torch.tensor([[5], [6]]) + result = torch.block_diag(A, B, torch.tensor([1, 2, 3])) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_5(): + pytorch_code = textwrap.dedent( + """ + import torch + tensors = torch.tensor([[0,1,2]]), torch.tensor([[0],[1]]), torch.tensor([[20]]) + result = torch.block_diag(*tensors) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_6(): + pytorch_code = textwrap.dedent( + """ + import torch + result = torch.block_diag(torch.tensor([[4], [3], [2]])) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_7(): + """Test with expression arguments""" + pytorch_code = textwrap.dedent( + """ + import torch + A = torch.tensor([[1, 2], [3, 4]]) + B = torch.tensor([[5, 6], [7, 8]]) + result = torch.block_diag(A + 1, B * 2) + """ + ) obj.run(pytorch_code, ["result"]) diff --git a/tests/test_cartesian_prod.py b/tests/test_cartesian_prod.py index 625094d1c..f8113cfea 100644 --- a/tests/test_cartesian_prod.py +++ b/tests/test_cartesian_prod.py @@ -1,90 +1,90 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import textwrap - -from apibase import APIBase - -obj = APIBase("torch.cartesian_prod") - - -def test_case_1(): - pytorch_code = textwrap.dedent( - """ - import torch - a = torch.tensor([1, 2, 3]) - b = torch.tensor([5, 6]) - result = torch.cartesian_prod(a, b) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_2(): - pytorch_code = textwrap.dedent( - """ - import torch - a = torch.tensor([1, 2, 3]) - result = torch.cartesian_prod(a) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_3(): - pytorch_code = textwrap.dedent( - """ - import torch - result = torch.cartesian_prod(torch.tensor([1, 2, 4, 5]), torch.tensor([5, 6]), torch.tensor([7])) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_4(): - pytorch_code = textwrap.dedent( - """ - import torch - a = torch.tensor([1, 2, 3]) - b = torch.tensor([5, 6]) - c = (a, b) - result = torch.cartesian_prod(*c) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_5(): - pytorch_code = textwrap.dedent( - """ - import torch - a = torch.tensor([1, 2, 3]) - b = torch.tensor([5, 6]) - result = torch.cartesian_prod(*[a, b]) - """ - ) - obj.run(pytorch_code, ["result"]) - - -def test_case_6(): - """Test with expression arguments""" - pytorch_code = textwrap.dedent( - """ - import torch - a = torch.tensor([1, 2]) - b = torch.tensor([3, 4]) - result = torch.cartesian_prod(a + 1, b * 2) - """ - ) +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import textwrap + +from apibase import APIBase + +obj = APIBase("torch.cartesian_prod") + + +def test_case_1(): + pytorch_code = textwrap.dedent( + """ + import torch + a = torch.tensor([1, 2, 3]) + b = torch.tensor([5, 6]) + result = torch.cartesian_prod(a, b) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_2(): + pytorch_code = textwrap.dedent( + """ + import torch + a = torch.tensor([1, 2, 3]) + result = torch.cartesian_prod(a) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_3(): + pytorch_code = textwrap.dedent( + """ + import torch + result = torch.cartesian_prod(torch.tensor([1, 2, 4, 5]), torch.tensor([5, 6]), torch.tensor([7])) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_4(): + pytorch_code = textwrap.dedent( + """ + import torch + a = torch.tensor([1, 2, 3]) + b = torch.tensor([5, 6]) + c = (a, b) + result = torch.cartesian_prod(*c) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_5(): + pytorch_code = textwrap.dedent( + """ + import torch + a = torch.tensor([1, 2, 3]) + b = torch.tensor([5, 6]) + result = torch.cartesian_prod(*[a, b]) + """ + ) + obj.run(pytorch_code, ["result"]) + + +def test_case_6(): + """Test with expression arguments""" + pytorch_code = textwrap.dedent( + """ + import torch + a = torch.tensor([1, 2]) + b = torch.tensor([3, 4]) + result = torch.cartesian_prod(a + 1, b * 2) + """ + ) obj.run(pytorch_code, ["result"]) diff --git a/tests/test_nn_ConvTranspose2d.py b/tests/test_nn_ConvTranspose2d.py index d1947e5bb..b2f5c4567 100644 --- a/tests/test_nn_ConvTranspose2d.py +++ b/tests/test_nn_ConvTranspose2d.py @@ -256,4 +256,3 @@ def test_case_15(): """ ) obj.run(pytorch_code, ["result"], check_value=False) - diff --git a/tests/test_nn_ConvTranspose3d.py b/tests/test_nn_ConvTranspose3d.py index e7e0e9184..8db588815 100644 --- a/tests/test_nn_ConvTranspose3d.py +++ b/tests/test_nn_ConvTranspose3d.py @@ -239,4 +239,3 @@ def test_case_14(): """ ) obj.run(pytorch_code, ["result"], check_value=False) - diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index e617424da..0ffc700f6 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -1,21 +1,34 @@ # A image for testing PaConvert FROM registry.baidubce.com/paddlepaddle/paddle:latest-dev-cuda11.8-cudnn8.6-trt8.5-gcc82 -RUN apt-get update && \ - apt-get install -y net-tools +ENV PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple + +RUN apt-get -o Acquire::Retries=5 update || true; \ + apt-get -o Acquire::Retries=5 install -y net-tools || true RUN ln -sf `which python3.10` /usr/bin/python RUN ln -sf `which pip3.10` /usr/local/bin/pip -RUN python -m pip install astor +RUN python -m pip install --no-cache-dir \ + astor \ + pandas \ + openpyxl \ + pytest-timeout \ + pytest-rerunfailures \ + pytest-xdist -RUN python -m pip install -U torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 +RUN wget -q https://paddle-paconvert.bj.bcebos.com/torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl \ + https://paddle-paconvert.bj.bcebos.com/torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl && \ + python -m pip install --no-cache-dir \ + torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl \ + torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl && \ + rm -f torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl \ + torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl -RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb && \ +RUN wget --tries=5 --timeout=60 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb && \ dpkg -i cuda-keyring_1.1-1_all.deb && \ rm cuda-keyring_1.1-1_all.deb -RUN apt-get update && \ - apt-get install -y \ - cudnn9-cuda-11 +RUN apt-get -o Acquire::Retries=5 update || true; \ + apt-get -o Acquire::Retries=5 install -y cudnn9-cuda-11