From 8909c4095719a22db7281a4c5f34fa42430a8fd1 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 23 Sep 2024 16:51:03 +0800 Subject: [PATCH 1/4] wip --- arg.json | 20 ++++++++++++++++++-- docker-bake.hcl | 5 ++++- pytorch/Dockerfile | 26 +++++++++++--------------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/arg.json b/arg.json index bc7c54c..1968693 100644 --- a/arg.json +++ b/arg.json @@ -150,18 +150,34 @@ ], "pytorch": [ { - "pytorch_version": "2.1.0", + "torch_version": "2.1.0", + "torch_vision_version": "0.16.0", + "torch_audio_version": "2.1.0", + "torch_npu_version": "2.1.0.post3", "cann_tag": "8.0.rc1-910b-ubuntu22.04-py3.8", "tags": [ "2.1.0" ] }, { - "pytorch_version": "2.2.0", + "torch_version": "2.2.0", + "torch_vision_version": "0.17.0", + "torch_audio_version": "2.2.0", + "torch_npu_version": "2.2.0", "cann_tag": "8.0.rc1-910b-ubuntu22.04-py3.8", "tags": [ "2.2.0" ] + }, + { + "torch_version": "nightly", + "torch_vision_version": "nightly", + "torch_audio_version": "nightly", + "torch_npu_version": "nightly", + "cann_tag": "8.0.rc3.alpha002-910b-ubuntu22.04-py3.9", + "tags": [ + "nightly" + ] } ], "mindspore": [ diff --git a/docker-bake.hcl b/docker-bake.hcl index c0ec684..6ec51cf 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -68,7 +68,10 @@ target "pytorch" { } args = { BASE_VERSION = "${item.cann_tag}" - PYTORCH_VERSION = "${item.pytorch_version}" + TORCH_VERSION = "${item.torch_version}" + TORCH_VISION_VERSION = "${item.torch_vision_version}" + TORCH_AUDIO_VERSION = "${item.torch_audio_version}" + TORCH_NPU_VERSION = "${item.torch_npu_version}" } tags = generate_tags("pytorch", "${item.tags}") } diff --git a/pytorch/Dockerfile b/pytorch/Dockerfile index 5bd2990..4902a42 100644 --- a/pytorch/Dockerfile +++ b/pytorch/Dockerfile @@ -2,30 +2,26 @@ ARG BASE_VERSION=latest FROM ascendai/cann:${BASE_VERSION} as official # Arguments -ARG PYTORCH_VERSION=2.2.0 +ARG TORCH_VERSION=2.1.0 +ARG TORCH_VISION_VERSION=0.16.0 +ARG TORCH_AUDIO_VERSION=2.1.0 +ARG TORCH_NPU_VERSION=2.1.0.post3 # Change the default shell SHELL [ "/bin/bash", "-c" ] # Install pytorch, torch-npu and related packages -RUN if [ "${PYTORCH_VERSION}" == "2.1.0" ]; then \ - TORCH_VISION_VERSION=0.16.0; \ - TORCH_AUDIO_VERSION=2.1.0; \ - TORCH_NPU_VERSION=2.1.0; \ - elif [ "${PYTORCH_VERSION}" == "2.2.0" ]; then \ - TORCH_VISION_VERSION=0.17.0; \ - TORCH_AUDIO_VERSION=2.2.0; \ - TORCH_NPU_VERSION=2.2.0; \ - else \ - echo "Not supported version: ${PYTORCH_VERSION}."; \ - exit 1; \ - fi && \ +RUN \ # Uninstall the latest numpy and sympy first, as the right versions will be installed again \ # after installing following packages \ pip uninstall -y numpy sympy && \ + # Nightly build \ + if [ ${TORCH_VERSION == "nightly"} ]; then \ + exit 0; \ + fi && \ pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cpu \ - torch==${PYTORCH_VERSION} \ + torch==${TORCH_VERSION} \ torchvision==${TORCH_VISION_VERSION} \ torchaudio==${TORCH_AUDIO_VERSION} && \ - pip install --no-cache-dir \ + pip install --no-cache-dir --index-url https://pypi.org/simple \ torch-npu==${TORCH_NPU_VERSION} From c32c5721ff348c71165f1eec74010a58ef11e44a Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Tue, 24 Sep 2024 09:14:34 +0800 Subject: [PATCH 2/4] wip --- pytorch/Dockerfile | 4 ---- pytorch/nightly.Dockerfile | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 pytorch/nightly.Dockerfile diff --git a/pytorch/Dockerfile b/pytorch/Dockerfile index 4902a42..353dc73 100644 --- a/pytorch/Dockerfile +++ b/pytorch/Dockerfile @@ -15,10 +15,6 @@ RUN \ # Uninstall the latest numpy and sympy first, as the right versions will be installed again \ # after installing following packages \ pip uninstall -y numpy sympy && \ - # Nightly build \ - if [ ${TORCH_VERSION == "nightly"} ]; then \ - exit 0; \ - fi && \ pip install --no-cache-dir --index-url https://download.pytorch.org/whl/cpu \ torch==${TORCH_VERSION} \ torchvision==${TORCH_VISION_VERSION} \ diff --git a/pytorch/nightly.Dockerfile b/pytorch/nightly.Dockerfile new file mode 100644 index 0000000..e81ada5 --- /dev/null +++ b/pytorch/nightly.Dockerfile @@ -0,0 +1,46 @@ +# Multi-arch builders, see https://github.com/docker/buildx/issues/805 +FROM pytorch/manylinuxaarch64-builder:cpu-aarch64-main AS builder-arm64 +FROM pytorch/manylinux-builder:cpu-main AS builder-amd64 +FROM builder-${TARGETARCH} AS builder + +#RUN apt-get update \ +# && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ +# git \ +# gcc-10 \ +# g++-10 \ +# make \ +# cmake \ +# ninja-build \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/* \ +# && rm -rf /var/tmp/* \ +# && rm -rf /tmp/* +# +## torch_npu requires gcc-10 and g++-10 +#RUN ln -s /usr/bin/gcc-10 /usr/bin/gcc && \ +# ln -s /usr/bin/g++-10 /usr/bin/g++ + +# Get source code and install dependencies +RUN git clone https://gitee.com/ascend/pytorch.git --depth 1 +RUN pip install --no-cache-dir wheel && \ + pip install --no-cache-dir -r pytorch/requirements.txt + +# The distribution package can be found at /pytorch/dist/torch_npu-xxx.whl +RUN py_version=$(python --version | awk '{print $2}' | cut -d '.' -f 1,2) && \ + bash pytorch/ci/build.sh --python=${py_version} + +ARG BASE_VERSION +FROM ascendai/cann:${BASE_VERSION} as official + +# Change the default shell +SHELL [ "/bin/bash", "-c" ] + +# Install pytorch, torch-npu and related packages +RUN \ + # Uninstall the latest numpy and sympy first, as the right versions will be installed again \ + # after installing following packages \ + pip uninstall -y numpy sympy && \ + # Nightly build \ + if [ ${TORCH_VERSION == "nightly"} ]; then \ + exit 0; \ + fi From 76ad0e2d38e67d17ef8a6f9bb9cde2dc45741aac Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Tue, 24 Sep 2024 10:06:48 +0800 Subject: [PATCH 3/4] ci: inherit secrets --- .github/workflows/docker.yml | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a5a9869..253c3d6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -56,13 +56,7 @@ jobs: set: | *.cache-from=type=gha *.cache-to=type=gha,mode=min - secrets: - DOCKERHUB_OWNER: ${{ secrets.DOCKERHUB_OWNER }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - QUAY_OWNER: ${{ secrets.QUAY_OWNER }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + secrets: inherit cann: name: cann @@ -72,13 +66,7 @@ jobs: with: target: 'cann' push: ${{ github.event_name == 'release' }} - secrets: - DOCKERHUB_OWNER: ${{ secrets.DOCKERHUB_OWNER }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - QUAY_OWNER: ${{ secrets.QUAY_OWNER }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + secrets: inherit pytorch: name: pytorch @@ -88,13 +76,7 @@ jobs: with: target: 'pytorch' push: ${{ github.event_name == 'release' }} - secrets: - DOCKERHUB_OWNER: ${{ secrets.DOCKERHUB_OWNER }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - QUAY_OWNER: ${{ secrets.QUAY_OWNER }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + secrets: inherit mindspore: name: mindspore @@ -104,10 +86,4 @@ jobs: with: target: 'mindspore' push: ${{ github.event_name == 'release' }} - secrets: - DOCKERHUB_OWNER: ${{ secrets.DOCKERHUB_OWNER }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - QUAY_OWNER: ${{ secrets.QUAY_OWNER }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + secrets: inherit From ec96b857685a0ef62b0ed6e0aa5c3cd2e77804e3 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Tue, 24 Sep 2024 19:23:02 +0800 Subject: [PATCH 4/4] wip --- pytorch/nightly.Dockerfile | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/pytorch/nightly.Dockerfile b/pytorch/nightly.Dockerfile index e81ada5..ada70e4 100644 --- a/pytorch/nightly.Dockerfile +++ b/pytorch/nightly.Dockerfile @@ -3,31 +3,13 @@ FROM pytorch/manylinuxaarch64-builder:cpu-aarch64-main AS builder-arm64 FROM pytorch/manylinux-builder:cpu-main AS builder-amd64 FROM builder-${TARGETARCH} AS builder -#RUN apt-get update \ -# && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ -# git \ -# gcc-10 \ -# g++-10 \ -# make \ -# cmake \ -# ninja-build \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/* \ -# && rm -rf /var/tmp/* \ -# && rm -rf /tmp/* -# -## torch_npu requires gcc-10 and g++-10 -#RUN ln -s /usr/bin/gcc-10 /usr/bin/gcc && \ -# ln -s /usr/bin/g++-10 /usr/bin/g++ - # Get source code and install dependencies +ARG PY_VERSION=3.9 RUN git clone https://gitee.com/ascend/pytorch.git --depth 1 -RUN pip install --no-cache-dir wheel && \ - pip install --no-cache-dir -r pytorch/requirements.txt +RUN python${PY_VERSION} -m pip install --no-cache-dir -r pytorch/requirements.txt # The distribution package can be found at /pytorch/dist/torch_npu-xxx.whl -RUN py_version=$(python --version | awk '{print $2}' | cut -d '.' -f 1,2) && \ - bash pytorch/ci/build.sh --python=${py_version} +RUN bash pytorch/ci/build.sh --python=${PY_VERSION} ARG BASE_VERSION FROM ascendai/cann:${BASE_VERSION} as official @@ -41,6 +23,3 @@ RUN \ # after installing following packages \ pip uninstall -y numpy sympy && \ # Nightly build \ - if [ ${TORCH_VERSION == "nightly"} ]; then \ - exit 0; \ - fi