From 92936a265f6bc4c2fb5a89f15fa2799528274840 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 10 Feb 2025 09:49:23 +0800 Subject: [PATCH 1/5] wip --- .github/workflows/docker.yml | 8 ++++---- CONTRIBUTING_zh.md | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 CONTRIBUTING_zh.md diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7f429a5..f9687ce 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -72,7 +72,7 @@ jobs: name: python needs: - prepare - if: ${{ github.event_name != 'pull_request' || (!cancelled() && contains(needs.prepare.outputs.pr-labels, 'python')) }} + if: ${{ !cancelled() && (github.event_name != 'pull_request' || contains(needs.prepare.outputs.pr-labels, 'python')) }} uses: ./.github/workflows/_docker-build.yml with: target: 'python' @@ -88,7 +88,7 @@ jobs: needs: - prepare - python - if: ${{ github.event_name != 'pull_request' || (!cancelled() && contains(needs.prepare.outputs.pr-labels, 'cann')) }} + if: ${{ !cancelled() && (github.event_name != 'pull_request' || contains(needs.prepare.outputs.pr-labels, 'cann')) }} uses: ./.github/workflows/_docker-build.yml with: target: 'cann' @@ -100,7 +100,7 @@ jobs: needs: - prepare - cann - if: ${{ github.event_name != 'pull_request' || (!cancelled() && contains(needs.prepare.outputs.pr-labels, 'pytorch')) }} + if: ${{ !cancelled() && (github.event_name != 'pull_request' || contains(needs.prepare.outputs.pr-labels, 'pytorch')) }} uses: ./.github/workflows/_docker-build.yml with: target: 'pytorch' @@ -112,7 +112,7 @@ jobs: needs: - prepare - cann - if: ${{ github.event_name != 'pull_request' || (!cancelled() && contains(needs.prepare.outputs.pr-labels, 'mindspore')) }} + if: ${{ !cancelled() && (github.event_name != 'pull_request' || contains(needs.prepare.outputs.pr-labels, 'mindspore')) }} uses: ./.github/workflows/_docker-build.yml with: target: 'mindspore' diff --git a/CONTRIBUTING_zh.md b/CONTRIBUTING_zh.md new file mode 100644 index 0000000..f9304e3 --- /dev/null +++ b/CONTRIBUTING_zh.md @@ -0,0 +1,24 @@ + +# 贡献指南 + +## 代码结构 + +``` +. +├── arg.json # 构建参数矩阵 +├── cann/ # CANN 镜像目录 +├── docker-bake.hcl # Bake 配置 +├── mindspore/ # MindSpore 镜像目录 +├── python/ # Python 镜像目录 +└── pytorch/ # PyTorch 镜像目录 +``` + +```plantuml +@startuml +Alice -> Bob: Hello +Bob -> Alice: Hi! +@enduml +``` + +## + From 0fc824baac0d1e7cc83560b73c496f79466973a5 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 10 Feb 2025 09:56:09 +0800 Subject: [PATCH 2/5] wip --- CONTRIBUTING_zh.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING_zh.md b/CONTRIBUTING_zh.md index f9304e3..d1ab2de 100644 --- a/CONTRIBUTING_zh.md +++ b/CONTRIBUTING_zh.md @@ -13,11 +13,12 @@ └── pytorch/ # PyTorch 镜像目录 ``` -```plantuml -@startuml -Alice -> Bob: Hello -Bob -> Alice: Hi! -@enduml +```mermaid + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; ``` ## From 9f35c92864dedfd577d8c94518fbc3e1f091a44e Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 10 Feb 2025 10:56:42 +0800 Subject: [PATCH 3/5] wip --- CONTRIBUTING_zh.md | 50 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING_zh.md b/CONTRIBUTING_zh.md index d1ab2de..0541f8d 100644 --- a/CONTRIBUTING_zh.md +++ b/CONTRIBUTING_zh.md @@ -1,4 +1,3 @@ - # 贡献指南 ## 代码结构 @@ -13,13 +12,50 @@ └── pytorch/ # PyTorch 镜像目录 ``` +镜像关系如下图所示。 + +- [python](./python): 由于 CANN Toolkit 依赖 Python 环境， + 因此单独构建 Python 镜像作为 CANN 镜像的基础镜像，其 Python 包采用源码构建而成， + 详情见 [python.sh](./python/python.sh)。 + +- [cann](./cann): CANN 镜像中安装了 `ascend-cann-toolkit` 和 `ascend-cann-kernels` + 以及运行时必要的依赖，详情见 [cann.sh](./cann/cann.sh)。 + +- [pytorch](./pytorch): PyTorch 镜像将 CANN 镜像作为基础镜像， + 同时安装了 `torch` 和 `torch_npu`，详情见 [Dockerfile](./pytorch/Dockerfile)。 + +- [mindspore](./mindspore): MindSpore 镜像将 CANN 镜像作为基础镜像， + 同时安装了 `mindspore`，详情见 [Dockerfile](./mindspore/Dockerfile)。 + ```mermaid - graph TD; - A-->B; - A-->C; - B-->D; - C-->D; +graph TD; + Python-->CANN; + CANN-->PyTorch; + CANN-->MindSpore; ``` -## +## 镜像构建 + +提供两种镜像构建方式：使用 [Bake][0] 构建和使用传统的 `docker build` 构建。 + +- Bake 构建方式：CI/CD 中使用 Bake 来构建镜像，以此简化构建配置。 + Bake 配置文件见 [docker-bake.hcl](./docker-bake.hcl) 和 [arg.json](./arg.json)。 + 其中 `docker-bake.hcl` 中定义了各类镜像的构建配置，`arg.json` 中定义了各类镜像的构建参数。 + +- 传统构建方式：对于需要自定义构建镜像的用户，可以选择使用传统的 `docker build` 方式。 + +[0]: https://docs.docker.com/build/bake/ + +## CI/CD + +基于 GitHub Actions 构建 CI/CD 系统，运行在 [GitHub-hosted runner][10] 上。 + +提交 Pull request 时可触发镜像构建，只有当 release 时才会真正发布镜像。 +工作流运行详情见：[actions/workflows/docker.yml][11]。 + +> [!NOTE] +> 为节省服务器资源，只有 Pull request 中添加了对应的 label，才会构建相应的镜像。 +[10]: https://github.com/actions/runner-images +[11]: https://github.com/openmerlin/dockerfile/actions/workflows/docker.yml +[12]: https://github.com/openmerlin/dockerfile/labels From 43ac5f02d809493e011c9dc5564bf43a80bc16b3 Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 10 Feb 2025 11:14:09 +0800 Subject: [PATCH 4/5] wip --- CONTRIBUTING_zh.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING_zh.md b/CONTRIBUTING_zh.md index 0541f8d..9c12aef 100644 --- a/CONTRIBUTING_zh.md +++ b/CONTRIBUTING_zh.md @@ -50,11 +50,11 @@ graph TD; 基于 GitHub Actions 构建 CI/CD 系统，运行在 [GitHub-hosted runner][10] 上。 -提交 Pull request 时可触发镜像构建，只有当 release 时才会真正发布镜像。 +提交 Pull request 时可触发镜像构建，只有当 release 时才会发布镜像。 工作流运行详情见：[actions/workflows/docker.yml][11]。 > [!NOTE] -> 为节省服务器资源，只有 Pull request 中添加了对应的 label，才会构建相应的镜像。 +> 为节省服务器资源，只有 Pull request 中添加了对应的 [label][12]，才会构建相应的镜像。 [10]: https://github.com/actions/runner-images [11]: https://github.com/openmerlin/dockerfile/actions/workflows/docker.yml From 7acca6db1472ea3267b6e8520f8c7bc8e15b5bab Mon Sep 17 00:00:00 2001 From: Yuanhao Ji Date: Mon, 10 Feb 2025 15:26:10 +0800 Subject: [PATCH 5/5] cann --- cann/cann.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cann/cann.sh b/cann/cann.sh index ac10f6f..be95cac 100755 --- a/cann/cann.sh +++ b/cann/cann.sh @@ -2,7 +2,7 @@ set -e -get_architecture() { +_get_architecture() { # not case sensitive shopt -s nocasematch @@ -22,6 +22,10 @@ get_architecture() { echo "${ARCH}" } +_retry() { + "$@" || (sleep 10 && "$@") || (sleep 20 && "$@") || (sleep 40 && "$@") +} + download_file() { set +e @@ -59,9 +63,12 @@ download_cann() { else local url_prefix="${url}/CANN/CANN%20${CANN_VERSION}" fi - local url_suffix="response-content-type=application/octet-stream" - local toolkit_url="${url_prefix}/${TOOLKIT_FILE}?${url_suffix}" - local kernels_url="${url_prefix}/${KERNELS_FILE}?${url_suffix}" + # local url_suffix="response-content-type=application/octet-stream" + # local toolkit_url="${url_prefix}/${TOOLKIT_FILE}?${url_suffix}" + # local kernels_url="${url_prefix}/${KERNELS_FILE}?${url_suffix}" + + local toolkit_url="${url_prefix}/${TOOLKIT_FILE}" + local kernels_url="${url_prefix}/${KERNELS_FILE}" if [ ! -f "${TOOLKIT_PATH}" ]; then download_file "${toolkit_url}" "${TOOLKIT_PATH}" @@ -121,7 +128,7 @@ install_cann() { } PLATFORM=${PLATFORM:=$(uname -s)/$(uname -m)} -ARCH=$(get_architecture) +ARCH=$(_get_architecture) CANN_HOME=${CANN_HOME:="/usr/local/Ascend"} CANN_CHIP=${CANN_CHIP:="910b"} CANN_VERSION=${CANN_VERSION:="8.0.RC1"}