Sync/upstream 20251216

.github/workflows/pre-check.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Build run package
         run: |
           docker buildx build \
-            -f docker/Dockerfile.cuda12.x.ubuntu20.04 \
+            -f docker/Dockerfile.cuda13.x.ubuntu22.04 \
             --platform linux/amd64 \
             --target package \
             --output type=local,dest=dist \

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Release run installer
+name: release
 
 on:
   push:
@@ -9,13 +9,24 @@ permissions:
   contents: write
 
 jobs:
-  build-release:
+  build-and-release:
+    name: build-run (${{ matrix.name }})
     runs-on: ubuntu-22.04
 
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: cuda12.9-ubuntu22.04
+            dockerfile: docker/Dockerfile.cuda12.9.ubuntu22.04
+          - name: cuda13-ubuntu22.04
+            dockerfile: docker/Dockerfile.cuda13.x.ubuntu22.04
+
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout source
+        uses: actions/checkout@v4
 
-      - name: Setup Docker Buildx
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       - name: Free disk space
@@ -27,20 +38,23 @@ jobs:
           sudo docker system prune -af || true
           df -h
 
-      - name: Build .run installer (Docker)
+      - name: Build run package
         run: |
+          BUILD_DATE=$(date +%Y%m%d)
+
           docker buildx build \
-            -f docker/Dockerfile.cuda12.x.ubuntu20.04 \
             --platform linux/amd64 \
-            --build-arg BUILD_DATE=$(date +%Y%m%d) \
-            --output type=local,dest=dist \
+            --progress=plain \
+            -f ${{ matrix.dockerfile }} \
+            --build-arg BUILD_DATE=${BUILD_DATE} \
+            --output type=local,dest=dist/${{ matrix.name }} \
             .
 
-      - name: List artifacts
-        run: ls -lh dist
+          echo "Produced files:"
+          ls -lh dist/${{ matrix.name }}
 
-      - name: Upload to GitHub Release
+      - name: Upload run to GitHub Release
         uses: softprops/action-gh-release@v2
         with:
           files: |
-            dist/*.run
+            dist/${{ matrix.name }}/*.run

README.md

@@ -4,33 +4,43 @@ These tests check both the performance and the correctness of [NCCL](http://gith
 
 ## Build
 
-To build the tests, just type `make`.
+To build the tests, just type `make` or `make -j`
 
-If CUDA is not installed in /usr/local/cuda, you may specify CUDA\_HOME. Similarly, if NCCL is not installed in /usr, you may specify NCCL\_HOME.
+If CUDA is not installed in `/usr/local/cuda`, you may specify `CUDA_HOME`. Similarly, if NCCL is not installed in `/usr`, you may specify `NCCL_HOME`.
 
 ```shell
 $ make CUDA_HOME=/path/to/cuda NCCL_HOME=/path/to/nccl
 ```
 
-NCCL tests rely on MPI to work on multiple processes, hence multiple nodes. If you want to compile the tests with MPI support, you need to set MPI=1 and set MPI\_HOME to the path where MPI is installed.
+NCCL tests rely on MPI to work on multiple processes, hence multiple nodes. If you want to compile the tests with MPI support, you need to set `MPI=1` and set `MPI_HOME` to the path where MPI is installed.
 
 ```shell
 $ make MPI=1 MPI_HOME=/path/to/mpi CUDA_HOME=/path/to/cuda NCCL_HOME=/path/to/nccl
 ```
 
+You can also add a suffix to the name of the generated binaries with `NAME_SUFFIX`. For example when compiling with the MPI versions you could use:
+
+```shell
+$ make MPI=1 NAME_SUFFIX=_mpi MPI_HOME=/path/to/mpi CUDA_HOME=/path/to/cuda NCCL_HOME=/path/to/nccl
+```
+
+This will generate test binaries with names such as `all_reduce_perf_mpi`.
+
 ## Usage
 
-NCCL tests can run on multiple processes, multiple threads, and multiple CUDA devices per thread. The number of process is managed by MPI and is therefore not passed to the tests as argument. The total number of ranks (=CUDA devices) will be equal to (number of processes)\*(number of threads)\*(number of GPUs per thread).
+NCCL tests can run on multiple processes, multiple threads, and multiple CUDA devices per thread. The number of process is managed by MPI and is therefore not passed to the tests as argument. The total number of ranks (=CUDA devices) will be equal to `(number of processes)*(number of threads)*(number of GPUs per thread)`.
 
 ### Quick examples
 
 Run on single node with 8 GPUs (`-g 8`), scanning from 8 Bytes to 128MBytes :
+
 ```shell
 $ ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 8
 ```
 
 Run 64 MPI processes on nodes with 8 GPUs each, for a total of 64 GPUs spread across 8 nodes :
 (NB: The nccl-tests binaries must be compiled with `MPI=1` for this case)
+
 ```shell
 $ mpirun -np 64 -N 8 ./build/all_reduce_perf -b 8 -e 8G -f 2 -g 1
 ```
@@ -58,7 +68,7 @@ All tests support the same set of arguments :
   * `-r,--root <root/all>` Specify which root to use. Only for operations with a root like broadcast or reduce. Default : 0.
 * Performance
   * `-n,--iters <iteration count>` number of iterations. Default : 20.
-  * `-w,--warmup_iters <warmup iteration count>` number of warmup iterations (not timed). Default : 5.
+  * `-w,--warmup_iters <warmup iteration count>` number of warmup iterations (not timed). Default : 1.
   * `-m,--agg_iters <aggregation count>` number of operations to aggregate together in each iteration. Default : 1.
   * `-N,--run_cycles <cycle count>` run & print each cycle. Default : 1; 0=infinite.
   * `-a,--average <0/1/2/3>` Report performance as an average across all ranks (MPI=1 only). <0=Rank0,1=Avg,2=Min,3=Max>. Default : 1.
@@ -67,11 +77,32 @@ All tests support the same set of arguments :
   * `-c,--check <check iteration count>` perform count iterations, checking correctness of results on each iteration. This can be quite slow on large numbers of GPUs. Default : 1.
   * `-z,--blocking <0/1>` Make NCCL collective blocking, i.e. have CPUs wait and sync after each collective. Default : 0.
   * `-G,--cudagraph <num graph launches>` Capture iterations as a CUDA graph and then replay specified number of times. Default : 0.
-  * `-C,--report_cputime <0/1>]` Report CPU time instead of latency. Default : 0.
-  * `-R,--local_register <1/0>` enable local buffer registration on send/recv buffers. Default : 0.
+  * `-C,--report_cputime <0/1>` Report CPU time instead of latency. Default : 0.
+  * `-R,--local_register <0/1/2>` enable local (1) or symmetric (2) buffer registration on send/recv buffers. Default : 0.
+  * `-S,--report_timestamps <0/1>` Add timestamp (`"%Y-%m-%d %H:%M:%S"`) to each performance report line. Default : 0.
+  * `-J,--output_file <file>` Write [JSON] output to filepath. Infer type from suffix (only `json` supported presently).
   * `-T,--timeout <time in seconds>` timeout each test after specified number of seconds. Default : disabled.
 
-## Copyright
+### Running multiple operations in parallel
+
+NCCL tests allow to partition the set of GPUs into smaller sets, each executing the same operation in parallel.
+To split the GPUs, NCCL will compute a "color" for each rank, based on the `NCCL_TESTS_SPLIT` environment variable, then all ranks
+with the same color will end up in the same group. The resulting group is printed next to each GPU at the beginning of the test.
 
-NCCL tests are provided under the BSD license. All source code and accompanying documentation is copyright (c) 2016-2024, NVIDIA CORPORATION. All rights reserved.
+`NCCL_TESTS_SPLIT` takes the following syntax: `<operation><value>`. Operation can be `AND`, `OR`, `MOD` or `DIV`. The `&`, `|`, `%`, and `/` symbols are also supported. The value can be either decimal, hexadecimal (prefixed by `0x`) or binary (prefixed by `0b`).
+
+`NCCL_TESTS_SPLIT_MASK="<value>"` is equivalent to `NCCL_TESTS_SPLIT="&<value>"`.
+
+Here are a few examples:
+
+ - `NCCL_TESTS_SPLIT="AND 0x7"` or `NCCL_TESTS_SPLIT="MOD 8"`: On systems with 8 GPUs, run 8 parallel operations, each with 1 GPU per node (purely communicating over the inter-node network)
+
+- `NCCL_TESTS_SPLIT="OR 0x7"` or `NCCL_TESTS_SPLIT="DIV 8"`: On systems with 8 GPUs, run one operation per node, purely intra-node.
+
+- `NCCL_TESTS_SPLIT="AND 0x1"` or `NCCL_TESTS_SPLIT="MOD 2"`: Run two operations, each operation using every other rank.
+
+Note that the reported bandwidth is per group, hence to get the total bandwidth used by all groups, one must multiply by the number of groups.
+
+## Copyright
 
+NCCL tests are provided under the BSD license. All source code and accompanying documentation is copyright (c) 2016-2025, NVIDIA CORPORATION. All rights reserved.

docker/Dockerfile.cuda12.9.ubuntu22.04

@@ -0,0 +1,153 @@
+###########################
+# Build-time configuration
+###########################
+
+# Base OS and CUDA versions
+ARG UBUNTU_VERSION=22.04
+ARG CUDA_VERSION=12.9.1
+ARG CUDART_VERSION=12.9.79
+ARG CUDART_MAJOR_VERSION=12
+
+# NCCL versions
+ARG NCCL_PACKAGE_VERSION=2.29.2-1+cuda12.9
+ARG NCCL_SO_VERSION=2.29.2
+
+# OpenMPI versions
+# - MPI_VERSION: full OpenMPI version
+# - MPI_SERIES: major.minor series used in download URL
+ARG MPI_VERSION=4.1.8
+ARG MPI_SERIES=4.1
+
+# Build date (override at build time)
+ARG BUILD_DATE=20260128
+
+###########################
+# Build Stage
+###########################
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build
+
+# Re-declare build args for this stage (values are inherited)
+ARG UBUNTU_VERSION
+ARG CUDA_VERSION
+ARG CUDART_VERSION
+ARG NCCL_PACKAGE_VERSION
+ARG NCCL_SO_VERSION
+ARG MPI_VERSION
+ARG MPI_SERIES
+ARG BUILD_DATE
+
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /workspace
+
+# -------------------------
+# 1. Base build dependencies
+# -------------------------
+RUN { apt-get -o Acquire::http::No-Cache=true update > build.log 2>&1 && \
+    apt-get install -y --no-install-recommends \
+        build-essential gcc g++ curl git wget ca-certificates \
+        make automake autoconf libtool pkg-config \
+        python3 python3-pip gzip xz-utils >> build.log 2>&1 && \
+    rm -rf /var/lib/apt/lists/* && rm -f build.log; } || (cat build.log && false)
+
+# -------------------------
+# 2. Install CUDA keyring and restore NVIDIA repository
+# -------------------------
+RUN { wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb > build.log 2>&1 && \
+    dpkg -i cuda-keyring_1.1-1_all.deb >> build.log 2>&1 && \
+    apt-get update >> build.log 2>&1 && rm -f build.log; } || (cat build.log && false)
+
+# -------------------------
+# 3. Install NCCL (pinned version)
+# -------------------------
+RUN { apt-get install -y --no-install-recommends \
+    --allow-change-held-packages \
+    libnccl2=${NCCL_PACKAGE_VERSION} \
+    libnccl-dev=${NCCL_PACKAGE_VERSION} > build.log 2>&1 && \
+    apt-mark hold libnccl2 libnccl-dev >> build.log 2>&1 && \
+    ldconfig >> build.log 2>&1 && \
+    rm -rf /var/lib/apt/lists/* && rm -f build.log; } || (cat build.log && false)
+
+# -------------------------
+# 4. Build OpenMPI from source
+# -------------------------
+RUN wget https://download.open-mpi.org/release/open-mpi/v${MPI_SERIES}/openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \
+    tar zxvf openmpi-${MPI_VERSION}.tar.gz > /dev/null 2>&1 && \
+    cd openmpi-${MPI_VERSION} && \
+    ./configure --prefix=/usr/local/sihpc --with-cuda=/usr/local/cuda > /dev/null  2>&1 && \
+    make -j$(nproc) > /dev/null 2>&1 && make install > /dev/null 2>&1 && \
+    rm -rf /workspace/openmpi-${MPI_VERSION} /workspace/openmpi-${MPI_VERSION}.tar.gz
+
+# -------------------------
+# 5. Build nccl-tests
+# -------------------------
+RUN git clone --depth 1 --single-branch -b sync/upstream-20251216 https://github.com/scitix/nccl-tests.git > /dev/null 2>&1 && \
+    cd nccl-tests && \
+    { make MPI=1 MPI_HOME=/usr/local/sihpc > build.log 2>&1 && rm -f build.log || (cat build.log && false); } && \
+    mkdir -p /usr/local/sihpc/libexec/nccl-tests && \
+    cp -rf build/*_perf /usr/local/sihpc/libexec/nccl-tests/ && \
+    mkdir -p /usr/local/sihpc/bin && \
+    cp scripts/nccl_perf /usr/local/sihpc/bin/nccl_perf && \
+    cp scripts/nccl_test /usr/local/sihpc/libexec/nccl-tests/nccl_test && \
+    cp scripts/env.sh /usr/local/sihpc/env.sh && \
+    cp scripts/install_sihpc /usr/local/sihpc/bin/install_sihpc && \
+    cp scripts/uninstall_sihpc /usr/local/sihpc/bin/uninstall_sihpc && \
+    rm -rf /workspace/nccl-tests
+
+# -------------------------
+# 6. Collect runtime libraries (strict selection)
+# -------------------------
+RUN { set -e && \
+    mkdir -p /usr/local/sihpc/lib > build.log 2>&1 && \
+    cp /usr/local/cuda/lib64/libcudart* /usr/local/sihpc/lib/ >> build.log 2>&1 && \
+    cp /usr/lib/x86_64-linux-gnu/libnccl.so* /usr/local/sihpc/lib/ >> build.log 2>&1 && \
+    rm -f build.log; } || (cat build.log && false)
+#    cp /lib/x86_64-linux-gnu/libltdl.so.7.3.1 /usr/local/sihpc/lib/ && \
+#    cp /usr/lib/x86_64-linux-gnu/libhwloc.so* /usr/local/sihpc/lib/ && \
+#    cp /usr/lib/x86_64-linux-gnu/libevent_core* /usr/local/sihpc/lib/ && \
+#    cp /usr/lib/x86_64-linux-gnu/libevent_pthreads* /usr/local/sihpc/lib/
+
+# -------------------------
+# 7. Fix library symlinks
+# -------------------------
+RUN cd /usr/local/sihpc/lib && \
+    rm -f libcudart.so libcudart.so.${CUDART_MAJOR_VERSION} && \
+    ln -sf libnccl.so.${NCCL_SO_VERSION} libnccl.so.2 && \
+    ln -sf libnccl.so.2 libnccl.so && \
+    ln -sf libcudart.so.${CUDART_VERSION} libcudart.so.${CUDART_MAJOR_VERSION} && \
+    ln -sf libcudart.so.${CUDART_MAJOR_VERSION} libcudart.so
+#    rm -f libevent_core-2.1.so.7 && \
+#    ln -sf libhwloc.so.15.1.0 libhwloc.so.15 && \
+#    ln -sf libhwloc.so.15.1.0 libhwloc.so && \
+#    ln -sf libevent_core-2.1.so.7.0.0 libevent_core-2.1.so.7 && \
+#    ln -sf libevent_core-2.1.so.7 libevent_core-2.1.so && \
+#    ln -sf libevent_pthreads-2.1.so.7.0.0 libevent_pthreads-2.1.so.7 && \
+#    ln -sf libevent_pthreads-2.1.so.7 libevent_pthreads-2.1.so && \
+#    ln -sf libltdl.so.7.3.1 libltdl.so.7 && \
+#    ln -sf libltdl.so.7 libltdl.so
+
+###########################
+# Package Stage
+###########################
+FROM ubuntu:${UBUNTU_VERSION} AS package
+
+# Re-declare args for this stage (values are inherited)
+ARG UBUNTU_VERSION
+ARG NCCL_PACKAGE_VERSION
+ARG MPI_VERSION
+ARG BUILD_DATE
+
+# Expose versions/date as environment variables for runtime shell expansion
+ENV NCCL_PACKAGE_VERSION=${NCCL_PACKAGE_VERSION} \
+    MPI_VERSION=${MPI_VERSION} \
+    BUILD_DATE=${BUILD_DATE}
+
+COPY --from=build /usr/local/sihpc /usr/local/sihpc
+
+WORKDIR /
+RUN apt-get update && apt-get install -y --no-install-recommends makeself && \
+    SAFE_NCCL_PKG=$(printf '%s\n' "${NCCL_PACKAGE_VERSION}" | tr '+' '-') && \
+    PACKAGE_FILENAME="sicl-nccl${SAFE_NCCL_PKG}-ompi${MPI_VERSION}-ubuntu${UBUNTU_VERSION}-${BUILD_DATE}.run" && \
+    { makeself --gzip /usr/local/sihpc \
+      "${PACKAGE_FILENAME}" \
+      "SiHPC MPI + NCCL + NCCL-tests Portable Installer" \
+      ./bin/install_sihpc > build.log 2>&1 && rm -f build.log; } || (cat build.log && false)