Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .wordlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ CmpltRedir
CNP
DAC
DCQCN
DeepSeek
DirectTrans
DLB
DNS
DriveNets
DSCP
ECMP
ECN
Expand Down Expand Up @@ -87,22 +89,30 @@ acsctl
allreduce
balancer
baremetal
baselining
bd
bnxt
btl
cli
conf
currentColor
datapath
desc
egressing
frontend
gbits
hostfile
hostname
hostnames
hotspots
href
ibstat
ico
integrations
ip
libibverbs
linecap
linejoin
loopback
lspci
mca
Expand Down Expand Up @@ -134,17 +144,22 @@ png
proc
prometheus
rdma
rect
rvs
rx
ry
subnet
subnets
sudo
svg
sys
sysctl
tcp
td
topologies
ulimit
unroutable
viewBox
virbr
virtualized
vllm
Expand Down
14 changes: 7 additions & 7 deletions docs/.readthedocs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ build:
jobs:
post_checkout:
# Monorepo guard: cancel the build unless this change touched docs/
- git fetch origin develop --depth 1 || true
# Fetch develop's history (not just its tip) so the common ancestor with
# this branch is reachable, then review every commit since that ancestor.
- git fetch origin develop || true
- |
if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; then
BASE=origin/develop # PR build: compare against the develop base
else
BASE=HEAD^ # branch/tag build (latest): compare against previous commit
fi
if git diff --quiet "$BASE" HEAD -- docs/; then
BASE=$(git merge-base origin/develop HEAD || true)
# Fail open: if no common ancestor is found (e.g. develop history not
# fetched on a PR build), build rather than risk cancelling wrongly.
if [ -n "$BASE" ] && git diff --quiet "$BASE" HEAD -- docs/; then
exit 183
fi
python:
Expand Down
36 changes: 36 additions & 0 deletions docs/_static/css/custom.css
Original file line number Diff line number Diff line change
@@ -1,3 +1,39 @@
/* Disable the instinct-design theme's scroll-reveal animation: its JS sets
sections to opacity:0 until scrolled into view, which makes long pages
appear to "load in" on scroll. Force all sections fully visible so the
whole page renders at once (matching the old "instinct" flavor). */
.bd-main section[id] {
opacity: 1 !important;
transform: none !important;
}

/* Truncate landing-page tile text so long titles/descriptions don't make
cards uneven: clamp to a fixed number of lines with a trailing ellipsis. */
.id-card-title {
display: -webkit-box;
-webkit-line-clamp: 2;
line-clamp: 2;
-webkit-box-orient: vertical;
overflow: hidden;
text-overflow: ellipsis;
}
.id-card-desc {
display: -webkit-box;
-webkit-line-clamp: 3;
line-clamp: 3;
-webkit-box-orient: vertical;
overflow: hidden;
text-overflow: ellipsis;
}

/* Hide the primary sidebar toggle (carat) only when the primary sidebar is
empty. The theme adds `hide-on-wide` to the sidebar exactly when it has no
content (see pydata_sphinx_theme layout.html: `{% if not sidebars %}`), so
restoring a primary sidebar automatically brings the toggle back. */
.bd-container__inner:has(.bd-sidebar-primary.hide-on-wide) .sidebar-toggle.primary-toggle {
display: none;
}

/* Network troubleshooting page */

/* Custom width for rccl errors table */
Expand Down
12 changes: 8 additions & 4 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@
if os.environ.get("READTHEDOCS", "") == "True":
html_context["READTHEDOCS"] = True

project = "AMD Instinct Hub"
html_title = "GPU cluster networking documentation"
html_title = "Cluster Documentation Hub"
author = "Advanced Micro Devices, Inc."
copyright = "Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved."
copyright = "Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved."
version = "0.1.0"
release = version
setting_all_article_info = False
Expand All @@ -22,14 +21,19 @@

html_theme = "rocm_docs_theme"
html_theme_options = {
"flavor": "instinct",
"flavor": "instinct-design",
"link_main_doc": True,
"use_download_button": True,
}
extensions = ["rocm_docs"]

html_static_path = ['_static']

# Disable the sidebar on the landing page only; other pages keep the theme default
html_sidebars = {
"index": []
}

html_extra_path = ["llms.txt"]

EXCLUDED_DIRS = {
Expand Down
111 changes: 81 additions & 30 deletions docs/index.rst
Original file line number Diff line number Diff line change
@@ -1,40 +1,91 @@
.. meta::
:description: How to perform network validation testing on optimized hardware
:description: Central hub for AMD Instinct™ cluster design and deployment documentation.
:keywords: network validation, DCGPU, PCIe, Infiniband, RoCE, ROCm, RCCL, machine learning, LLM, usage, tutorial
:html_theme.sidebar_secondary.remove: true

********************************************************************
Cluster network performance validation for AMD Instinct accelerators
********************************************************************
.. raw:: html

When running HPC and AI applications in a cluster network environment, performance is only as fast as the slowest
individual node in the network. To achieve optimal performance, each server must be configured for maximum data transfer
rates and bandwidth utilization based on the available hardware. It is crucial to validate both host and device
performance in single-node and multi-node environments using the appropriate benchmarking tools.
<div class="id-hero-eyebrow">
<span class="id-hero-eyebrow-rule"></span>AMD Instinct™ — Cluster Design and Deployment
</div>

Refer to the relevant networking guides for step-by-step instructions on validating network configurations in
single-node and multi-node environments. These guides cover system settings, device configurations, networking tools,
and performance tests to ensure AMD Instinct™-powered GPU clusters achieve optimal speed and bandwidth during operation.
************************************************************************************************************************
Cluster Documentation Hub
************************************************************************************************************************

.. grid:: 2
:gutter: 3
.. raw:: html

.. grid-item-card:: How to
<style>
/* Landing page only: hide footers (this style ships only in index.html) */
.prev-next-footer,
.bd-footer-content,
.rocm-footer {
display: none;
}
</style>

* :doc:`Enabling MPI <how-to/gpu-enabled-mpi>`
* :doc:`Single-node network configuration <how-to/single-node-config>`
* :doc:`Multi-node network configuration <how-to/multi-node-config>`
* :doc:`RoCE network configuration <how-to/roce-network-config>`
* `Multi-node inference load balancing <./how-to/multi-node-inference-lb.html>`_
.. raw:: html

<div class="id-deck id-deck--primary">
<div class="id-deck-head">
<div class="id-section-label">Start here</div>
<h2 class="id-deck-title">Design and Guides</h2>
<p class="id-deck-sub">Configure, validate, and optimize AMD Instinct™-based clusters.</p>
</div>
<div class="id-grid">
<a class="id-card" href="https://instinct.docs.amd.com/projects/system-acceptance/en/latest/">
<span class="id-card-ico"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><rect width="8" height="4" x="8" y="2" rx="1" ry="1"/><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"/><path d="m9 14 2 2 4-4"/></svg></span>
<span class="id-card-title">Instinct Customer Acceptance Guide</span>
<span class="id-card-desc">Comprehensive guide for configuring, validating, benchmarking, and baselining AMD Instinct&trade;-based systems.</span>
<span class="id-card-go" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14"/><path d="m12 5 7 7-7 7"/></svg></span>
</a>
<a class="id-card" href="overview.html">
<span class="id-card-ico"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><rect x="16" y="16" width="6" height="6" rx="1"/><rect x="2" y="16" width="6" height="6" rx="1"/><rect x="9" y="2" width="6" height="6" rx="1"/><path d="M5 16v-3a1 1 0 0 1 1-1h12a1 1 0 0 1 1 1v3"/><path d="M12 12V8"/></svg></span>
<span class="id-card-title">Cluster Networking Guide</span>
<span class="id-card-desc">Optimize the network for AMD Instinct™ GPU applications.</span>
<span class="id-card-go" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14"/><path d="m12 5 7 7-7 7"/></svg></span>
</a>
<a class="id-card" href="https://instinct.docs.amd.com/projects/MI3XX-reference/latest/index.html">
<span class="id-card-ico"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><rect x="4" y="4" width="16" height="16" rx="2"/><rect x="9" y="9" width="6" height="6"/><path d="M9 2v2M15 2v2M9 20v2M15 20v2M20 9h2M20 14h2M2 9h2M2 14h2"/></svg></span>
<span class="id-card-title">MI3XX Reference Design</span>
<span class="id-card-desc">Reference design materials and topology examples for the AMD Instinct™ MI3XX platform.</span>
<span class="id-card-go" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14"/><path d="m12 5 7 7-7 7"/></svg></span>
</a>
<a class="id-card" href="https://instinct.docs.amd.com/projects/dc-design/latest/index.html">
<span class="id-card-ico"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M12 10h.01"/><path d="M12 14h.01"/><path d="M12 6h.01"/><path d="M16 10h.01"/><path d="M16 14h.01"/><path d="M16 6h.01"/><path d="M8 10h.01"/><path d="M8 14h.01"/><path d="M8 6h.01"/><path d="M9 22v-3a1 1 0 0 1 1-1h4a1 1 0 0 1 1 1v3"/><rect x="4" y="2" width="16" height="20" rx="2"/></svg></span>
<span class="id-card-title">Data Center Design Guide</span>
<span class="id-card-desc">How to plan and design the modern AI data center.</span>
<span class="id-card-go" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14"/><path d="m12 5 7 7-7 7"/></svg></span>
</a>
<a class="id-card" href="https://docs.amd.com/v/u/en-US/AMD_Instinct_AMD-DriveNets_System_Reference_Architecture_RF-72513">
<span class="id-card-ico"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M10 22V7a1 1 0 0 0-1-1H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-5a1 1 0 0 0-1-1H2"/><rect x="14" y="2" width="8" height="8" rx="1"/></svg></span>
<span class="id-card-title">AMD-DriveNets System Reference Architecture</span>
<span class="id-card-desc">Validated, end-to-end reference architecture for building and operating large-scale AI GPU clusters using AMD Instinct MI350-series (MI355X) compute paired with the DriveNets AI Fabric networking solution.</span>
<span class="id-card-go" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14"/><path d="m12 5 7 7-7 7"/></svg></span>
</a>
</div>
</div>

.. grid-item-card:: Reference
.. raw:: html

* :doc:`Hardware support <reference/hardware-support>`


.. note::

AMD Instinct systems vary in form and configuration, and cluster design adds additional layers of complexity. The
guidelines in this documentation are written at a high level for broad applicability across diverse environments.
While certain scenarios may include specific hardware examples, your setup will likely differ in terms of GPUs and
CPUs per server, firmware versions, and network interconnects. Adjustments might be necessary to align with your
particular configuration.
<div class="id-deck">
<div class="id-deck-head">
<div class="id-section-label">Reference</div>
<h2 class="id-deck-title">Articles and Overviews</h2>
<p class="id-deck-sub">Technical analyses and background reading that span AMD Instinct™ products.</p>
</div>
<div class="id-grid">
<a class="id-card" href="https://rocm.blogs.amd.com/artificial-intelligence/amd-comparative-analysis/README.html">
<span class="id-card-ico"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 3v18h18"/><path d="m19 9-5 5-4-4-3 3"/></svg></span>
<span class="id-card-title">Comparative Analysis of Scale-Out RoCE Network Traffic Patterns and Loads in Training Large Language Models</span>
<span class="id-card-desc">Compares RoCE network traffic patterns and loads across GPT-4, Llama 3, DeepSeek-V2, and Grok 4.0 LLM training.</span>
<span class="id-card-go" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14"/><path d="m12 5 7 7-7 7"/></svg></span>
</a>
<a class="id-card" href="https://rocm.blogs.amd.com/artificial-intelligence/amd-net-traffic/README.html">
<span class="id-card-ico"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="m3 16 4 4 4-4"/><path d="M7 20V4"/><path d="m21 8-4-4-4 4"/><path d="M17 4v16"/></svg></span>
<span class="id-card-title">AMD Instinct™ Network Traffic, Congestion Trends, and Harmonics in Scale-Out Networks for AI Training Clusters</span>
<span class="id-card-desc">Explore how synchronized GPU collectives create harmonic congestion in AI clusters and the strategies to diagnose and mitigate it.</span>
<span class="id-card-go" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14"/><path d="m12 5 7 7-7 7"/></svg></span>
</a>
</div>
</div>
40 changes: 40 additions & 0 deletions docs/overview.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
.. meta::
:description: How to perform network validation testing on optimized hardware
:keywords: network validation, DCGPU, PCIe, Infiniband, RoCE, ROCm, RCCL, machine learning, LLM, usage, tutorial

********************************************************************
Cluster network performance validation for AMD Instinct accelerators
********************************************************************

When running HPC and AI applications in a cluster network environment, performance is only as fast as the slowest
individual node in the network. To achieve optimal performance, each server must be configured for maximum data transfer
rates and bandwidth utilization based on the available hardware. It is crucial to validate both host and device
performance in single-node and multi-node environments using the appropriate benchmarking tools.

Refer to the relevant networking guides for step-by-step instructions on validating network configurations in
single-node and multi-node environments. These guides cover system settings, device configurations, networking tools,
and performance tests to ensure AMD Instinct™-powered GPU clusters achieve optimal speed and bandwidth during operation.

.. grid:: 2
:gutter: 3

.. grid-item-card:: How to

* :doc:`Enabling MPI <how-to/gpu-enabled-mpi>`
* :doc:`Single-node network configuration <how-to/single-node-config>`
* :doc:`Multi-node network configuration <how-to/multi-node-config>`
* :doc:`RoCE network configuration <how-to/roce-network-config>`
* `Multi-node inference load balancing <./how-to/multi-node-inference-lb.html>`_

.. grid-item-card:: Reference

* :doc:`Hardware support <reference/hardware-support>`


.. note::

AMD Instinct systems vary in form and configuration, and cluster design adds additional layers of complexity. The
guidelines in this documentation are written at a high level for broad applicability across diverse environments.
While certain scenarios may include specific hardware examples, your setup will likely differ in terms of GPUs and
CPUs per server, firmware versions, and network interconnects. Adjustments might be necessary to align with your
particular configuration.
3 changes: 3 additions & 0 deletions docs/sphinx/_toc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ defaults:
maxdepth: 6
root: index
subtrees:
- entries:
- file: overview.rst
title: Overview
- caption: How to
entries:
- file: how-to/gpu-enabled-mpi
Expand Down
3 changes: 3 additions & 0 deletions docs/sphinx/_toc.yml.in
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ defaults:
maxdepth: 6
root: index
subtrees:
- entries:
- file: overview.rst
title: Overview
- caption: How to
entries:
- file: how-to/gpu-enabled-mpi
Expand Down
31 changes: 31 additions & 0 deletions infrastructure/dc-design/.readthedocs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details

version: 2

build:
os: ubuntu-22.04
tools:
python: "3.10"
jobs:
post_checkout:
# Monorepo guard: cancel the build unless this change touched infrastructure/dc-design/
# Fetch develop's history (not just its tip) so the common ancestor with
# this branch is reachable, then review every commit since that ancestor.
- git fetch origin develop || true
- |
BASE=$(git merge-base origin/develop HEAD || true)
# Fail open: if no common ancestor is found (e.g. develop history not
# fetched on a PR build), build rather than risk cancelling wrongly.
if [ -n "$BASE" ] && git diff --quiet "$BASE" HEAD -- infrastructure/dc-design/; then
exit 183
fi
python:
install:
# Shared across all docsets; dependabot tracks this single file (.github/dependabot.yml)
- requirements: docs/sphinx/requirements.txt

sphinx:
configuration: infrastructure/dc-design/conf.py

formats: [htmlzip]
Loading
Loading