diff --git a/deployments/glm52-sparse-ckv/.env.example b/deployments/glm52-sparse-ckv/.env.example new file mode 100644 index 000000000000..a9cdbb2a38d9 --- /dev/null +++ b/deployments/glm52-sparse-ckv/.env.example @@ -0,0 +1,7 @@ +IMAGE_NAME=fujitsupolycom/glm52-sparse-ckv:v19-20260721 +MODEL_PATH=/srv/ai/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid +HF_CACHE_PATH=/srv/ai/huggingface +CACHE_PATH=/srv/ai/cache/glm52-sparse-ckv-v19 +TMP_PATH=/srv/ai/tmp/glm52-sparse-ckv-v19 +PORT=5802 +CUDA_VISIBLE_DEVICES=0,1,2,3 diff --git a/deployments/glm52-sparse-ckv/Dockerfile b/deployments/glm52-sparse-ckv/Dockerfile new file mode 100644 index 000000000000..4dace9e677e5 --- /dev/null +++ b/deployments/glm52-sparse-ckv/Dockerfile @@ -0,0 +1,106 @@ +ARG BASE_IMAGE=voipmonitor/vllm@sha256:41078d5fac36b802e0f3798057a4b383c24296cb8570f6f4fc138c34d8cfa303 + +FROM ${BASE_IMAGE} AS sources + +ARG VLLM_REPOSITORY=https://github.com/FujitsuPolycom/vllm.git +ARG VLLM_BASE_REF=7ea567a2458a4800a6a0e3e0a6ba41fcbd00d146 +ARG VLLM_REF=a81a0811314e23bc6b2baa69f34247aacea71e23 +ARG SPARKINFER_REPOSITORY=https://github.com/FujitsuPolycom/sparkinfer.git +ARG SPARKINFER_REF=a82fc76f9a2321e3cfd3065a9fa78452a27ddf9f + +USER root + +RUN set -eux; \ + git clone --filter=blob:none "${VLLM_REPOSITORY}" /src/vllm; \ + git -C /src/vllm checkout --detach "${VLLM_REF}"; \ + test "$(git -C /src/vllm rev-parse HEAD)" = "${VLLM_REF}"; \ + git -C /src/vllm cat-file -e "${VLLM_BASE_REF}^{commit}"; \ + mkdir -p /src/vllm-overlay; \ + git -C /src/vllm diff --name-only --diff-filter=ACMRTUXB \ + "${VLLM_BASE_REF}..${VLLM_REF}" -- vllm \ + | while IFS= read -r path; do \ + mkdir -p "/src/vllm-overlay/$(dirname "${path}")"; \ + cp "/src/vllm/${path}" "/src/vllm-overlay/${path}"; \ + done; \ + test "$(find /src/vllm-overlay/vllm -type f | wc -l)" = "39"; \ + git clone --filter=blob:none "${SPARKINFER_REPOSITORY}" /src/sparkinfer; \ + git -C /src/sparkinfer checkout --detach "${SPARKINFER_REF}"; \ + test "$(git -C /src/sparkinfer rev-parse HEAD)" = "${SPARKINFER_REF}"; \ + python -m pip wheel --no-cache-dir --no-deps \ + --wheel-dir /dist /src/sparkinfer + +FROM ${BASE_IMAGE} + +ARG VLLM_REF=a81a0811314e23bc6b2baa69f34247aacea71e23 +ARG SPARKINFER_REF=a82fc76f9a2321e3cfd3065a9fa78452a27ddf9f +ARG RUN_CPU_TESTS=1 + +USER root + +COPY --from=sources /dist/ /opt/release/dist/ +COPY --from=sources /src/vllm-overlay/vllm/ /opt/release/vllm/ +COPY --from=sources /src/vllm/tests/ /opt/release-tests/vllm-tests/ +COPY --from=sources /src/sparkinfer/tests/ /opt/release-tests/sparkinfer-tests/ + +RUN set -eux; \ + python -m pip install --no-cache-dir --upgrade \ + nvidia-cutlass-dsl==4.6.0 \ + nvidia-cutlass-dsl-libs-base==4.6.0 \ + nvidia-cutlass-dsl-libs-core==4.6.0 \ + nvidia-cutlass-dsl-libs-cu12==4.6.0 \ + nvidia-cutlass-dsl-libs-cu13==4.6.0 \ + quack-kernels==0.6.1 \ + pytest==8.4.2; \ + python -m pip install --no-cache-dir --no-deps --force-reinstall \ + /opt/release/dist/sparkinfer-*.whl; \ + for root in \ + /opt/venv/lib/python3.12/site-packages/vllm \ + /opt/vllm/vllm; do \ + test -d "${root}"; \ + cp -a /opt/release/vllm/. "${root}/"; \ + done; \ + python -c 'from sparkinfer.comm.pcie import SelectedRecordCopyExchange, SelectedRecordExchange, SelectedRecordExchangeInitializationError; assert SelectedRecordCopyExchange and SelectedRecordExchange and SelectedRecordExchangeInitializationError'; \ + test -x /usr/local/bin/serve-gilded-gnosis.sh; \ + test -x /usr/local/bin/serve-glm52-hybrid-v19.sh; \ + python -m compileall -q \ + /opt/venv/lib/python3.12/site-packages/sparkinfer \ + /opt/venv/lib/python3.12/site-packages/vllm; \ + mkdir -p /opt/sparkinfer/comm/pcie; \ + cp /opt/venv/lib/python3.12/site-packages/sparkinfer/comm/pcie/*.cu \ + /opt/sparkinfer/comm/pcie/; \ + cp /opt/venv/lib/python3.12/site-packages/sparkinfer/comm/pcie/pcie_selected_records*.py \ + /opt/sparkinfer/comm/pcie/; \ + mkdir -p /opt/release-tests/sparkinfer/comm/pcie; \ + cp /opt/venv/lib/python3.12/site-packages/sparkinfer/comm/pcie/pcie_selected_records*.cu \ + /opt/venv/lib/python3.12/site-packages/sparkinfer/comm/pcie/pcie_selected_records*.py \ + /opt/release-tests/sparkinfer/comm/pcie/; \ + if [ "${RUN_CPU_TESTS}" = "1" ]; then \ + python -m pytest -q -p no:cacheprovider \ + /opt/release-tests/sparkinfer-tests/comm/test_pcie_selected_records.py \ + /opt/release-tests/sparkinfer-tests/comm/test_pcie_selected_records_ce.py \ + /opt/release-tests/sparkinfer-tests/comm/test_pcie_selected_records_gpu.py \ + /opt/release-tests/sparkinfer-tests/comm/test_pcie_selected_records_ce_gpu.py \ + /opt/release-tests/sparkinfer-tests/comm/test_pcie_dcp_a2a.py; \ + python -m pytest -q -p no:cacheprovider \ + /opt/release-tests/vllm-tests/v1/attention/test_b12x_sparse_ckv_decode_policy.py \ + /opt/release-tests/vllm-tests/v1/attention/test_b12x_ckv_prefetch_policy.py \ + /opt/release-tests/vllm-tests/v1/attention/test_indexer_dcp_localize.py \ + /opt/release-tests/vllm-tests/v1/worker/test_workspace.py; \ + python -m pytest -q -p no:cacheprovider \ + /opt/release-tests/vllm-tests/v1/core/test_kv_cache_utils.py \ + -k 'lockstep_mla or mixed_dcp_replicated_groups'; \ + fi; \ + rm -rf /opt/release /opt/release-tests /opt/sparkinfer + +COPY --chmod=0755 serve.sh /usr/local/bin/serve-glm52-sparse-ckv.sh + +LABEL org.opencontainers.image.source="https://github.com/FujitsuPolycom/vllm" \ + org.opencontainers.image.revision="${VLLM_REF}" \ + ai01.hardware="ASUS WRX90E-SAGE SE; Threadripper PRO 9965WX; 128 GiB RAM; 4x RTX PRO 6000 Blackwell 96GB at 400W" \ + ai01.base="voipmonitor/vllm@sha256:41078d5fac36b802e0f3798057a4b383c24296cb8570f6f4fc138c34d8cfa303" \ + ai01.vllm="FujitsuPolycom/vllm@${VLLM_REF}" \ + ai01.sparkinfer="FujitsuPolycom/sparkinfer@${SPARKINFER_REF}" \ + ai01.features="replicated indexer; depth-3 CKV prefetch; selected-record sparse decode; bulk copy-engine transport" \ + local-inference.cutlass_dsl.version="4.6.0" \ + local-inference.vllm.commit="${VLLM_REF}" \ + local-inference.vllm.repo="https://github.com/FujitsuPolycom/vllm.git" diff --git a/deployments/glm52-sparse-ckv/README.md b/deployments/glm52-sparse-ckv/README.md new file mode 100644 index 000000000000..8b42bd33334c --- /dev/null +++ b/deployments/glm52-sparse-ckv/README.md @@ -0,0 +1,155 @@ +# GLM-5.2 Sparse CKV Integration Release + +This folder reproduces the validated GLM-5.2 sparse-CKV stack from immutable +commits on the FujitsuPolycom forks. It is intended as an experimental release +and review target, not as a general vLLM distribution. + +## Validated Host + +- ASUS WRX90E-SAGE SE +- AMD Ryzen Threadripper PRO 9965WX +- 128 GiB system RAM +- 4x RTX PRO 6000 Blackwell 96GB, 400 W each +- TP4, DCP4, MTP3 +- Full CUDA peer access; ACS and IOMMU disabled on the validated host + +The four GPUs are attached through separate CPU root ports rather than a Gen5 +P2P switch. Transport results are therefore specific to this host topology. + +The selected-record transports require CUDA IPC peer access between every GPU. +Check topology before launch: + +```bash +nvidia-smi topo -m +nvidia-smi topo -p2p r +./preflight.sh +``` + +## Pinned Sources + +| Component | Pin | +| --- | --- | +| Base image | `voipmonitor/vllm@sha256:41078d5fac36b802e0f3798057a4b383c24296cb8570f6f4fc138c34d8cfa303` | +| vLLM | `FujitsuPolycom/vllm@a81a0811314e23bc6b2baa69f34247aacea71e23` | +| Sparkinfer | `FujitsuPolycom/sparkinfer@a82fc76f9a2321e3cfd3065a9fa78452a27ddf9f` | +| Model | `madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid@68babde27a97a4c980c2494e830dd424975cd5a3` | + +The Docker build verifies both Git SHAs before installing anything. It also +runs the focused CPU policy and transport tests used for the validated image. +The runtime toolchain is pinned to CUTLASS DSL 4.6.0 and QuACK 0.6.1, matching +the validated image rather than the older packages in the v19 base. +The base vLLM package metadata still declares CUTLASS DSL 4.5.3, so `pip check` +reports the intentional 4.6.0 override; startup and focused tests use 4.6.0. + +This is an overlay build, not a source-only CUDA stack build. It intentionally +inherits the pinned `voipmonitor/vllm` image for CUDA, NCCL, FlashInfer, +and InstantTensor, and requires separate access to the pinned Hugging Face +checkpoint. The release installs its own small launcher so the fixed KV budget +and NCCL all-reduce mode cannot be silently dropped by a base-image launcher. + +## Included Features + +- Lockstep allocation for mixed MLA KV-cache groups +- Replicated sparse-indexer KV with DCP-sharded main CKV +- Native 432-byte CKV history gather and depth-3 Shared-layer prefetch +- Per-sequence MTP union and deduplication +- Selected-record sparse CKV decode for C1-C8 +- Direct CUDA-IPC P2P and copy-engine transports +- One-shot bulk transfer for the three Shared layers + +The default Compose selects the copy-engine transport because it won the TP4 +A/B on the validated host. The direct transport remains available by changing +`VLLM_B12X_MLA_SPARSE_DECODE_TRANSPORT` from `ce` to `direct`. + +## Build And Launch + +```bash +git clone --branch codex/glm52-sparse-ckv-release-20260721 \ + https://github.com/FujitsuPolycom/vllm.git +cd vllm/deployments/glm52-sparse-ckv + +# Skip this download when the pinned model revision is already local. +hf download madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid \ + --revision 68babde27a97a4c980c2494e830dd424975cd5a3 \ + --local-dir /srv/ai/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid + +cp .env.example .env +# Edit MODEL_PATH and the cache paths in .env. +./preflight.sh +docker compose config --quiet +docker compose build --pull +docker compose up -d +docker compose logs -f glm52-sparse-ckv +``` + +Keep the Compose entrypoint set to `serve-glm52-sparse-ckv.sh`. The inherited +v19 launcher does not accept this release's fixed `KV_CACHE_MEMORY_BYTES` or +`ALLREDUCE_MODE` controls; using it changes memory allocation and can OOM. + +The first startup performs model loading, JIT compilation, warmup, and CUDA +graph capture. Do not benchmark the first request. The API is ready when this +returns the served model: + +```bash +curl -fsS http://127.0.0.1:5802/v1/models +python validate-api.py +``` + +Confirm that the intended format and features activated: + +```bash +docker compose logs glm52-sparse-ckv 2>&1 | \ + grep -E 'kv_gmem_stride=432|replicat|prefetch|sparse.*CKV|transport=ce' +``` + +The validated profile uses a fixed 3,426,112,942-byte KV allocation per GPU, +`max-model-len=300000`, batch 2048, graph 32, and eight active sequences. The +reported logical KV pool was 302,047 tokens. + +The image assembled only from the public pins in this folder also completed a +four-GPU startup, CUDA-graph capture, and the two-run deterministic API probe at +that exact 302,047-token capacity. + +## Feature Controls + +| Variable | Default | Purpose | +| --- | ---: | --- | +| `VLLM_DCP_REPLICATE_INDEXER_CACHE` | `1` | Replicate the small indexer cache while keeping main CKV DCP-sharded | +| `VLLM_B12X_MLA_CKV_GATHER` / `DCP_CKV_GATHER` | `1` | Gather full CKV for the prefill path | +| `VLLM_B12X_MLA_CKV_PREFETCH_DEPTH` | `3` | Prefetch all three Shared layers after each Full layer | +| `VLLM_B12X_MLA_SPARSE_DECODE_CKV_GATHER` | `1` | Enable selected-record sparse decode | +| `VLLM_B12X_MLA_SPARSE_DECODE_TRANSPORT` | `ce` | Select copy-engine or direct P2P transport | +| `VLLM_B12X_MLA_SPARSE_DECODE_BULK_PREFETCH` | `1` | Transfer S1/S2/S3 together with one synchronization | +| `VLLM_B12X_MLA_SPARSE_DECODE_MAX_SEQS` | `8` | Enable the pooled fast path through C8 | + +## Current Boundaries + +- Sparse selected-record decode is validated only with + `KV_CACHE_DTYPE=nvfp4_ds_mla`, `KV_FP8_ROPE=0`, and the native 432-byte + NVFP4 + BF16-RoPE record. +- TP4/DCP4/MTP3 is the production-tested topology. DCP1 and DCP4 were A/B + tested; TP6/TP8 and DCP6/DCP8 still need physical validation. +- Both explicit `ce` and `direct` transports fail closed when peer + initialization is unavailable. `auto` may fall back from CE to direct, but + this release defaults to strict `ce` and requires full peer access. +- LMCache is not part of this release. +- FP8-RoPE support, calibrated MLA outer scales, and the newer absorbed-QBMM + memory optimization are intentionally excluded until separately validated. + +See [RESULTS.md](RESULTS.md) for the configuration and performance matrix. +Machine-readable source and runtime pins are in +[VERSION_LOCK.json](VERSION_LOCK.json). + +## Security + +The validated Compose uses host networking and `privileged: true` to preserve +the known-working GPU/P2P environment. Run it only on a trusted, isolated host. +Do not expose the unauthenticated vLLM port directly to an untrusted network. + +## Credits + +The scheduling design builds on Koush's sparse-CKV, shared-layer lookahead, and +MTP-deduplication work. The transport and kernel integration builds on Luke +Alonso's Sparkinfer/B12X infrastructure. Implementation, testing, benchmarking, +and documentation were developed with OpenAI Codex assistance and manually +reviewed on the validated host. diff --git a/deployments/glm52-sparse-ckv/RESULTS.md b/deployments/glm52-sparse-ckv/RESULTS.md new file mode 100644 index 000000000000..3e9953effd46 --- /dev/null +++ b/deployments/glm52-sparse-ckv/RESULTS.md @@ -0,0 +1,115 @@ +# Validation Results + +## Daily Profile + +Validated on ai01 with TP4/DCP4/MTP3, copy-engine sparse decode, replicated +indexer, depth-3 prefetch, 300,000 maximum model length, batch 2048, graph 32, +and the 432-byte NVFP4 + BF16-RoPE cache format. + +| Metric | Result | +| --- | ---: | +| Coding peak, five-run median | 105.2 tok/s | +| Coding peak, five-run mean | 104.5 tok/s | +| Coding peak range | 101.6-108.0 tok/s | +| Reported KV capacity | 302,047 tokens | + +Cold prefill on the same final image: + +| Context | Prefill | +| ---: | ---: | +| 8K | 3,326 tok/s | +| 32K | 3,139 tok/s | +| 64K | 2,966 tok/s | +| 128K | 2,719 tok/s | +| 256K | 2,356 tok/s | + +Deterministic temperature-zero outputs matched byte-for-byte across two runs +at 8,095, 64,032, and 127,998 prompt tokens. + +## Public Image Gate + +On 2026-07-21, the Dockerfile was rebuilt on ai01 using only the public source +pins in this folder. Build-time focused tests reported: + +- Sparkinfer transport/policy: 56 passed, 4 GPU-only skipped +- vLLM attention/workspace policy: 120 passed, 18 skipped +- Mixed-DCP KV allocation: 3 passed, 75 deselected + +The resulting image then completed a four-GPU TP4/DCP4/MTP3 startup with the +exact daily profile. Runtime logs confirmed: + +- `--disable-custom-all-reduce` and the 3,426,112,942-byte KV pin +- 302,047 logical KV tokens +- replicated sparse-indexer KV +- selected-record CKV decode through C8 with MTP3 and depth 3 +- one bulk CKV exchange for S1/S2/S3 +- successful CUDA graph capture + +`validate-api.py` returned `sparse-ckv-ready` identically twice. Historical +performance numbers below were produced by the same pinned source stack before +the public packaging layer was added; the release-image gate verifies startup +and correctness, not a fresh performance rerun. + +## Matched 90K Coding Matrix + +These arms used the same fixed 3,426,112,942-byte KV allocation, model, graph +size, batch size, temperature zero, reasoning disabled, one warmup, and five +measured coding runs. The shorter model limit lets DCP1 and DCP4 use an +identical request profile. + +| Topology | MTP | Sparse decode | Coding median | KV tokens | +| --- | ---: | --- | ---: | ---: | +| DCP1 | 0 | Off/no-op | 56.7 tok/s | 93,888 | +| DCP1 | 3 | Off/no-op | 135.7 tok/s | 92,480 | +| DCP4 | 0 | Off | 44.4 tok/s | 305,920 | +| DCP4 | 0 | CE | 45.4 tok/s | 305,920 | +| DCP4 | 3 | Off | 82.9 tok/s | 302,080 | +| DCP4 | 3 | CE | 99.2 tok/s | 302,080 | + +Sparse selected-record decode adds about 2.4% at MTP0 and 19.6% at MTP3 in +this matched coding test. The daily 300K profile is configuration-sensitive and +is reported separately rather than mixed into the DCP gap calculation. + +In the matched bulk-prefetch A/B, enabling the one-shot three-layer exchange +improved the 64K+ geometric mean by 8.5%, moved the coding median from 99.63 to +103.79 tok/s, and left the 302,047-token KV capacity unchanged. + +## Interpretation + +- MTP3 is the intended deployment regime and receives the largest sparse-gather + benefit because candidate rows share one per-sequence union. +- DCP4 provides roughly 3.3x the logical KV capacity of DCP1 with this fixed + per-GPU allocation. +- Sparse decode recovers a meaningful portion of the DCP4 decode penalty, but + the strict DCP4 coding result does not fully match DCP1. +- Results are for one 4-GPU PCIe host. Treat TP8/DCP8 projections as hypotheses + until measured on that topology. + +## Benchmark Reproduction + +The development runs used `llm-inference-bench` v0.4.29. The exact benchmark +script used on ai01 has SHA-256 +`0591b2cfe7767b8a048cfd57011526ca5fc64ea29415362c9ab1204c6c5d55e9`. +Representative commands for the measured request shapes are: + +```bash +BENCH=/opt/llm-inference-bench +PY="$BENCH/.venv/bin/python" + +# Sustained decode matrix plus five sequential coding-peak requests. +$PY "$BENCH/llm_decode_bench.py" --host 127.0.0.1 --port 5802 \ + --skip-prefill --contexts 8k,32k,64k --concurrency 1,2,4,8 \ + --duration 8 --max-tokens 512 --temperature 0 \ + --coding-peak --coding-peak-runs 5 --coding-peak-temperature 0 \ + --display-mode screen --output benchmark-results.json + +# One-sample cold-prefill ladder used to check long-context scaling. +$PY "$BENCH/llm_decode_bench.py" --host 127.0.0.1 --port 5802 \ + --prefill-only --prefill-contexts 8k,32k,64k,128k,256k \ + --temperature 0 --display-mode screen --output prefill-results.json +``` + +Coding peak uses the harness's built-in `/mnt/test.py`-compatible coding +prompt and 2,000-token output budget. Results above are historical validation +of the pinned source stack; the Docker build and public-image gate run +correctness tests, not performance benchmarks. diff --git a/deployments/glm52-sparse-ckv/VERSION_LOCK.json b/deployments/glm52-sparse-ckv/VERSION_LOCK.json new file mode 100644 index 000000000000..29a7ab0f88dd --- /dev/null +++ b/deployments/glm52-sparse-ckv/VERSION_LOCK.json @@ -0,0 +1,51 @@ +{ + "release": "glm52-sparse-ckv-v19-20260721", + "base_image": { + "repository": "voipmonitor/vllm", + "tag_used_during_validation": "gilded-gnosis-v19-vllm7ea567a-b12x4cfa530-fi801d57a-cu132-20260718", + "digest": "sha256:41078d5fac36b802e0f3798057a4b383c24296cb8570f6f4fc138c34d8cfa303" + }, + "vllm": { + "repository": "https://github.com/FujitsuPolycom/vllm.git", + "branch": "codex/sparse-ckv-bulk-prefetch", + "base_image_source_commit": "7ea567a2458a4800a6a0e3e0a6ba41fcbd00d146", + "commit": "a81a0811314e23bc6b2baa69f34247aacea71e23" + }, + "sparkinfer": { + "repository": "https://github.com/FujitsuPolycom/sparkinfer.git", + "branch": "codex/sparse-ckv-release-20260721", + "commit": "a82fc76f9a2321e3cfd3065a9fa78452a27ddf9f" + }, + "model": { + "repository": "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid", + "revision": "68babde27a97a4c980c2494e830dd424975cd5a3", + "metadata_sha256": { + "config.json": "254974797e9f455716a30ab5505ba68272181b20b58a3693e54f94fb8056f3ef", + "model.safetensors.index.json": "6eb773222d932418dd0530c63aca498f86ef424da2a4526ccba76b59726da234", + "tokenizer_config.json": "98b1271574f41abf89427ae2dda030d94dc9478f0edc5a8bd240db213c6fd5fc", + "mxfp8_tier_nokvb.json": "ebcd6087180033d4512fafa5f154f4fecfbc1ee5e5051448f34859cccc4430f0" + } + }, + "runtime": { + "python": "3.12.3", + "pytorch": "2.12.0+cu132", + "cutlass_dsl": "4.6.0", + "quack_kernels": "0.6.1", + "pytest": "8.4.2", + "sparkinfer": "1.0.1", + "flashinfer": "0.6.15+cu132", + "nccl_python_package": "2.29.7", + "nccl_preload": "2.30.4" + }, + "launch": { + "script": "serve-glm52-sparse-ckv.sh", + "sha256": "65de06284212a017f88a031798baa80f2be951c952c12fc0bb1ef0e97d7af024", + "allreduce_mode": "nccl", + "kv_cache_memory_bytes": 3426112942 + }, + "validated_host": { + "nvidia_driver": "610.43.02", + "docker": "29.6.1", + "docker_compose": "5.3.1" + } +} diff --git a/deployments/glm52-sparse-ckv/docker-compose.yml b/deployments/glm52-sparse-ckv/docker-compose.yml new file mode 100644 index 000000000000..bcefe85608f1 --- /dev/null +++ b/deployments/glm52-sparse-ckv/docker-compose.yml @@ -0,0 +1,127 @@ +services: + glm52-sparse-ckv: + build: + context: . + args: + BASE_IMAGE: voipmonitor/vllm@sha256:41078d5fac36b802e0f3798057a4b383c24296cb8570f6f4fc138c34d8cfa303 + VLLM_BASE_REF: 7ea567a2458a4800a6a0e3e0a6ba41fcbd00d146 + VLLM_REF: a81a0811314e23bc6b2baa69f34247aacea71e23 + SPARKINFER_REF: a82fc76f9a2321e3cfd3065a9fa78452a27ddf9f + RUN_CPU_TESTS: "1" + image: ${IMAGE_NAME:-fujitsupolycom/glm52-sparse-ckv:v19-20260721} + container_name: glm52-sparse-ckv + entrypoint: ["/usr/local/bin/serve-glm52-sparse-ckv.sh"] + network_mode: host + ipc: host + privileged: true + init: true + shm_size: 32gb + gpus: all + restart: "no" + ulimits: + memlock: -1 + stack: 67108864 + nofile: + soft: 1048576 + hard: 1048576 + environment: + MODEL_FAMILY: glm52-hybrid + MODEL: /model + GPUS: ${CUDA_VISIBLE_DEVICES:-0,1,2,3} + CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0,1,2,3} + CUDA_DEVICE_ORDER: PCI_BUS_ID + CUDA_DEVICE_MAX_CONNECTIONS: "32" + CUTE_DSL_ARCH: sm_120a + TORCH_CUDA_ARCH_LIST: 12.0a + OMP_NUM_THREADS: "16" + PORT: ${PORT:-5802} + TP: "4" + DCP: "4" + DCP_BACKEND: a2a + DCP_A2A_MAX_TOKENS: "16" + DCP_A2A_LARGE_BACKEND: ag_rs + MTP: "3" + MAX_NUM_SEQS: "8" + GRAPH: "32" + MAX_MODEL_LEN: "300000" + MAX_BATCHED_TOKENS: "2048" + GPU_MEMORY_UTILIZATION: "0.98" + KV_CACHE_MEMORY_BYTES: "3426112942" + MOE_MODE: a16 + QUANTIZATION: nvfp4_nf3_hybrid + ONLINE_QUANT: nf3-mxfp8 + KV_CACHE_DTYPE: nvfp4_ds_mla + KV_FP8_ROPE: "0" + NF3_GRID188: "1" + VLLM_NF3_GRID188_DECODE: "1" + B12X_DENSE_SPLITK_TURBO: "1" + B12X_MLA_SM120_UNIFIED: "1" + B12X_MOE_FORCE_A16: "1" + B12X_MOE_FORCE_A8: "0" + B12X_W4A16_TC_DECODE: "1" + B12X_W4A8_TINY_DECODE: "1" + LOAD_FORMAT: instanttensor + INSTANTTENSOR_BACKEND: BUFFERED + ALLREDUCE_MODE: nccl + F8_DMA: "0" + B12X_PCIE_DMA: "0" + VLLM_USE_B12X_PCIE_DMA: "0" + VLLM_ENABLE_PCIE_ALLREDUCE: "0" + VLLM_ALLOW_CUSTOM_ALLREDUCE_PCIE: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_USE_B12X_WO_PROJECTION: "1" + VLLM_USE_B12X_MHC: "1" + VLLM_USE_B12X_FP8_GEMM: "1" + VLLM_USE_B12X_MOE: "1" + VLLM_USE_B12X_SPARSE_INDEXER: "1" + VLLM_USE_B12X_DCP_A2A: "1" + VLLM_DCP_A2A_MAX_TOKENS: "16" + VLLM_DCP_A2A_LARGE_BACKEND: ag_rs + VLLM_DCP_PROJECT_BEFORE_MERGE: "1" + VLLM_DCP_PROJECT_BEFORE_MERGE_MIN_PREFILL_TOKENS: "1024" + VLLM_B12X_MLA_DCP_GATHER_IN_WORKSPACE: "1" + VLLM_DCP_GLOBAL_TOPK: "1" + VLLM_DCP_REPLICATE_INDEXER_CACHE: "1" + VLLM_DCP_SHARD_DRAFT: "1" + DCP_PREFILL_WORKSPACE: "1" + DCP_QUERY_SPLIT: "0" + VLLM_DCP_QUERY_SPLIT: "0" + DCP_CKV_GATHER: "1" + VLLM_B12X_MLA_CKV_GATHER: "1" + VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS: "16" + VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS: "300000" + VLLM_B12X_MLA_CKV_PREFETCH_DEPTH: "3" + VLLM_B12X_MLA_SPARSE_DECODE_CKV_GATHER: "1" + VLLM_B12X_MLA_SPARSE_DECODE_TRANSPORT: ce + VLLM_B12X_MLA_SPARSE_DECODE_BULK_PREFETCH: "1" + VLLM_B12X_MLA_SPARSE_DECODE_MAX_SEQS: "8" + VLLM_B12X_MLA_SPARSE_DECODE_POOL_RECORDS: "0" + VLLM_DISABLE_COMPILE_CACHE: "1" + VLLM_DISABLED_KERNELS: MarlinFP8ScaledMMLinearKernel + VLLM_DISABLE_SHARED_EXPERTS_STREAM: "0" + VLLM_USE_AOT_COMPILE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_USE_MEGA_AOT_ARTIFACT: "1" + VLLM_USE_FLASHINFER_SAMPLER: "1" + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_MEMORY_PROFILE_INCLUDE_ATTN: "1" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "1" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SAFETENSORS_FAST_GPU: "1" + NCCL_IB_DISABLE: "1" + NCCL_P2P_LEVEL: SYS + NCCL_PROTO: LL,LL128,Simple + NCCL_SOCKET_IFNAME: lo + GLOO_SOCKET_IFNAME: lo + XDG_CACHE_HOME: /cache + VLLM_CACHE_DIR: /cache/jit/vllm + TRITON_CACHE_DIR: /cache/triton + TORCHINDUCTOR_CACHE_DIR: /cache/torchinductor + TORCH_EXTENSIONS_DIR: /cache/jit/torch_extensions + TMPDIR: /container-tmp + volumes: + - ${MODEL_PATH:?set MODEL_PATH in .env}:/model:ro + - ${HF_CACHE_PATH:-/srv/ai/huggingface}:/root/.cache/huggingface + - ${CACHE_PATH:-/srv/ai/cache/glm52-sparse-ckv-v19}:/cache + - ${TMP_PATH:-/srv/ai/tmp/glm52-sparse-ckv-v19}:/container-tmp diff --git a/deployments/glm52-sparse-ckv/preflight.sh b/deployments/glm52-sparse-ckv/preflight.sh new file mode 100755 index 000000000000..c20c758eba90 --- /dev/null +++ b/deployments/glm52-sparse-ckv/preflight.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ -f .env ]; then + set -a + # shellcheck disable=SC1091 + source ./.env + set +a +fi + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +warn() { + printf 'WARN: %s\n' "$*" >&2 +} + +for command in docker nvidia-smi sha256sum; do + command -v "${command}" >/dev/null || fail "missing ${command}" +done + +docker compose version >/dev/null || fail "Docker Compose plugin is unavailable" +docker info >/dev/null || fail "Docker daemon is unavailable" + +gpu_count="$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l)" +if [ "${gpu_count}" -lt 4 ]; then + fail "four GPUs are required; found ${gpu_count}" +fi + +model_path="${MODEL_PATH:-/srv/ai/models/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid}" +model_files=( + "config.json:254974797e9f455716a30ab5505ba68272181b20b58a3693e54f94fb8056f3ef" + "model.safetensors.index.json:6eb773222d932418dd0530c63aca498f86ef424da2a4526ccba76b59726da234" + "tokenizer_config.json:98b1271574f41abf89427ae2dda030d94dc9478f0edc5a8bd240db213c6fd5fc" + "mxfp8_tier_nokvb.json:ebcd6087180033d4512fafa5f154f4fecfbc1ee5e5051448f34859cccc4430f0" +) +for entry in "${model_files[@]}"; do + file="${entry%%:*}" + expected="${entry#*:}" + path="${model_path}/${file}" + test -f "${path}" || fail "missing ${path}" + actual="$(sha256sum "${path}" | awk '{print $1}')" + test "${actual}" = "${expected}" || fail "checksum mismatch for ${path}" +done + +printf '%s\n' 'GPU topology:' +nvidia-smi topo -m +printf '%s\n' 'Peer-read capability:' +peer_matrix="$(nvidia-smi topo -p2p r)" +printf '%s\n' "${peer_matrix}" + +IFS=',' read -r -a selected_gpus <<< "${CUDA_VISIBLE_DEVICES:-0,1,2,3}" +if [ "${#selected_gpus[@]}" -ne 4 ]; then + fail "this profile requires exactly four CUDA_VISIBLE_DEVICES entries" +fi +for source in "${selected_gpus[@]}"; do + [[ "${source}" =~ ^[0-9]+$ ]] || fail "preflight expects numeric GPU indices" + for destination in "${selected_gpus[@]}"; do + [ "${source}" = "${destination}" ] && continue + status="$(printf '%s\n' "${peer_matrix}" | awk \ + -v row="GPU${source}" -v column="$((destination + 2))" \ + '$1 == row {print $column}')" + [ "${status}" = "OK" ] || \ + fail "peer reads are ${status:-unknown} from GPU${source} to GPU${destination}" + done +done + +if find /sys/kernel/iommu_groups -mindepth 1 -print -quit 2>/dev/null | grep -q .; then + warn "IOMMU groups are active; the validated host used amd_iommu=off iommu=off" +fi + +if command -v lspci >/dev/null; then + if lspci -vv 2>/dev/null | grep -Eq 'ACSCtl:.*(SrcValid\+|TransBlk\+|ReqRedir\+|CmpltRedir\+)'; then + warn "ACS redirect appears enabled; verify that peer traffic is not redirected" + fi +fi + +available_gib="$(df -BG --output=avail "${model_path}" | tail -1 | tr -dc '0-9')" +if [ -n "${available_gib}" ] && [ "${available_gib}" -lt 60 ]; then + warn "less than 60 GiB is free on the model filesystem" +fi + +printf '%s\n' 'Power and link state:' +nvidia-smi --query-gpu=index,name,power.limit,pci.bus_id,pcie.link.gen.current,pcie.link.width.current \ + --format=csv,noheader +printf '%s\n' 'Preflight complete. Review any warnings before launch.' diff --git a/deployments/glm52-sparse-ckv/serve.sh b/deployments/glm52-sparse-ckv/serve.sh new file mode 100755 index 000000000000..c0fb4945bc5e --- /dev/null +++ b/deployments/glm52-sparse-ckv/serve.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODEL="${MODEL:-/model}" +SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-GLM-5.2-MXFP8-NVFP4-NF3-Hybrid}" +HOST="${HOST:-0.0.0.0}" +PORT="${PORT:-5802}" +TP="${TP:-4}" +DCP="${DCP:-4}" +DCP_BACKEND="${DCP_BACKEND:-a2a}" +MTP="${MTP:-3}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-300000}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-8}" +MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-2048}" +GRAPH="${GRAPH:-32}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.98}" +KV_CACHE_MEMORY_BYTES="${KV_CACHE_MEMORY_BYTES:-3426112942}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-nvfp4_ds_mla}" +LOAD_FORMAT="${LOAD_FORMAT:-instanttensor}" +MOE_BACKEND="${MOE_BACKEND:-b12x}" +QUANTIZATION="${QUANTIZATION:-nvfp4_nf3_hybrid}" +ALLREDUCE_MODE="${ALLREDUCE_MODE:-nccl}" +GLM52_INDEX_TOPK_PATTERN="${GLM52_INDEX_TOPK_PATTERN:-FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS}" +QUANTIZATION_CONFIG_JSON="${QUANTIZATION_CONFIG_JSON:-}" +if [[ -z "${QUANTIZATION_CONFIG_JSON}" ]]; then + QUANTIZATION_CONFIG_JSON='{"linear":{"weight":"mxfp8"},"shared_experts":{"weight":"mxfp8"},"ignore":["re:^model\\.layers\\.0\\.","re:.*\\.self_attn\\.indexer\\.","re:.*\\.mlp\\.gate$","model.layers.78.eh_proj","lm_head"]}' +fi + +[[ "${TP}" =~ ^[0-9]+$ ]] || { echo "TP must be an integer" >&2; exit 2; } +[[ "${DCP}" =~ ^[0-9]+$ ]] || { echo "DCP must be an integer" >&2; exit 2; } +[[ "${MTP}" =~ ^[0-9]+$ ]] || { echo "MTP must be an integer" >&2; exit 2; } +[[ "${MAX_NUM_SEQS}" =~ ^[0-9]+$ ]] || { echo "MAX_NUM_SEQS must be an integer" >&2; exit 2; } +[[ "${GRAPH}" =~ ^[0-9]+$ ]] || { echo "GRAPH must be an integer" >&2; exit 2; } +[[ "${#GLM52_INDEX_TOPK_PATTERN}" -eq 78 ]] || { echo "GLM52_INDEX_TOPK_PATTERN must contain 78 layer entries" >&2; exit 2; } + +export CUDA_VISIBLE_DEVICES="${GPUS:-${CUDA_VISIBLE_DEVICES:-0,1,2,3}}" +export VLLM_USE_B12X_PCIE_DMA="${B12X_PCIE_DMA:-0}" +export VLLM_PCIE_DMA_FP8="${F8_DMA:-0}" +export B12X_PCIE_DMA_FP8="${F8_DMA:-0}" + +allreduce_args=() +case "${ALLREDUCE_MODE}" in + nccl) + export VLLM_ENABLE_PCIE_ALLREDUCE=0 + export VLLM_ALLOW_CUSTOM_ALLREDUCE_PCIE=0 + export VLLM_ALLREDUCE_USE_SYMM_MEM=0 + allreduce_args=(--disable-custom-all-reduce) + ;; + b12x) + export VLLM_ENABLE_PCIE_ALLREDUCE=1 + export VLLM_PCIE_ALLREDUCE_BACKEND=b12x + export VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE=64KB + export VLLM_PCIE_ONESHOT_FUSED_ADD_RMS_NORM_MAX_SIZE=84KB + ;; + *) + echo "ALLREDUCE_MODE must be nccl or b12x" >&2 + exit 2 + ;; +esac + +dcp_args=(--decode-context-parallel-size "${DCP}") +if [[ "${DCP}" != "1" ]]; then + dcp_args+=( + --dcp-comm-backend "${DCP_BACKEND}" + --dcp-kv-cache-interleave-size 1 + ) +fi + +spec_args=() +if [[ "${MTP}" != "0" ]]; then + spec_json="$(printf '{\"model\":\"%s\",\"method\":\"mtp\",\"num_speculative_tokens\":%s,\"moe_backend\":\"%s\",\"draft_sample_method\":\"probabilistic\"}' "${MODEL}" "${MTP}" "${MOE_BACKEND}")" + spec_args=(--speculative-config "${spec_json}") +fi + +kv_args=() +if [[ -n "${KV_CACHE_MEMORY_BYTES}" ]]; then + kv_args=(--kv-cache-memory-bytes "${KV_CACHE_MEMORY_BYTES}") +fi + +hf_overrides="$(printf '{\"use_index_cache\":true,\"index_topk_pattern\":\"%s\"}' "${GLM52_INDEX_TOPK_PATTERN}")" + +cmd=( + vllm serve "${MODEL}" + --served-model-name "${SERVED_MODEL_NAME}" + --host "${HOST}" + --port "${PORT}" + --trust-remote-code + --tensor-parallel-size "${TP}" + "${allreduce_args[@]}" + "${dcp_args[@]}" + --kv-cache-dtype "${KV_CACHE_DTYPE}" + --attention-backend B12X_MLA_SPARSE + --moe-backend "${MOE_BACKEND}" + --quantization "${QUANTIZATION}" + --quantization-config "${QUANTIZATION_CONFIG_JSON}" + --load-format "${LOAD_FORMAT}" + -cc.pass_config.fuse_allreduce_rms=True + --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" + "${kv_args[@]}" + --max-model-len "${MAX_MODEL_LEN}" + --max-num-seqs "${MAX_NUM_SEQS}" + --max-num-batched-tokens "${MAX_BATCHED_TOKENS}" + --max-cudagraph-capture-size "${GRAPH}" + --async-scheduling + --enable-chunked-prefill + --enable-prefix-caching + --enable-flashinfer-autotune + --enable-auto-tool-choice + --tool-call-parser glm47 + --reasoning-parser glm45 + --default-chat-template-kwargs '{"reasoning_effort":"high"}' + --enable-prompt-tokens-details + --enable-force-include-usage + --enable-request-id-headers + --hf-overrides "${hf_overrides}" + "${spec_args[@]}" +) + +if [[ "${DRY_RUN:-0}" == "1" ]]; then + printf 'Command:' + printf ' %q' "${cmd[@]}" + printf '\n' + exit 0 +fi + +exec "${cmd[@]}" diff --git a/deployments/glm52-sparse-ckv/validate-api.py b/deployments/glm52-sparse-ckv/validate-api.py new file mode 100755 index 000000000000..8e17921be9e9 --- /dev/null +++ b/deployments/glm52-sparse-ckv/validate-api.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +import os +import sys +import urllib.request + +base_url = os.getenv("BASE_URL", "http://127.0.0.1:5802").rstrip("/") + + +def request_json(path: str, payload: dict | None = None) -> dict: + data = None if payload is None else json.dumps(payload).encode() + request = urllib.request.Request( + f"{base_url}{path}", + data=data, + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=600) as response: + return json.load(response) + + +models = request_json("/v1/models").get("data", []) +if not models: + raise SystemExit("server returned no models") +model = models[0]["id"] +payload = { + "model": model, + "messages": [ + { + "role": "user", + "content": "Return exactly the text: sparse-ckv-ready", + } + ], + "temperature": 0, + "max_tokens": 32, + "reasoning_effort": "none", +} +outputs = [ + request_json("/v1/chat/completions", payload)["choices"][0]["message"]["content"] + for _ in range(2) +] +if outputs[0] != outputs[1]: + print(json.dumps(outputs, indent=2), file=sys.stderr) + raise SystemExit("temperature-zero smoke outputs differ") +expected = "sparse-ckv-ready" +if outputs[0].strip() != expected: + print(json.dumps(outputs, indent=2), file=sys.stderr) + raise SystemExit(f"expected {expected!r}") +print(json.dumps({"model": model, "output": outputs[0]}, indent=2))