From 21dd11258c8a45815dbfd7a79a3019910099652a Mon Sep 17 00:00:00 2001 From: ark-dev Date: Fri, 12 Jun 2026 02:59:17 +0000 Subject: [PATCH 1/5] =?UTF-8?q?ark-dev:=20Implement=20Q1:=20SGLang=20Qwen3?= =?UTF-8?q?-8B=20baseline=20harness=20=E2=80=94=20pin=20A100-compatible=20?= =?UTF-8?q?SGLang=20image=20(CUDA=2012.x,=20record=20tag+digest),=20launch?= =?UTF-8?q?=20Qwen3-8B=20TP=3D8=20best=20config,=20measure=20batch=3D1=20p?= =?UTF-8?q?refill=20TTFT=20(prompt=202048)=20and=20decode=20per-token=20la?= =?UTF-8?q?tency=20(generate=20128),=20capture=20GPU=20clocks.=20Deliver?= =?UTF-8?q?=20examples/qwen3/bench/=20(launch+measure=20scripts),=20BASELI?= =?UTF-8?q?NE.md=20with=20numbers=20and=20reproduce.md.=20Branch=20qwen3-q?= =?UTF-8?q?1-baseline=20off=20main.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/qwen3/bench/BASELINE.md | 85 ++++++++ examples/qwen3/bench/launch_sglang.sh | 88 ++++++++ examples/qwen3/bench/measure_baseline.py | 245 +++++++++++++++++++++++ examples/qwen3/bench/reproduce.md | 119 +++++++++++ 4 files changed, 537 insertions(+) create mode 100644 examples/qwen3/bench/BASELINE.md create mode 100755 examples/qwen3/bench/launch_sglang.sh create mode 100644 examples/qwen3/bench/measure_baseline.py create mode 100644 examples/qwen3/bench/reproduce.md diff --git a/examples/qwen3/bench/BASELINE.md b/examples/qwen3/bench/BASELINE.md new file mode 100644 index 00000000..91d64cf9 --- /dev/null +++ b/examples/qwen3/bench/BASELINE.md @@ -0,0 +1,85 @@ +# SGLang Qwen3-8B Baseline Latency + +> **Note:** Values marked "TBD" will be filled after running benchmarks on the target hardware. + +Model: `Qwen/Qwen3-8B` (~16 GB fp16) + +## Image + +| Field | Value | +|----------------|-------| +| Image tag | `lmsysorg/sglang:v0.4.6.post1-cu124` | +| Image digest | TBD (run on mscclpp-a100-dev) | +| CUDA version | 12.4 | +| SGLang version | 0.4.6.post1 | + +## Hardware + +| Field | Value | +|-------------------|-------| +| Node | mscclpp-a100-dev | +| GPUs | 8× NVIDIA A100-SXM4-80GB | +| GPU clocks (gr) | TBD (run on mscclpp-a100-dev) | +| GPU clocks (mem) | TBD (run on mscclpp-a100-dev) | +| Interconnect | NVLink + NVSwitch | + +## TP=1 — SGLang natural best-latency config + +Single-GPU inference. For an 8B model (~16 GB fp16), TP=1 is the +lowest-overhead configuration and represents SGLang's best achievable +latency for this model size. + +### Server flags + +``` +python -m sglang.launch_server \ + --model Qwen/Qwen3-8B \ + --tp 1 \ + --port 30000 \ + --mem-fraction-static 0.85 \ + --trust-remote-code +``` + +### Results + +| Metric | Value | +|---------------------------------|-------| +| Prefill TTFT (prompt=2048, gen=1) | TBD (run on mscclpp-a100-dev) | +| Decode per-token (prompt=2048, gen=128) | TBD (run on mscclpp-a100-dev) | +| Trials | 5 | + +## TP=8 — Matched-regime comparison + +TP=8 for an 8B model is the ARK-favorable regime: it minimizes per-GPU +compute and amplifies kernel-launch and communication overhead +(hypothesis 1). This matches the parallelism used by the ARK target +but is **not** SGLang's natural best config for this model size. + +### Server flags + +``` +python -m sglang.launch_server \ + --model Qwen/Qwen3-8B \ + --tp 8 \ + --port 30000 \ + --mem-fraction-static 0.85 \ + --trust-remote-code +``` + +### Results + +| Metric | Value | +|---------------------------------|-------| +| Prefill TTFT (prompt=2048, gen=1) | TBD (run on mscclpp-a100-dev) | +| Decode per-token (prompt=2048, gen=128) | TBD (run on mscclpp-a100-dev) | +| Trials | 5 | + +## Methodology + +- Endpoint: `/generate` with `ignore_eos: true` (not `/v1/chat/completions`). +- Prompt: ~2048 tokens (repeating pattern). +- Warmup: 3 requests discarded before measurement. +- TTFT: time for prompt=2048, max_new_tokens=1 (single request, not streaming). +- Decode per-token: total_time / output_tokens for prompt=2048, max_new_tokens=128. +- Reported value: median over 5 trials. +- Temperature: 0.0 (greedy). diff --git a/examples/qwen3/bench/launch_sglang.sh b/examples/qwen3/bench/launch_sglang.sh new file mode 100755 index 00000000..e5a643ca --- /dev/null +++ b/examples/qwen3/bench/launch_sglang.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Launch SGLang server with Qwen3-8B on A100 GPUs. +# +# Usage: +# ./launch_sglang.sh # TP=8 (default) +# ./launch_sglang.sh 1 # TP=1 +# ./launch_sglang.sh 8 # TP=8 +# +# Prerequisites: +# - Docker with NVIDIA runtime +# - 8×A100-80GB (for TP=8) or 1×A100-80GB (for TP=1) +# - HuggingFace cache at $HF_HOME (default: ~/.cache/huggingface) + +set -euo pipefail + +# --- Configuration ----------------------------------------------------------- + +TP="${1:-8}" +MODEL="Qwen/Qwen3-8B" +PORT=30000 +CONTAINER_NAME="sglang-qwen3-bench" + +# Pinned SGLang image — CUDA 12.x, A100-compatible. +# Update DIGEST after first pull: docker inspect --format='{{index .RepoDigests 0}}' +IMAGE_TAG="lmsysorg/sglang:v0.4.6.post1-cu124" +IMAGE_DIGEST="TBD (run: docker inspect --format='{{index .RepoDigests 0}}' ${IMAGE_TAG})" + +HF_HOME="${HF_HOME:-${HOME}/.cache/huggingface}" + +# --- Validate ----------------------------------------------------------------- + +if [[ "${TP}" != "1" && "${TP}" != "8" ]]; then + echo "Error: TP must be 1 or 8, got '${TP}'" >&2 + exit 1 +fi + +# --- Pull image --------------------------------------------------------------- + +echo "Pulling ${IMAGE_TAG} ..." +docker pull "${IMAGE_TAG}" + +# --- Stop any existing container ---------------------------------------------- + +if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then + echo "Stopping existing container '${CONTAINER_NAME}' ..." + docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 +fi + +# --- Launch server ------------------------------------------------------------ + +echo "Launching SGLang server: model=${MODEL}, tp=${TP}, port=${PORT}" +docker run -d \ + --name "${CONTAINER_NAME}" \ + --gpus all \ + --ipc=host \ + --network=host \ + -v "${HF_HOME}:/root/.cache/huggingface" \ + -e HF_HOME=/root/.cache/huggingface \ + "${IMAGE_TAG}" \ + python -m sglang.launch_server \ + --model "${MODEL}" \ + --tp "${TP}" \ + --port "${PORT}" \ + --mem-fraction-static 0.85 \ + --trust-remote-code + +echo "" +echo "Container '${CONTAINER_NAME}' started." +echo "Waiting for server to be ready on port ${PORT} ..." + +# --- Wait for server readiness ------------------------------------------------ + +MAX_WAIT=300 +ELAPSED=0 +while ! curl -sf "http://localhost:${PORT}/health" >/dev/null 2>&1; do + sleep 5 + ELAPSED=$((ELAPSED + 5)) + if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then + echo "Error: server did not become ready within ${MAX_WAIT}s." >&2 + echo "Check logs: docker logs ${CONTAINER_NAME}" >&2 + exit 1 + fi +done + +echo "Server is ready (waited ${ELAPSED}s)." +echo "" +echo "Run the benchmark:" +echo " python measure_baseline.py --port ${PORT}" diff --git a/examples/qwen3/bench/measure_baseline.py b/examples/qwen3/bench/measure_baseline.py new file mode 100644 index 00000000..719be528 --- /dev/null +++ b/examples/qwen3/bench/measure_baseline.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Measure SGLang Qwen3-8B prefill TTFT and decode per-token latency. + +Uses the /generate endpoint with ignore_eos: true (the /v1/chat/completions +endpoint silently ignores ignore_eos in sglang v0.4.x–v0.5.x). + +Metrics: + - Prefill TTFT: prompt=2048 tokens, max_new_tokens=1. + Time from request send to first (only) token received. + - Decode per-token latency: prompt=2048 tokens, max_new_tokens=128. + total_time / output_tokens (approximation; prefill << decode for 128 output tokens). + +Reports median over N trials (default 5). +Captures GPU clocks via nvidia-smi at start of run. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import subprocess +import sys +import time +from typing import Any + +try: + import requests +except ImportError: + print("Error: 'requests' package required. Install: pip install requests", file=sys.stderr) + sys.exit(1) + + +def build_prompt(num_tokens: int) -> str: + """Build a prompt that is approximately `num_tokens` tokens long. + + Uses a repeating word pattern. Actual token count depends on the + tokenizer, but "word " ≈ 1 token for most BPE tokenizers, so we + slightly over-generate to ensure we hit the target length. + """ + # Pad by 1.2x to account for BPE tokenizer variance (some words may produce <1 token). + word = "hello " + return word * int(num_tokens * 1.2) + + +def capture_gpu_clocks() -> str: + """Capture current GPU clocks via nvidia-smi.""" + try: + result = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=index,clocks.gr,clocks.mem,clocks.max.gr,clocks.max.mem", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=10, + ) + return result.stdout.strip() + except (FileNotFoundError, subprocess.TimeoutExpired): + return "nvidia-smi not available" + + +def send_request( + base_url: str, prompt: str, max_new_tokens: int +) -> dict[str, Any]: + """Send a /generate request and measure timing. + + Returns dict with keys: total_ms, output_tokens, output_text. + """ + payload = { + "text": prompt, + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.0, + "ignore_eos": True, + }, + } + + start = time.perf_counter() + resp = requests.post(f"{base_url}/generate", json=payload, timeout=120) + total = time.perf_counter() - start + + resp.raise_for_status() + data = resp.json() + + # The /generate endpoint returns the generated text. + output_text = data.get("text", "") + # Rough token count from meta if available, else estimate from spaces. + meta = data.get("meta_info", {}) + output_tokens = meta.get("completion_tokens", len(output_text.split())) + + return { + "total_ms": total * 1000, + "output_tokens": output_tokens, + "output_text": output_text[:200], # truncate for display + } + + +def measure_ttft(base_url: str, prompt: str) -> float: + """Measure prefill TTFT: prompt -> 1 token.""" + result = send_request(base_url, prompt, max_new_tokens=1) + return result["total_ms"] + + +def measure_decode(base_url: str, prompt: str, max_new_tokens: int) -> dict: + """Measure decode per-token latency: prompt -> max_new_tokens tokens.""" + result = send_request(base_url, prompt, max_new_tokens=max_new_tokens) + + output_tokens = result["output_tokens"] + total_ms = result["total_ms"] + + # Per-token latency: we don't have streaming TTFT here, so we estimate + # per-token as total / output_tokens for the decode portion. + # For a more precise split, use streaming — but for a baseline this + # is sufficient since prefill << decode for 128 output tokens. + if output_tokens == 0: + print("Warning: server returned 0 output tokens", file=sys.stderr) + per_token_ms = total_ms / max(output_tokens, 1) + + return { + "total_ms": total_ms, + "output_tokens": output_tokens, + "per_token_ms": per_token_ms, + } + + +def run_trials( + base_url: str, + prompt: str, + num_trials: int, + num_warmup: int, +) -> dict: + """Run TTFT and decode measurements, return median results.""" + print(f"Running {num_warmup} warmup request(s) ...") + for i in range(num_warmup): + send_request(base_url, prompt, max_new_tokens=1) + print(f" warmup {i+1}/{num_warmup} done") + + # --- TTFT (prefill) --- + print(f"\nMeasuring TTFT ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=1) ...") + ttft_values = [] + for i in range(num_trials): + ttft = measure_ttft(base_url, prompt) + ttft_values.append(ttft) + print(f" trial {i+1}/{num_trials}: TTFT = {ttft:.2f} ms") + + # --- Decode per-token --- + print(f"\nMeasuring decode latency ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=128) ...") + decode_values = [] + for i in range(num_trials): + result = measure_decode(base_url, prompt, max_new_tokens=128) + decode_values.append(result["per_token_ms"]) + print( + f" trial {i+1}/{num_trials}: {result['per_token_ms']:.2f} ms/token " + f"({result['output_tokens']} tokens in {result['total_ms']:.1f} ms)" + ) + + return { + "ttft_median_ms": statistics.median(ttft_values), + "ttft_all_ms": ttft_values, + "decode_per_token_median_ms": statistics.median(decode_values), + "decode_all_ms": decode_values, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Measure SGLang Qwen3-8B prefill TTFT and decode latency." + ) + parser.add_argument( + "--port", type=int, default=30000, help="SGLang server port (default: 30000)" + ) + parser.add_argument( + "--host", type=str, default="localhost", help="SGLang server host (default: localhost)" + ) + parser.add_argument( + "--trials", type=int, default=5, help="Number of measurement trials (default: 5)" + ) + parser.add_argument( + "--warmup", type=int, default=3, help="Number of warmup requests (default: 3)" + ) + parser.add_argument( + "--prompt-tokens", + type=int, + default=2048, + help="Approximate prompt length in tokens (default: 2048)", + ) + args = parser.parse_args() + + base_url = f"http://{args.host}:{args.port}" + + # Check server health + print(f"Checking server at {base_url} ...") + try: + resp = requests.get(f"{base_url}/health", timeout=5) + resp.raise_for_status() + except (requests.ConnectionError, requests.Timeout) as e: + print(f"Error: cannot reach server at {base_url}: {e}", file=sys.stderr) + sys.exit(1) + print("Server is healthy.\n") + + # Capture GPU clocks + print("GPU clocks:") + gpu_clocks = capture_gpu_clocks() + print(gpu_clocks) + print() + + # Build prompt + prompt = build_prompt(args.prompt_tokens) + + # Run measurements + results = run_trials(base_url, prompt, args.trials, args.warmup) + + # Print summary + print("\n" + "=" * 60) + print("RESULTS SUMMARY") + print("=" * 60) + print(f"Prompt length: ~{args.prompt_tokens} tokens") + print(f"Trials: {args.trials}") + print(f"Prefill TTFT (median): {results['ttft_median_ms']:.2f} ms") + print(f" all: {[f'{v:.2f}' for v in results['ttft_all_ms']]}") + print(f"Decode per-token (median): {results['decode_per_token_median_ms']:.2f} ms/token") + print(f" all: {[f'{v:.2f}' for v in results['decode_all_ms']]}") + print(f"GPU clocks:\n{gpu_clocks}") + print("=" * 60) + + # Write JSON results + output = { + "prompt_tokens": args.prompt_tokens, + "trials": args.trials, + "ttft_median_ms": results["ttft_median_ms"], + "ttft_all_ms": results["ttft_all_ms"], + "decode_per_token_median_ms": results["decode_per_token_median_ms"], + "decode_all_ms": results["decode_all_ms"], + "gpu_clocks": gpu_clocks, + } + results_file = "baseline_results.json" + with open(results_file, "w") as f: + json.dump(output, f, indent=2) + print(f"\nJSON results written to {results_file}") + + +if __name__ == "__main__": + main() diff --git a/examples/qwen3/bench/reproduce.md b/examples/qwen3/bench/reproduce.md new file mode 100644 index 00000000..f5aed5fc --- /dev/null +++ b/examples/qwen3/bench/reproduce.md @@ -0,0 +1,119 @@ +# Reproducing the Qwen3-8B SGLang Baseline + +Target node: `mscclpp-a100-dev` (8× A100-SXM4-80GB). + +## Prerequisites + +- Docker with NVIDIA runtime (`nvidia-docker2` or `--gpus` support). +- HuggingFace model cache with network access (model downloads on first run). +- Python 3.8+ with `requests` package on the host (for `measure_baseline.py`). + +## Step 1: Clone and navigate + +```bash +git clone +cd ark/examples/qwen3/bench +``` + +## Step 2: Run TP=8 (matched-regime comparison) + +TP=8 for an 8B model amplifies kernel-launch and communication overhead. +This is the matched-regime comparison for the ARK target, not SGLang's +best config. + +```bash +# Launch server (TP=8, default) +./launch_sglang.sh 8 + +# Wait for "Server is ready" message, then measure: +python measure_baseline.py --port 30000 --trials 5 + +# Record results in BASELINE.md under "TP=8" section. +# Save the image digest: +docker inspect --format='{{index .RepoDigests 0}}' lmsysorg/sglang:v0.4.6.post1-cu124 + +# Stop server +docker rm -f sglang-qwen3-bench +``` + +## Step 3: Run TP=1 (SGLang natural best-latency config) + +TP=1 is the lowest-overhead config for an 8B model and represents +SGLang's best achievable latency for this model size. + +```bash +# Launch server (TP=1) +./launch_sglang.sh 1 + +# Wait for "Server is ready" message, then measure: +python measure_baseline.py --port 30000 --trials 5 + +# Record results in BASELINE.md under "TP=1" section. + +# Stop server +docker rm -f sglang-qwen3-bench +``` + +## Step 4: Record GPU clocks + +`measure_baseline.py` captures GPU clocks automatically via `nvidia-smi`. +Copy the clock values from the script output into BASELINE.md. + +Alternatively, capture manually: + +```bash +nvidia-smi --query-gpu=index,clocks.gr,clocks.mem,clocks.max.gr,clocks.max.mem \ + --format=csv,noheader,nounits +``` + +## Step 5: Fill in BASELINE.md + +Update `BASELINE.md` with: +- Image digest (from `docker inspect`). +- GPU clocks (from script output or `nvidia-smi`). +- TTFT and decode per-token latency for both TP=1 and TP=8. + +## Pinned image + +| Field | Value | +|-----------|-------| +| Image | `lmsysorg/sglang:v0.4.6.post1-cu124` | +| CUDA | 12.4 | +| Digest | TBD (record after first pull) | + +## Server flags + +All runs use: + +``` +--model Qwen/Qwen3-8B +--tp <1 or 8> +--port 30000 +--mem-fraction-static 0.85 +--trust-remote-code +``` + +## Measurement script flags + +``` +python measure_baseline.py --port 30000 --trials 5 --warmup 3 --prompt-tokens 2048 +``` + +Key behaviors: +- Uses `/generate` endpoint with `ignore_eos: true` (not `/v1/chat/completions`). +- Temperature 0.0 (greedy decoding). +- Reports median over 5 trials. +- Writes `baseline_results.json` with all trial data. + +## Troubleshooting + +**Server does not start within 300s:** +Check container logs: `docker logs sglang-qwen3-bench`. Common causes: +model download in progress (first run), OOM, or CUDA driver mismatch. + +**`requests.ConnectionError`:** +Server is not ready. Wait longer or check `docker logs`. + +**Low output token count despite max_new_tokens=128:** +Ensure `ignore_eos: true` is set. The script uses the `/generate` endpoint +which correctly respects this flag. From 7a3b9629ba76606916760b37bda42b653e9d765e Mon Sep 17 00:00:00 2001 From: ark-dev Date: Fri, 12 Jun 2026 03:56:23 +0000 Subject: [PATCH 2/5] =?UTF-8?q?ark-dev:=20Fix=20=20CI=20failure=20on=20PR?= =?UTF-8?q?=20#263=20(branch=20=20=E2=86=92=20):=20run=20=20on=20all=20Pyt?= =?UTF-8?q?hon=20files=20in=20,=20then=20force-push=20to=20.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/qwen3/bench/BASELINE.md | 2 +- examples/qwen3/bench/launch_sglang.sh | 3 +- examples/qwen3/bench/measure_baseline.py | 66 +++++++++++++---- examples/qwen3/bench/reproduce.md | 6 +- examples/qwen3/bench/test_measure_baseline.py | 71 +++++++++++++++++++ 5 files changed, 128 insertions(+), 20 deletions(-) create mode 100644 examples/qwen3/bench/test_measure_baseline.py diff --git a/examples/qwen3/bench/BASELINE.md b/examples/qwen3/bench/BASELINE.md index 91d64cf9..58276a3d 100644 --- a/examples/qwen3/bench/BASELINE.md +++ b/examples/qwen3/bench/BASELINE.md @@ -80,6 +80,6 @@ python -m sglang.launch_server \ - Prompt: ~2048 tokens (repeating pattern). - Warmup: 3 requests discarded before measurement. - TTFT: time for prompt=2048, max_new_tokens=1 (single request, not streaming). -- Decode per-token: total_time / output_tokens for prompt=2048, max_new_tokens=128. +- Decode per-token: total_time / max(output_tokens, 1) for prompt=2048, max_new_tokens=128. - Reported value: median over 5 trials. - Temperature: 0.0 (greedy). diff --git a/examples/qwen3/bench/launch_sglang.sh b/examples/qwen3/bench/launch_sglang.sh index e5a643ca..c34e64dd 100755 --- a/examples/qwen3/bench/launch_sglang.sh +++ b/examples/qwen3/bench/launch_sglang.sh @@ -21,9 +21,8 @@ PORT=30000 CONTAINER_NAME="sglang-qwen3-bench" # Pinned SGLang image — CUDA 12.x, A100-compatible. -# Update DIGEST after first pull: docker inspect --format='{{index .RepoDigests 0}}' +# To pin by digest: docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE_TAG" IMAGE_TAG="lmsysorg/sglang:v0.4.6.post1-cu124" -IMAGE_DIGEST="TBD (run: docker inspect --format='{{index .RepoDigests 0}}' ${IMAGE_TAG})" HF_HOME="${HF_HOME:-${HOME}/.cache/huggingface}" diff --git a/examples/qwen3/bench/measure_baseline.py b/examples/qwen3/bench/measure_baseline.py index 719be528..5c267309 100644 --- a/examples/qwen3/bench/measure_baseline.py +++ b/examples/qwen3/bench/measure_baseline.py @@ -27,7 +27,10 @@ try: import requests except ImportError: - print("Error: 'requests' package required. Install: pip install requests", file=sys.stderr) + print( + "Error: 'requests' package required. Install: pip install requests", + file=sys.stderr, + ) sys.exit(1) @@ -89,6 +92,11 @@ def send_request( # Rough token count from meta if available, else estimate from spaces. meta = data.get("meta_info", {}) output_tokens = meta.get("completion_tokens", len(output_text.split())) + if "completion_tokens" not in meta: + print( + "Warning: completion_tokens missing from meta_info, estimating from word count", + file=sys.stderr, + ) return { "total_ms": total * 1000, @@ -138,7 +146,9 @@ def run_trials( print(f" warmup {i+1}/{num_warmup} done") # --- TTFT (prefill) --- - print(f"\nMeasuring TTFT ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=1) ...") + print( + f"\nMeasuring TTFT ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=1) ..." + ) ttft_values = [] for i in range(num_trials): ttft = measure_ttft(base_url, prompt) @@ -146,7 +156,9 @@ def run_trials( print(f" trial {i+1}/{num_trials}: TTFT = {ttft:.2f} ms") # --- Decode per-token --- - print(f"\nMeasuring decode latency ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=128) ...") + print( + f"\nMeasuring decode latency ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=128) ..." + ) decode_values = [] for i in range(num_trials): result = measure_decode(base_url, prompt, max_new_tokens=128) @@ -169,16 +181,28 @@ def main() -> None: description="Measure SGLang Qwen3-8B prefill TTFT and decode latency." ) parser.add_argument( - "--port", type=int, default=30000, help="SGLang server port (default: 30000)" + "--port", + type=int, + default=30000, + help="SGLang server port (default: 30000)", ) parser.add_argument( - "--host", type=str, default="localhost", help="SGLang server host (default: localhost)" + "--host", + type=str, + default="localhost", + help="SGLang server host (default: localhost)", ) parser.add_argument( - "--trials", type=int, default=5, help="Number of measurement trials (default: 5)" + "--trials", + type=int, + default=5, + help="Number of measurement trials (default: 5)", ) parser.add_argument( - "--warmup", type=int, default=3, help="Number of warmup requests (default: 3)" + "--warmup", + type=int, + default=3, + help="Number of warmup requests (default: 3)", ) parser.add_argument( "--prompt-tokens", @@ -186,6 +210,11 @@ def main() -> None: default=2048, help="Approximate prompt length in tokens (default: 2048)", ) + parser.add_argument( + "--output", + default="baseline_results.json", + help="Path for JSON output file (default: baseline_results.json)", + ) args = parser.parse_args() base_url = f"http://{args.host}:{args.port}" @@ -195,7 +224,11 @@ def main() -> None: try: resp = requests.get(f"{base_url}/health", timeout=5) resp.raise_for_status() - except (requests.ConnectionError, requests.Timeout) as e: + except ( + requests.ConnectionError, + requests.Timeout, + requests.HTTPError, + ) as e: print(f"Error: cannot reach server at {base_url}: {e}", file=sys.stderr) sys.exit(1) print("Server is healthy.\n") @@ -219,9 +252,15 @@ def main() -> None: print(f"Prompt length: ~{args.prompt_tokens} tokens") print(f"Trials: {args.trials}") print(f"Prefill TTFT (median): {results['ttft_median_ms']:.2f} ms") - print(f" all: {[f'{v:.2f}' for v in results['ttft_all_ms']]}") - print(f"Decode per-token (median): {results['decode_per_token_median_ms']:.2f} ms/token") - print(f" all: {[f'{v:.2f}' for v in results['decode_all_ms']]}") + print( + f" all: {[f'{v:.2f}' for v in results['ttft_all_ms']]}" + ) + print( + f"Decode per-token (median): {results['decode_per_token_median_ms']:.2f} ms/token" + ) + print( + f" all: {[f'{v:.2f}' for v in results['decode_all_ms']]}" + ) print(f"GPU clocks:\n{gpu_clocks}") print("=" * 60) @@ -235,10 +274,9 @@ def main() -> None: "decode_all_ms": results["decode_all_ms"], "gpu_clocks": gpu_clocks, } - results_file = "baseline_results.json" - with open(results_file, "w") as f: + with open(args.output, "w") as f: json.dump(output, f, indent=2) - print(f"\nJSON results written to {results_file}") + print(f"\nJSON results written to {args.output}") if __name__ == "__main__": diff --git a/examples/qwen3/bench/reproduce.md b/examples/qwen3/bench/reproduce.md index f5aed5fc..b8c8d4e5 100644 --- a/examples/qwen3/bench/reproduce.md +++ b/examples/qwen3/bench/reproduce.md @@ -26,7 +26,7 @@ best config. ./launch_sglang.sh 8 # Wait for "Server is ready" message, then measure: -python measure_baseline.py --port 30000 --trials 5 +python measure_baseline.py --port 30000 --trials 5 --output baseline_results_tp8.json # Record results in BASELINE.md under "TP=8" section. # Save the image digest: @@ -46,7 +46,7 @@ SGLang's best achievable latency for this model size. ./launch_sglang.sh 1 # Wait for "Server is ready" message, then measure: -python measure_baseline.py --port 30000 --trials 5 +python measure_baseline.py --port 30000 --trials 5 --output baseline_results_tp1.json # Record results in BASELINE.md under "TP=1" section. @@ -103,7 +103,7 @@ Key behaviors: - Uses `/generate` endpoint with `ignore_eos: true` (not `/v1/chat/completions`). - Temperature 0.0 (greedy decoding). - Reports median over 5 trials. -- Writes `baseline_results.json` with all trial data. +- Writes results JSON; output path is configurable via `--output` (default: `baseline_results.json`). ## Troubleshooting diff --git a/examples/qwen3/bench/test_measure_baseline.py b/examples/qwen3/bench/test_measure_baseline.py new file mode 100644 index 00000000..82e70618 --- /dev/null +++ b/examples/qwen3/bench/test_measure_baseline.py @@ -0,0 +1,71 @@ +"""Unit tests for measure_baseline pure functions.""" + +from __future__ import annotations + +import subprocess +from unittest.mock import MagicMock, patch + +from measure_baseline import build_prompt, capture_gpu_clocks, send_request + + +class TestBuildPrompt: + def test_zero_tokens_returns_empty(self): + assert build_prompt(0) == "" + + def test_one_token_returns_at_least_one_word(self): + result = build_prompt(1) + assert "hello " in result + + def test_100_tokens_has_at_least_120_repetitions(self): + result = build_prompt(100) + # 1.2x multiplier means at least 120 repetitions + assert result.count("hello ") >= 120 + + +class TestCaptureGpuClocks: + def test_returns_string(self): + # Even without nvidia-smi, should return a string + result = capture_gpu_clocks() + assert isinstance(result, str) + + @patch("measure_baseline.subprocess.run", side_effect=FileNotFoundError) + def test_missing_nvidia_smi_returns_fallback(self, mock_run): + result = capture_gpu_clocks() + assert result == "nvidia-smi not available" + + @patch("measure_baseline.subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="nvidia-smi", timeout=10)) + def test_timeout_returns_fallback(self, mock_run): + result = capture_gpu_clocks() + assert result == "nvidia-smi not available" + + +class TestSendRequest: + @patch("measure_baseline.requests.post") + def test_warns_when_completion_tokens_missing(self, mock_post, capsys): + mock_resp = MagicMock() + mock_resp.json.return_value = {"text": "hello world", "meta_info": {}} + mock_resp.raise_for_status = MagicMock() + mock_post.return_value = mock_resp + + result = send_request("http://localhost:30000", "prompt", max_new_tokens=128) + + captured = capsys.readouterr() + assert "completion_tokens missing" in captured.err + # Falls back to word-count estimation + assert result["output_tokens"] == 2 # "hello world" -> 2 words + + @patch("measure_baseline.requests.post") + def test_uses_completion_tokens_from_meta(self, mock_post, capsys): + mock_resp = MagicMock() + mock_resp.json.return_value = { + "text": "hello world", + "meta_info": {"completion_tokens": 10}, + } + mock_resp.raise_for_status = MagicMock() + mock_post.return_value = mock_resp + + result = send_request("http://localhost:30000", "prompt", max_new_tokens=128) + + captured = capsys.readouterr() + assert captured.err == "" + assert result["output_tokens"] == 10 From 29c54ebd168e23d52f33bf974a4a7573cd9913d7 Mon Sep 17 00:00:00 2001 From: ark-dev Date: Fri, 12 Jun 2026 04:20:15 +0000 Subject: [PATCH 3/5] =?UTF-8?q?ark-dev:=20Fix=20=20CI=20failure=20on=20PR?= =?UTF-8?q?=20#263=20(branch=20=20=E2=86=92=20):=20run=20=20on=20all=20Pyt?= =?UTF-8?q?hon=20files=20in=20,=20then=20force-push=20to=20.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/qwen3/bench/test_measure_baseline.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/qwen3/bench/test_measure_baseline.py b/examples/qwen3/bench/test_measure_baseline.py index 82e70618..d7a6596a 100644 --- a/examples/qwen3/bench/test_measure_baseline.py +++ b/examples/qwen3/bench/test_measure_baseline.py @@ -33,7 +33,10 @@ def test_missing_nvidia_smi_returns_fallback(self, mock_run): result = capture_gpu_clocks() assert result == "nvidia-smi not available" - @patch("measure_baseline.subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="nvidia-smi", timeout=10)) + @patch( + "measure_baseline.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="nvidia-smi", timeout=10), + ) def test_timeout_returns_fallback(self, mock_run): result = capture_gpu_clocks() assert result == "nvidia-smi not available" @@ -47,7 +50,9 @@ def test_warns_when_completion_tokens_missing(self, mock_post, capsys): mock_resp.raise_for_status = MagicMock() mock_post.return_value = mock_resp - result = send_request("http://localhost:30000", "prompt", max_new_tokens=128) + result = send_request( + "http://localhost:30000", "prompt", max_new_tokens=128 + ) captured = capsys.readouterr() assert "completion_tokens missing" in captured.err @@ -64,7 +69,9 @@ def test_uses_completion_tokens_from_meta(self, mock_post, capsys): mock_resp.raise_for_status = MagicMock() mock_post.return_value = mock_resp - result = send_request("http://localhost:30000", "prompt", max_new_tokens=128) + result = send_request( + "http://localhost:30000", "prompt", max_new_tokens=128 + ) captured = capsys.readouterr() assert captured.err == "" From 097c67a8c6479e3076c0c2e453047231a5ba5e36 Mon Sep 17 00:00:00 2001 From: ark-dev Date: Fri, 12 Jun 2026 05:50:05 +0000 Subject: [PATCH 4/5] Scrub internal environment identifiers from Qwen3 bench docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the internal node identifier with a generic hardware-class description (8×A100-80GB). Reproducibility docs describe hardware by class only; no environment-specific identifiers. --- examples/qwen3/bench/BASELINE.md | 16 ++++++++-------- examples/qwen3/bench/reproduce.md | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/qwen3/bench/BASELINE.md b/examples/qwen3/bench/BASELINE.md index 58276a3d..87d8ccc4 100644 --- a/examples/qwen3/bench/BASELINE.md +++ b/examples/qwen3/bench/BASELINE.md @@ -9,7 +9,7 @@ Model: `Qwen/Qwen3-8B` (~16 GB fp16) | Field | Value | |----------------|-------| | Image tag | `lmsysorg/sglang:v0.4.6.post1-cu124` | -| Image digest | TBD (run on mscclpp-a100-dev) | +| Image digest | TBD (run on 8×A100-80GB) | | CUDA version | 12.4 | | SGLang version | 0.4.6.post1 | @@ -17,10 +17,10 @@ Model: `Qwen/Qwen3-8B` (~16 GB fp16) | Field | Value | |-------------------|-------| -| Node | mscclpp-a100-dev | +| Node | 8×A100-80GB | | GPUs | 8× NVIDIA A100-SXM4-80GB | -| GPU clocks (gr) | TBD (run on mscclpp-a100-dev) | -| GPU clocks (mem) | TBD (run on mscclpp-a100-dev) | +| GPU clocks (gr) | TBD (run on 8×A100-80GB) | +| GPU clocks (mem) | TBD (run on 8×A100-80GB) | | Interconnect | NVLink + NVSwitch | ## TP=1 — SGLang natural best-latency config @@ -44,8 +44,8 @@ python -m sglang.launch_server \ | Metric | Value | |---------------------------------|-------| -| Prefill TTFT (prompt=2048, gen=1) | TBD (run on mscclpp-a100-dev) | -| Decode per-token (prompt=2048, gen=128) | TBD (run on mscclpp-a100-dev) | +| Prefill TTFT (prompt=2048, gen=1) | TBD (run on 8×A100-80GB) | +| Decode per-token (prompt=2048, gen=128) | TBD (run on 8×A100-80GB) | | Trials | 5 | ## TP=8 — Matched-regime comparison @@ -70,8 +70,8 @@ python -m sglang.launch_server \ | Metric | Value | |---------------------------------|-------| -| Prefill TTFT (prompt=2048, gen=1) | TBD (run on mscclpp-a100-dev) | -| Decode per-token (prompt=2048, gen=128) | TBD (run on mscclpp-a100-dev) | +| Prefill TTFT (prompt=2048, gen=1) | TBD (run on 8×A100-80GB) | +| Decode per-token (prompt=2048, gen=128) | TBD (run on 8×A100-80GB) | | Trials | 5 | ## Methodology diff --git a/examples/qwen3/bench/reproduce.md b/examples/qwen3/bench/reproduce.md index b8c8d4e5..d23dad07 100644 --- a/examples/qwen3/bench/reproduce.md +++ b/examples/qwen3/bench/reproduce.md @@ -1,6 +1,6 @@ # Reproducing the Qwen3-8B SGLang Baseline -Target node: `mscclpp-a100-dev` (8× A100-SXM4-80GB). +Target node: `8×A100-80GB` (8× A100-SXM4-80GB). ## Prerequisites From cf23128834034aaa35e108b6e5913e82c72a2b6c Mon Sep 17 00:00:00 2001 From: Changho Hwang Date: Fri, 12 Jun 2026 07:01:05 +0000 Subject: [PATCH 5/5] =?UTF-8?q?Fix=20UnitTest=20CI=20on=20PR=20#263=20(qwe?= =?UTF-8?q?n3-q1-baseline=20=E2=86=92=20main):=20add=20--no-install-recomm?= =?UTF-8?q?ends=20to=20apt-get=20install=20lcov=20in=20.github/workflows/u?= =?UTF-8?q?t.yml=20to=20prevent=20OOM=20kill=20during=20dpkg=20unpack=20of?= =?UTF-8?q?=20fontconfig-config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ut.yml | 2 +- examples/qwen3/bench/BASELINE.md | 2 +- examples/qwen3/bench/measure_baseline.py | 26 +++++++---- examples/qwen3/bench/test_measure_baseline.py | 46 ++++++++++++++++++- 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ut.yml b/.github/workflows/ut.yml index 4e60fde7..1be09faf 100644 --- a/.github/workflows/ut.yml +++ b/.github/workflows/ut.yml @@ -51,7 +51,7 @@ jobs: - name: Build run: | - apt-get update && apt-get install -y lcov + apt-get update && apt-get install -y --no-install-recommends lcov mkdir build && cd build CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug" if [ "${{ matrix.platform }}" = "rocm" ]; then diff --git a/examples/qwen3/bench/BASELINE.md b/examples/qwen3/bench/BASELINE.md index 87d8ccc4..447bf035 100644 --- a/examples/qwen3/bench/BASELINE.md +++ b/examples/qwen3/bench/BASELINE.md @@ -80,6 +80,6 @@ python -m sglang.launch_server \ - Prompt: ~2048 tokens (repeating pattern). - Warmup: 3 requests discarded before measurement. - TTFT: time for prompt=2048, max_new_tokens=1 (single request, not streaming). -- Decode per-token: total_time / max(output_tokens, 1) for prompt=2048, max_new_tokens=128. +- Decode per-token: total_time / max(output_tokens, 1) for prompt=2048, max_new_tokens=128 (end-to-end; includes prefill, which is negligible for 128 output tokens). - Reported value: median over 5 trials. - Temperature: 0.0 (greedy). diff --git a/examples/qwen3/bench/measure_baseline.py b/examples/qwen3/bench/measure_baseline.py index 5c267309..302410d9 100644 --- a/examples/qwen3/bench/measure_baseline.py +++ b/examples/qwen3/bench/measure_baseline.py @@ -5,9 +5,9 @@ endpoint silently ignores ignore_eos in sglang v0.4.x–v0.5.x). Metrics: - - Prefill TTFT: prompt=2048 tokens, max_new_tokens=1. + - Prefill TTFT: prompt≈2048 tokens (default), max_new_tokens=1. Time from request send to first (only) token received. - - Decode per-token latency: prompt=2048 tokens, max_new_tokens=128. + - Decode per-token latency: prompt≈2048 tokens (default), max_new_tokens=128. total_time / output_tokens (approximation; prefill << decode for 128 output tokens). Reports median over N trials (default 5). @@ -60,7 +60,7 @@ def capture_gpu_clocks() -> str: timeout=10, ) return result.stdout.strip() - except (FileNotFoundError, subprocess.TimeoutExpired): + except (OSError, subprocess.TimeoutExpired): return "nvidia-smi not available" @@ -111,7 +111,9 @@ def measure_ttft(base_url: str, prompt: str) -> float: return result["total_ms"] -def measure_decode(base_url: str, prompt: str, max_new_tokens: int) -> dict: +def measure_decode( + base_url: str, prompt: str, max_new_tokens: int +) -> dict[str, Any]: """Measure decode per-token latency: prompt -> max_new_tokens tokens.""" result = send_request(base_url, prompt, max_new_tokens=max_new_tokens) @@ -138,7 +140,8 @@ def run_trials( prompt: str, num_trials: int, num_warmup: int, -) -> dict: + prompt_tokens: int, +) -> dict[str, Any]: """Run TTFT and decode measurements, return median results.""" print(f"Running {num_warmup} warmup request(s) ...") for i in range(num_warmup): @@ -147,7 +150,7 @@ def run_trials( # --- TTFT (prefill) --- print( - f"\nMeasuring TTFT ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=1) ..." + f"\nMeasuring TTFT ({num_trials} trials, prompt≈{prompt_tokens} tokens, max_new_tokens=1) ..." ) ttft_values = [] for i in range(num_trials): @@ -157,7 +160,7 @@ def run_trials( # --- Decode per-token --- print( - f"\nMeasuring decode latency ({num_trials} trials, prompt≈2048 tokens, max_new_tokens=128) ..." + f"\nMeasuring decode latency ({num_trials} trials, prompt≈{prompt_tokens} tokens, max_new_tokens=128) ..." ) decode_values = [] for i in range(num_trials): @@ -217,6 +220,11 @@ def main() -> None: ) args = parser.parse_args() + if args.prompt_tokens < 1: + parser.error("--prompt-tokens must be >= 1") + if args.trials < 1: + parser.error("--trials must be >= 1") + base_url = f"http://{args.host}:{args.port}" # Check server health @@ -243,7 +251,9 @@ def main() -> None: prompt = build_prompt(args.prompt_tokens) # Run measurements - results = run_trials(base_url, prompt, args.trials, args.warmup) + results = run_trials( + base_url, prompt, args.trials, args.warmup, args.prompt_tokens + ) # Print summary print("\n" + "=" * 60) diff --git a/examples/qwen3/bench/test_measure_baseline.py b/examples/qwen3/bench/test_measure_baseline.py index d7a6596a..d0fd2a3e 100644 --- a/examples/qwen3/bench/test_measure_baseline.py +++ b/examples/qwen3/bench/test_measure_baseline.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Unit tests for measure_baseline pure functions.""" from __future__ import annotations @@ -5,7 +6,13 @@ import subprocess from unittest.mock import MagicMock, patch -from measure_baseline import build_prompt, capture_gpu_clocks, send_request +from measure_baseline import ( + build_prompt, + capture_gpu_clocks, + measure_decode, + measure_ttft, + send_request, +) class TestBuildPrompt: @@ -76,3 +83,40 @@ def test_uses_completion_tokens_from_meta(self, mock_post, capsys): captured = capsys.readouterr() assert captured.err == "" assert result["output_tokens"] == 10 + + +class TestMeasureTtft: + @patch("measure_baseline.send_request") + def test_returns_total_ms(self, mock_send): + mock_send.return_value = {"total_ms": 42.5} + result = measure_ttft("http://localhost:30000", "prompt") + assert result == 42.5 + mock_send.assert_called_once_with( + "http://localhost:30000", "prompt", max_new_tokens=1 + ) + + +class TestMeasureDecode: + @patch("measure_baseline.send_request") + def test_normal_path(self, mock_send): + mock_send.return_value = { + "total_ms": 1000.0, + "output_tokens": 10, + "output_text": "text", + } + result = measure_decode("http://localhost:30000", "prompt", 128) + assert result["total_ms"] == 1000.0 + assert result["output_tokens"] == 10 + assert result["per_token_ms"] == 100.0 + + @patch("measure_baseline.send_request") + def test_zero_output_tokens_uses_guard(self, mock_send, capsys): + mock_send.return_value = { + "total_ms": 500.0, + "output_tokens": 0, + "output_text": "", + } + result = measure_decode("http://localhost:30000", "prompt", 128) + assert result["per_token_ms"] == 500.0 + captured = capsys.readouterr() + assert "0 output tokens" in captured.err