diff --git a/.github/workflows/ut.yml b/.github/workflows/ut.yml index 4e60fde7..1be09faf 100644 --- a/.github/workflows/ut.yml +++ b/.github/workflows/ut.yml @@ -51,7 +51,7 @@ jobs: - name: Build run: | - apt-get update && apt-get install -y lcov + apt-get update && apt-get install -y --no-install-recommends lcov mkdir build && cd build CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug" if [ "${{ matrix.platform }}" = "rocm" ]; then diff --git a/examples/qwen3/bench/BASELINE.md b/examples/qwen3/bench/BASELINE.md new file mode 100644 index 00000000..447bf035 --- /dev/null +++ b/examples/qwen3/bench/BASELINE.md @@ -0,0 +1,85 @@ +# SGLang Qwen3-8B Baseline Latency + +> **Note:** Values marked "TBD" will be filled after running benchmarks on the target hardware. + +Model: `Qwen/Qwen3-8B` (~16 GB fp16) + +## Image + +| Field | Value | +|----------------|-------| +| Image tag | `lmsysorg/sglang:v0.4.6.post1-cu124` | +| Image digest | TBD (run on 8×A100-80GB) | +| CUDA version | 12.4 | +| SGLang version | 0.4.6.post1 | + +## Hardware + +| Field | Value | +|-------------------|-------| +| Node | 8×A100-80GB | +| GPUs | 8× NVIDIA A100-SXM4-80GB | +| GPU clocks (gr) | TBD (run on 8×A100-80GB) | +| GPU clocks (mem) | TBD (run on 8×A100-80GB) | +| Interconnect | NVLink + NVSwitch | + +## TP=1 — SGLang natural best-latency config + +Single-GPU inference. For an 8B model (~16 GB fp16), TP=1 is the +lowest-overhead configuration and represents SGLang's best achievable +latency for this model size. + +### Server flags + +``` +python -m sglang.launch_server \ + --model Qwen/Qwen3-8B \ + --tp 1 \ + --port 30000 \ + --mem-fraction-static 0.85 \ + --trust-remote-code +``` + +### Results + +| Metric | Value | +|---------------------------------|-------| +| Prefill TTFT (prompt=2048, gen=1) | TBD (run on 8×A100-80GB) | +| Decode per-token (prompt=2048, gen=128) | TBD (run on 8×A100-80GB) | +| Trials | 5 | + +## TP=8 — Matched-regime comparison + +TP=8 for an 8B model is the ARK-favorable regime: it minimizes per-GPU +compute and amplifies kernel-launch and communication overhead +(hypothesis 1). This matches the parallelism used by the ARK target +but is **not** SGLang's natural best config for this model size. + +### Server flags + +``` +python -m sglang.launch_server \ + --model Qwen/Qwen3-8B \ + --tp 8 \ + --port 30000 \ + --mem-fraction-static 0.85 \ + --trust-remote-code +``` + +### Results + +| Metric | Value | +|---------------------------------|-------| +| Prefill TTFT (prompt=2048, gen=1) | TBD (run on 8×A100-80GB) | +| Decode per-token (prompt=2048, gen=128) | TBD (run on 8×A100-80GB) | +| Trials | 5 | + +## Methodology + +- Endpoint: `/generate` with `ignore_eos: true` (not `/v1/chat/completions`). +- Prompt: ~2048 tokens (repeating pattern). +- Warmup: 3 requests discarded before measurement. +- TTFT: time for prompt=2048, max_new_tokens=1 (single request, not streaming). +- Decode per-token: total_time / max(output_tokens, 1) for prompt=2048, max_new_tokens=128 (end-to-end; includes prefill, which is negligible for 128 output tokens). +- Reported value: median over 5 trials. +- Temperature: 0.0 (greedy). diff --git a/examples/qwen3/bench/launch_sglang.sh b/examples/qwen3/bench/launch_sglang.sh new file mode 100755 index 00000000..c34e64dd --- /dev/null +++ b/examples/qwen3/bench/launch_sglang.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Launch SGLang server with Qwen3-8B on A100 GPUs. +# +# Usage: +# ./launch_sglang.sh # TP=8 (default) +# ./launch_sglang.sh 1 # TP=1 +# ./launch_sglang.sh 8 # TP=8 +# +# Prerequisites: +# - Docker with NVIDIA runtime +# - 8×A100-80GB (for TP=8) or 1×A100-80GB (for TP=1) +# - HuggingFace cache at $HF_HOME (default: ~/.cache/huggingface) + +set -euo pipefail + +# --- Configuration ----------------------------------------------------------- + +TP="${1:-8}" +MODEL="Qwen/Qwen3-8B" +PORT=30000 +CONTAINER_NAME="sglang-qwen3-bench" + +# Pinned SGLang image — CUDA 12.x, A100-compatible. +# To pin by digest: docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE_TAG" +IMAGE_TAG="lmsysorg/sglang:v0.4.6.post1-cu124" + +HF_HOME="${HF_HOME:-${HOME}/.cache/huggingface}" + +# --- Validate ----------------------------------------------------------------- + +if [[ "${TP}" != "1" && "${TP}" != "8" ]]; then + echo "Error: TP must be 1 or 8, got '${TP}'" >&2 + exit 1 +fi + +# --- Pull image --------------------------------------------------------------- + +echo "Pulling ${IMAGE_TAG} ..." +docker pull "${IMAGE_TAG}" + +# --- Stop any existing container ---------------------------------------------- + +if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then + echo "Stopping existing container '${CONTAINER_NAME}' ..." + docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 +fi + +# --- Launch server ------------------------------------------------------------ + +echo "Launching SGLang server: model=${MODEL}, tp=${TP}, port=${PORT}" +docker run -d \ + --name "${CONTAINER_NAME}" \ + --gpus all \ + --ipc=host \ + --network=host \ + -v "${HF_HOME}:/root/.cache/huggingface" \ + -e HF_HOME=/root/.cache/huggingface \ + "${IMAGE_TAG}" \ + python -m sglang.launch_server \ + --model "${MODEL}" \ + --tp "${TP}" \ + --port "${PORT}" \ + --mem-fraction-static 0.85 \ + --trust-remote-code + +echo "" +echo "Container '${CONTAINER_NAME}' started." +echo "Waiting for server to be ready on port ${PORT} ..." + +# --- Wait for server readiness ------------------------------------------------ + +MAX_WAIT=300 +ELAPSED=0 +while ! curl -sf "http://localhost:${PORT}/health" >/dev/null 2>&1; do + sleep 5 + ELAPSED=$((ELAPSED + 5)) + if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then + echo "Error: server did not become ready within ${MAX_WAIT}s." >&2 + echo "Check logs: docker logs ${CONTAINER_NAME}" >&2 + exit 1 + fi +done + +echo "Server is ready (waited ${ELAPSED}s)." +echo "" +echo "Run the benchmark:" +echo " python measure_baseline.py --port ${PORT}" diff --git a/examples/qwen3/bench/measure_baseline.py b/examples/qwen3/bench/measure_baseline.py new file mode 100644 index 00000000..302410d9 --- /dev/null +++ b/examples/qwen3/bench/measure_baseline.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Measure SGLang Qwen3-8B prefill TTFT and decode per-token latency. + +Uses the /generate endpoint with ignore_eos: true (the /v1/chat/completions +endpoint silently ignores ignore_eos in sglang v0.4.x–v0.5.x). + +Metrics: + - Prefill TTFT: prompt≈2048 tokens (default), max_new_tokens=1. + Time from request send to first (only) token received. + - Decode per-token latency: prompt≈2048 tokens (default), max_new_tokens=128. + total_time / output_tokens (approximation; prefill << decode for 128 output tokens). + +Reports median over N trials (default 5). +Captures GPU clocks via nvidia-smi at start of run. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import subprocess +import sys +import time +from typing import Any + +try: + import requests +except ImportError: + print( + "Error: 'requests' package required. Install: pip install requests", + file=sys.stderr, + ) + sys.exit(1) + + +def build_prompt(num_tokens: int) -> str: + """Build a prompt that is approximately `num_tokens` tokens long. + + Uses a repeating word pattern. Actual token count depends on the + tokenizer, but "word " ≈ 1 token for most BPE tokenizers, so we + slightly over-generate to ensure we hit the target length. + """ + # Pad by 1.2x to account for BPE tokenizer variance (some words may produce <1 token). + word = "hello " + return word * int(num_tokens * 1.2) + + +def capture_gpu_clocks() -> str: + """Capture current GPU clocks via nvidia-smi.""" + try: + result = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=index,clocks.gr,clocks.mem,clocks.max.gr,clocks.max.mem", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=10, + ) + return result.stdout.strip() + except (OSError, subprocess.TimeoutExpired): + return "nvidia-smi not available" + + +def send_request( + base_url: str, prompt: str, max_new_tokens: int +) -> dict[str, Any]: + """Send a /generate request and measure timing. + + Returns dict with keys: total_ms, output_tokens, output_text. + """ + payload = { + "text": prompt, + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.0, + "ignore_eos": True, + }, + } + + start = time.perf_counter() + resp = requests.post(f"{base_url}/generate", json=payload, timeout=120) + total = time.perf_counter() - start + + resp.raise_for_status() + data = resp.json() + + # The /generate endpoint returns the generated text. + output_text = data.get("text", "") + # Rough token count from meta if available, else estimate from spaces. + meta = data.get("meta_info", {}) + output_tokens = meta.get("completion_tokens", len(output_text.split())) + if "completion_tokens" not in meta: + print( + "Warning: completion_tokens missing from meta_info, estimating from word count", + file=sys.stderr, + ) + + return { + "total_ms": total * 1000, + "output_tokens": output_tokens, + "output_text": output_text[:200], # truncate for display + } + + +def measure_ttft(base_url: str, prompt: str) -> float: + """Measure prefill TTFT: prompt -> 1 token.""" + result = send_request(base_url, prompt, max_new_tokens=1) + return result["total_ms"] + + +def measure_decode( + base_url: str, prompt: str, max_new_tokens: int +) -> dict[str, Any]: + """Measure decode per-token latency: prompt -> max_new_tokens tokens.""" + result = send_request(base_url, prompt, max_new_tokens=max_new_tokens) + + output_tokens = result["output_tokens"] + total_ms = result["total_ms"] + + # Per-token latency: we don't have streaming TTFT here, so we estimate + # per-token as total / output_tokens for the decode portion. + # For a more precise split, use streaming — but for a baseline this + # is sufficient since prefill << decode for 128 output tokens. + if output_tokens == 0: + print("Warning: server returned 0 output tokens", file=sys.stderr) + per_token_ms = total_ms / max(output_tokens, 1) + + return { + "total_ms": total_ms, + "output_tokens": output_tokens, + "per_token_ms": per_token_ms, + } + + +def run_trials( + base_url: str, + prompt: str, + num_trials: int, + num_warmup: int, + prompt_tokens: int, +) -> dict[str, Any]: + """Run TTFT and decode measurements, return median results.""" + print(f"Running {num_warmup} warmup request(s) ...") + for i in range(num_warmup): + send_request(base_url, prompt, max_new_tokens=1) + print(f" warmup {i+1}/{num_warmup} done") + + # --- TTFT (prefill) --- + print( + f"\nMeasuring TTFT ({num_trials} trials, prompt≈{prompt_tokens} tokens, max_new_tokens=1) ..." + ) + ttft_values = [] + for i in range(num_trials): + ttft = measure_ttft(base_url, prompt) + ttft_values.append(ttft) + print(f" trial {i+1}/{num_trials}: TTFT = {ttft:.2f} ms") + + # --- Decode per-token --- + print( + f"\nMeasuring decode latency ({num_trials} trials, prompt≈{prompt_tokens} tokens, max_new_tokens=128) ..." + ) + decode_values = [] + for i in range(num_trials): + result = measure_decode(base_url, prompt, max_new_tokens=128) + decode_values.append(result["per_token_ms"]) + print( + f" trial {i+1}/{num_trials}: {result['per_token_ms']:.2f} ms/token " + f"({result['output_tokens']} tokens in {result['total_ms']:.1f} ms)" + ) + + return { + "ttft_median_ms": statistics.median(ttft_values), + "ttft_all_ms": ttft_values, + "decode_per_token_median_ms": statistics.median(decode_values), + "decode_all_ms": decode_values, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Measure SGLang Qwen3-8B prefill TTFT and decode latency." + ) + parser.add_argument( + "--port", + type=int, + default=30000, + help="SGLang server port (default: 30000)", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="SGLang server host (default: localhost)", + ) + parser.add_argument( + "--trials", + type=int, + default=5, + help="Number of measurement trials (default: 5)", + ) + parser.add_argument( + "--warmup", + type=int, + default=3, + help="Number of warmup requests (default: 3)", + ) + parser.add_argument( + "--prompt-tokens", + type=int, + default=2048, + help="Approximate prompt length in tokens (default: 2048)", + ) + parser.add_argument( + "--output", + default="baseline_results.json", + help="Path for JSON output file (default: baseline_results.json)", + ) + args = parser.parse_args() + + if args.prompt_tokens < 1: + parser.error("--prompt-tokens must be >= 1") + if args.trials < 1: + parser.error("--trials must be >= 1") + + base_url = f"http://{args.host}:{args.port}" + + # Check server health + print(f"Checking server at {base_url} ...") + try: + resp = requests.get(f"{base_url}/health", timeout=5) + resp.raise_for_status() + except ( + requests.ConnectionError, + requests.Timeout, + requests.HTTPError, + ) as e: + print(f"Error: cannot reach server at {base_url}: {e}", file=sys.stderr) + sys.exit(1) + print("Server is healthy.\n") + + # Capture GPU clocks + print("GPU clocks:") + gpu_clocks = capture_gpu_clocks() + print(gpu_clocks) + print() + + # Build prompt + prompt = build_prompt(args.prompt_tokens) + + # Run measurements + results = run_trials( + base_url, prompt, args.trials, args.warmup, args.prompt_tokens + ) + + # Print summary + print("\n" + "=" * 60) + print("RESULTS SUMMARY") + print("=" * 60) + print(f"Prompt length: ~{args.prompt_tokens} tokens") + print(f"Trials: {args.trials}") + print(f"Prefill TTFT (median): {results['ttft_median_ms']:.2f} ms") + print( + f" all: {[f'{v:.2f}' for v in results['ttft_all_ms']]}" + ) + print( + f"Decode per-token (median): {results['decode_per_token_median_ms']:.2f} ms/token" + ) + print( + f" all: {[f'{v:.2f}' for v in results['decode_all_ms']]}" + ) + print(f"GPU clocks:\n{gpu_clocks}") + print("=" * 60) + + # Write JSON results + output = { + "prompt_tokens": args.prompt_tokens, + "trials": args.trials, + "ttft_median_ms": results["ttft_median_ms"], + "ttft_all_ms": results["ttft_all_ms"], + "decode_per_token_median_ms": results["decode_per_token_median_ms"], + "decode_all_ms": results["decode_all_ms"], + "gpu_clocks": gpu_clocks, + } + with open(args.output, "w") as f: + json.dump(output, f, indent=2) + print(f"\nJSON results written to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/examples/qwen3/bench/reproduce.md b/examples/qwen3/bench/reproduce.md new file mode 100644 index 00000000..d23dad07 --- /dev/null +++ b/examples/qwen3/bench/reproduce.md @@ -0,0 +1,119 @@ +# Reproducing the Qwen3-8B SGLang Baseline + +Target node: `8×A100-80GB` (8× A100-SXM4-80GB). + +## Prerequisites + +- Docker with NVIDIA runtime (`nvidia-docker2` or `--gpus` support). +- HuggingFace model cache with network access (model downloads on first run). +- Python 3.8+ with `requests` package on the host (for `measure_baseline.py`). + +## Step 1: Clone and navigate + +```bash +git clone +cd ark/examples/qwen3/bench +``` + +## Step 2: Run TP=8 (matched-regime comparison) + +TP=8 for an 8B model amplifies kernel-launch and communication overhead. +This is the matched-regime comparison for the ARK target, not SGLang's +best config. + +```bash +# Launch server (TP=8, default) +./launch_sglang.sh 8 + +# Wait for "Server is ready" message, then measure: +python measure_baseline.py --port 30000 --trials 5 --output baseline_results_tp8.json + +# Record results in BASELINE.md under "TP=8" section. +# Save the image digest: +docker inspect --format='{{index .RepoDigests 0}}' lmsysorg/sglang:v0.4.6.post1-cu124 + +# Stop server +docker rm -f sglang-qwen3-bench +``` + +## Step 3: Run TP=1 (SGLang natural best-latency config) + +TP=1 is the lowest-overhead config for an 8B model and represents +SGLang's best achievable latency for this model size. + +```bash +# Launch server (TP=1) +./launch_sglang.sh 1 + +# Wait for "Server is ready" message, then measure: +python measure_baseline.py --port 30000 --trials 5 --output baseline_results_tp1.json + +# Record results in BASELINE.md under "TP=1" section. + +# Stop server +docker rm -f sglang-qwen3-bench +``` + +## Step 4: Record GPU clocks + +`measure_baseline.py` captures GPU clocks automatically via `nvidia-smi`. +Copy the clock values from the script output into BASELINE.md. + +Alternatively, capture manually: + +```bash +nvidia-smi --query-gpu=index,clocks.gr,clocks.mem,clocks.max.gr,clocks.max.mem \ + --format=csv,noheader,nounits +``` + +## Step 5: Fill in BASELINE.md + +Update `BASELINE.md` with: +- Image digest (from `docker inspect`). +- GPU clocks (from script output or `nvidia-smi`). +- TTFT and decode per-token latency for both TP=1 and TP=8. + +## Pinned image + +| Field | Value | +|-----------|-------| +| Image | `lmsysorg/sglang:v0.4.6.post1-cu124` | +| CUDA | 12.4 | +| Digest | TBD (record after first pull) | + +## Server flags + +All runs use: + +``` +--model Qwen/Qwen3-8B +--tp <1 or 8> +--port 30000 +--mem-fraction-static 0.85 +--trust-remote-code +``` + +## Measurement script flags + +``` +python measure_baseline.py --port 30000 --trials 5 --warmup 3 --prompt-tokens 2048 +``` + +Key behaviors: +- Uses `/generate` endpoint with `ignore_eos: true` (not `/v1/chat/completions`). +- Temperature 0.0 (greedy decoding). +- Reports median over 5 trials. +- Writes results JSON; output path is configurable via `--output` (default: `baseline_results.json`). + +## Troubleshooting + +**Server does not start within 300s:** +Check container logs: `docker logs sglang-qwen3-bench`. Common causes: +model download in progress (first run), OOM, or CUDA driver mismatch. + +**`requests.ConnectionError`:** +Server is not ready. Wait longer or check `docker logs`. + +**Low output token count despite max_new_tokens=128:** +Ensure `ignore_eos: true` is set. The script uses the `/generate` endpoint +which correctly respects this flag. diff --git a/examples/qwen3/bench/test_measure_baseline.py b/examples/qwen3/bench/test_measure_baseline.py new file mode 100644 index 00000000..d0fd2a3e --- /dev/null +++ b/examples/qwen3/bench/test_measure_baseline.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Unit tests for measure_baseline pure functions.""" + +from __future__ import annotations + +import subprocess +from unittest.mock import MagicMock, patch + +from measure_baseline import ( + build_prompt, + capture_gpu_clocks, + measure_decode, + measure_ttft, + send_request, +) + + +class TestBuildPrompt: + def test_zero_tokens_returns_empty(self): + assert build_prompt(0) == "" + + def test_one_token_returns_at_least_one_word(self): + result = build_prompt(1) + assert "hello " in result + + def test_100_tokens_has_at_least_120_repetitions(self): + result = build_prompt(100) + # 1.2x multiplier means at least 120 repetitions + assert result.count("hello ") >= 120 + + +class TestCaptureGpuClocks: + def test_returns_string(self): + # Even without nvidia-smi, should return a string + result = capture_gpu_clocks() + assert isinstance(result, str) + + @patch("measure_baseline.subprocess.run", side_effect=FileNotFoundError) + def test_missing_nvidia_smi_returns_fallback(self, mock_run): + result = capture_gpu_clocks() + assert result == "nvidia-smi not available" + + @patch( + "measure_baseline.subprocess.run", + side_effect=subprocess.TimeoutExpired(cmd="nvidia-smi", timeout=10), + ) + def test_timeout_returns_fallback(self, mock_run): + result = capture_gpu_clocks() + assert result == "nvidia-smi not available" + + +class TestSendRequest: + @patch("measure_baseline.requests.post") + def test_warns_when_completion_tokens_missing(self, mock_post, capsys): + mock_resp = MagicMock() + mock_resp.json.return_value = {"text": "hello world", "meta_info": {}} + mock_resp.raise_for_status = MagicMock() + mock_post.return_value = mock_resp + + result = send_request( + "http://localhost:30000", "prompt", max_new_tokens=128 + ) + + captured = capsys.readouterr() + assert "completion_tokens missing" in captured.err + # Falls back to word-count estimation + assert result["output_tokens"] == 2 # "hello world" -> 2 words + + @patch("measure_baseline.requests.post") + def test_uses_completion_tokens_from_meta(self, mock_post, capsys): + mock_resp = MagicMock() + mock_resp.json.return_value = { + "text": "hello world", + "meta_info": {"completion_tokens": 10}, + } + mock_resp.raise_for_status = MagicMock() + mock_post.return_value = mock_resp + + result = send_request( + "http://localhost:30000", "prompt", max_new_tokens=128 + ) + + captured = capsys.readouterr() + assert captured.err == "" + assert result["output_tokens"] == 10 + + +class TestMeasureTtft: + @patch("measure_baseline.send_request") + def test_returns_total_ms(self, mock_send): + mock_send.return_value = {"total_ms": 42.5} + result = measure_ttft("http://localhost:30000", "prompt") + assert result == 42.5 + mock_send.assert_called_once_with( + "http://localhost:30000", "prompt", max_new_tokens=1 + ) + + +class TestMeasureDecode: + @patch("measure_baseline.send_request") + def test_normal_path(self, mock_send): + mock_send.return_value = { + "total_ms": 1000.0, + "output_tokens": 10, + "output_text": "text", + } + result = measure_decode("http://localhost:30000", "prompt", 128) + assert result["total_ms"] == 1000.0 + assert result["output_tokens"] == 10 + assert result["per_token_ms"] == 100.0 + + @patch("measure_baseline.send_request") + def test_zero_output_tokens_uses_guard(self, mock_send, capsys): + mock_send.return_value = { + "total_ms": 500.0, + "output_tokens": 0, + "output_text": "", + } + result = measure_decode("http://localhost:30000", "prompt", 128) + assert result["per_token_ms"] == 500.0 + captured = capsys.readouterr() + assert "0 output tokens" in captured.err