From cd47477aeb874623c021546ed639ed84dbd67ba2 Mon Sep 17 00:00:00 2001
From: ark-dev <ark-dev@local>
Date: Fri, 12 Jun 2026 05:24:10 +0000
Subject: [PATCH 1/4] =?UTF-8?q?ark-dev:=20Implement=20Q2:=20profile=20SGLa?=
 =?UTF-8?q?ng=20Qwen3-8B=20forward=20with=20torch.profiler=20for=20both=20?=
 =?UTF-8?q?prefill=20and=20decode=20phases;=20write=20PROFILE.md=20with=20?=
 =?UTF-8?q?a=20ranked=20per-component=20latency=20budget=20that=20re-ranks?=
 =?UTF-8?q?=20Q4=E2=80=93Q8.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 examples/qwen3/PROFILE.md                   |  60 +++
 examples/qwen3/bench/analyze_profile.sh     |  97 +++++
 examples/qwen3/bench/classify_kernels.py    | 317 +++++++++++++++
 examples/qwen3/bench/profile_sglang.py      | 299 ++++++++++++++
 examples/qwen3/bench/reproduce_profile.md   |  75 ++++
 examples/qwen3/bench/test_profile_sglang.py | 412 ++++++++++++++++++++
 6 files changed, 1260 insertions(+)
 create mode 100644 examples/qwen3/PROFILE.md
 create mode 100644 examples/qwen3/bench/analyze_profile.sh
 create mode 100644 examples/qwen3/bench/classify_kernels.py
 create mode 100644 examples/qwen3/bench/profile_sglang.py
 create mode 100644 examples/qwen3/bench/reproduce_profile.md
 create mode 100644 examples/qwen3/bench/test_profile_sglang.py

diff --git a/examples/qwen3/PROFILE.md b/examples/qwen3/PROFILE.md
new file mode 100644
index 00000000..b57d5b70
--- /dev/null
+++ b/examples/qwen3/PROFILE.md
@@ -0,0 +1,60 @@
+# Qwen3-8B Per-Component Latency Profile
+
+Per-component latency breakdown of SGLang Qwen3-8B (TP=8, batch=1) on
+8×A100-80GB.  Profiled with `torch.profiler` via `profile_sglang.py`.
+
+## Configuration
+
+| Parameter       | Value                                   |
+|-----------------|-----------------------------------------|
+| Model           | Qwen/Qwen3-8B                           |
+| SGLang image    | lmsysorg/sglang:v0.4.6.post1-cu124      |
+| Hardware        | 8×A100-80GB (SXM4)                      |
+| TP              | 8                                        |
+| Batch           | 1                                        |
+| Prompt length   | 2048 tokens                              |
+| Generate length | 128 tokens                               |
+| Profiler        | torch.profiler (wait=2, warmup=1, active=3) |
+
+## Prefill Phase (prompt=2048, max_new_tokens=1)
+
+| Component      | Kernel time (ms) | % of total | ARK target | Q-item |
+|----------------|-----------------|------------|------------|--------|
+| gemm_mlp       | TBD             | TBD        | TBD        | Q5     |
+| gemm_attention | TBD             | TBD        | TBD        | Q4     |
+| attention      | TBD             | TBD        | TBD        | Q6     |
+| norms_rope     | TBD             | TBD        | TBD        | Q7     |
+| nccl           | TBD             | TBD        | TBD        | Q8     |
+| embed_lm_head  | TBD             | TBD        | TBD        | —      |
+| other          | TBD             | TBD        | TBD        | —      |
+| **Total**      | **TBD**         | **100%**   |            |        |
+
+## Decode Phase (prompt=2048, max_new_tokens=128)
+
+| Component      | Kernel time (ms) | % of total | ARK target | Q-item |
+|----------------|-----------------|------------|------------|--------|
+| gemm_mlp       | TBD             | TBD        | TBD        | Q5     |
+| gemm_attention | TBD             | TBD        | TBD        | Q4     |
+| attention      | TBD             | TBD        | TBD        | Q6     |
+| norms_rope     | TBD             | TBD        | TBD        | Q7     |
+| nccl           | TBD             | TBD        | TBD        | Q8     |
+| embed_lm_head  | TBD             | TBD        | TBD        | —      |
+| other          | TBD             | TBD        | TBD        | —      |
+| **Total**      | **TBD**         | **100%**   |            |        |
+
+## Q4–Q8 Re-ranking
+
+Re-rank Q4–Q8 by descending kernel time once numbers land.  The component
+that consumes the most GPU time gets highest optimization priority.
+
+| Priority | Q-item | Component | Prefill % | Decode % |
+|----------|--------|-----------|-----------|----------|
+| 1        | TBD    | TBD       | TBD       | TBD      |
+| 2        | TBD    | TBD       | TBD       | TBD      |
+| 3        | TBD    | TBD       | TBD       | TBD      |
+| 4        | TBD    | TBD       | TBD       | TBD      |
+| 5        | TBD    | TBD       | TBD       | TBD      |
+
+Numbers and final ordering filled out-of-band after profiling on
+`mscclpp-a100-dev`.  See [reproduce_profile.md](bench/reproduce_profile.md)
+for repro steps.
diff --git a/examples/qwen3/bench/analyze_profile.sh b/examples/qwen3/bench/analyze_profile.sh
new file mode 100644
index 00000000..982b90a7
--- /dev/null
+++ b/examples/qwen3/bench/analyze_profile.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Analyze a torch.profiler Chrome-trace JSON for Qwen3-8B per-component budget.
+#
+# Usage:
+#   bash analyze_profile.sh <trace.json> [--tp 8]
+#
+# Steps:
+#   1. Run trace_analyzer.py (torch-profiler skill) for kernel/comm/gap sections.
+#   2. Run classify_kernels.py to produce the per-component latency budget.
+#
+# Prerequisites:
+#   - Python 3.10+
+#   - trace_analyzer.py path set via TRACE_ANALYZER env var, or auto-detected
+#     from the torch-profiler skill directory.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# --- Arguments ----------------------------------------------------------------
+
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 <trace.json> [--tp 8]" >&2
+    exit 1
+fi
+
+TRACE_FILE="$1"
+shift
+
+TP=8
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --tp) TP="$2"; shift 2 ;;
+        *) echo "Unknown argument: $1" >&2; exit 1 ;;
+    esac
+done
+
+if [ ! -f "${TRACE_FILE}" ]; then
+    echo "Error: trace file not found: ${TRACE_FILE}" >&2
+    exit 1
+fi
+
+# --- Locate trace_analyzer.py -------------------------------------------------
+
+TRACE_ANALYZER="${TRACE_ANALYZER:-}"
+if [ -z "${TRACE_ANALYZER}" ]; then
+    # Try common locations
+    for candidate in \
+        "${SCRIPT_DIR}/../../../.pi/skills/torch-profiler/scripts/trace_analyzer.py" \
+        "${HOME}/.pi/skills/torch-profiler/scripts/trace_analyzer.py"; do
+        if [ -f "${candidate}" ]; then
+            TRACE_ANALYZER="${candidate}"
+            break
+        fi
+    done
+fi
+
+# --- Step 1: trace_analyzer.py (if available) ---------------------------------
+
+OUTPUT_DIR="$(dirname "${TRACE_FILE}")/analysis"
+mkdir -p "${OUTPUT_DIR}"
+
+if [ -n "${TRACE_ANALYZER}" ] && [ -f "${TRACE_ANALYZER}" ]; then
+    echo "=== Trace Analyzer: kernel breakdown ==="
+    python3 "${TRACE_ANALYZER}" "${TRACE_FILE}" --section kernels --top-n 30 \
+        | tee "${OUTPUT_DIR}/kernels.txt"
+    echo ""
+
+    echo "=== Trace Analyzer: communication ==="
+    python3 "${TRACE_ANALYZER}" "${TRACE_FILE}" --section comm \
+        | tee "${OUTPUT_DIR}/comm.txt"
+    echo ""
+
+    echo "=== Trace Analyzer: GPU gaps ==="
+    python3 "${TRACE_ANALYZER}" "${TRACE_FILE}" --section gaps \
+        | tee "${OUTPUT_DIR}/gaps.txt"
+    echo ""
+
+    echo "=== Trace Analyzer: full summary ==="
+    python3 "${TRACE_ANALYZER}" "${TRACE_FILE}" --full \
+        -o "${OUTPUT_DIR}/full_analysis.md"
+    echo "Full analysis saved to ${OUTPUT_DIR}/full_analysis.md"
+    echo ""
+else
+    echo "Warning: trace_analyzer.py not found. Set TRACE_ANALYZER env var." >&2
+    echo "Skipping trace_analyzer.py step." >&2
+    echo ""
+fi
+
+# --- Step 2: per-component classification -------------------------------------
+
+echo "=== Per-component classification (TP=${TP}) ==="
+python3 "${SCRIPT_DIR}/classify_kernels.py" "${TRACE_FILE}" --tp "${TP}" \
+    | tee "${OUTPUT_DIR}/component_budget.md"
+
+echo ""
+echo "Analysis complete. Results in: ${OUTPUT_DIR}/"
diff --git a/examples/qwen3/bench/classify_kernels.py b/examples/qwen3/bench/classify_kernels.py
new file mode 100644
index 00000000..3b882b05
--- /dev/null
+++ b/examples/qwen3/bench/classify_kernels.py
@@ -0,0 +1,317 @@
+#!/usr/bin/env python3
+"""Classify GPU kernel names into per-component latency buckets.
+
+Component buckets
+-----------------
+  attention       FlashInfer / FlashAttention / cuDNN SDPA kernels
+  gemm_attention  GEMM kernels for Q/K/V/O projections
+  gemm_mlp        GEMM kernels for gate/up/down projections (SwiGLU MLP)
+  nccl            NCCL all-reduce / reduce-scatter / all-gather
+  norms_rope      RMSNorm, RoPE, QK-norm, SiLU, elementwise fused ops
+  embed_lm_head   Embedding lookup and lm_head GEMM (vocab-sized dim)
+  other           Kernel-launch gaps, CPU overhead, unclassified
+
+Classification table (kernel name patterns)
+-------------------------------------------
+  Pattern                           Component
+  ─────────────────────────────────  ────────────────
+  flash_*, fmha*, flashinfer*       attention
+  cudnn*sdpa*                       attention
+  cutlass_*, cublas*, sm{N}_xmma*   gemm_* (by shape)
+  ncclDevKernel_*, nccl_*           nccl
+  rms_norm*, rmsnorm*, layernorm*   norms_rope
+  fused_*norm*                      norms_rope
+  silu*, gelu*, elementwise*        norms_rope
+  rotary_*, rope_*                  norms_rope
+  (everything else)                 other
+
+GEMM shape disambiguation (Qwen3-8B, TP=8)
+-------------------------------------------
+  Attention projections  K/N ∈ {512, 128, 768}   (Q, K/V, fused QKV per GPU)
+  MLP projections        K/N ∈ {1792, 3584}      (gate/up per GPU, fused)
+  lm_head                K/N = 151936 or /TP     (vocab dimension)
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Component bucket names
+# ---------------------------------------------------------------------------
+
+ATTENTION = "attention"
+GEMM_ATTENTION = "gemm_attention"
+GEMM_MLP = "gemm_mlp"
+NCCL = "nccl"
+NORMS_ROPE = "norms_rope"
+EMBED_LM_HEAD = "embed_lm_head"
+OTHER = "other"
+
+ALL_COMPONENTS = [
+    ATTENTION,
+    GEMM_ATTENTION,
+    GEMM_MLP,
+    NCCL,
+    NORMS_ROPE,
+    EMBED_LM_HEAD,
+    OTHER,
+]
+
+# ---------------------------------------------------------------------------
+# Qwen3-8B model dimensions
+# ---------------------------------------------------------------------------
+
+HIDDEN = 4096
+NUM_HEADS = 32
+NUM_KV_HEADS = 8
+HEAD_DIM = 128
+INTERMEDIATE = 14336
+VOCAB = 151936
+
+# ---------------------------------------------------------------------------
+# Kernel name patterns → component (first match wins)
+# ---------------------------------------------------------------------------
+
+_KERNEL_PATTERNS: list[tuple[str, str]] = [
+    # Attention kernels
+    (r"flash_", ATTENTION),
+    (r"fmha", ATTENTION),
+    (r"flashinfer", ATTENTION),
+    (r"cudnn.*sdpa", ATTENTION),
+    (r"sdpa_", ATTENTION),
+    # NCCL
+    (r"ncclDevKernel", NCCL),
+    (r"nccl_", NCCL),
+    (r"ncclKernel", NCCL),
+    # Norms / RoPE / activations / elementwise
+    (r"rms_norm", NORMS_ROPE),
+    (r"rmsnorm", NORMS_ROPE),
+    (r"layernorm", NORMS_ROPE),
+    (r"layer_norm", NORMS_ROPE),
+    (r"fused_.*norm", NORMS_ROPE),
+    (r"rotary_", NORMS_ROPE),
+    (r"rope_", NORMS_ROPE),
+    (r"silu", NORMS_ROPE),
+    (r"gelu", NORMS_ROPE),
+    (r"elementwise", NORMS_ROPE),
+]
+
+_COMPILED_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(pat, re.IGNORECASE), comp) for pat, comp in _KERNEL_PATTERNS
+]
+
+# GEMM kernel name patterns (for shape-based disambiguation)
+_GEMM_PATTERNS: list[re.Pattern[str]] = [
+    re.compile(pat, re.IGNORECASE)
+    for pat in [
+        r"cutlass_",
+        r"cublas",
+        r"sm\d+_xmma",
+        r"gemm",
+        r"ampere_",
+        r"turing_",
+    ]
+]
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _is_gemm_kernel(name: str) -> bool:
+    """Return True if *name* matches a known GEMM kernel pattern."""
+    return any(p.search(name) for p in _GEMM_PATTERNS)
+
+
+def _classify_gemm_by_shape(
+    shapes: list[list[int]] | None,
+    tp: int = 8,
+) -> str:
+    """Disambiguate a GEMM kernel into attention / MLP / embed by shapes.
+
+    *shapes* comes from ``record_shapes=True`` in torch.profiler — a list of
+    input-tensor shapes.  We flatten all dimension values and look for
+    component-distinctive sizes.
+    """
+    if not shapes:
+        return OTHER
+
+    # Vocab-related dimensions
+    vocab_dims = {VOCAB, VOCAB // tp}
+
+    # Attention projection dimensions (per-GPU, excluding HIDDEN which is
+    # shared with MLP and therefore non-distinctive)
+    q_dim = HIDDEN // tp  # 512 for TP=8
+    kv_dim = (NUM_KV_HEADS * HEAD_DIM) // tp  # 128 for TP=8
+    fused_qkv = q_dim + 2 * kv_dim  # 768 for TP=8
+    attn_dims = {q_dim, kv_dim, fused_qkv}
+
+    # MLP projection dimensions (per-GPU)
+    mlp_dim = INTERMEDIATE // tp  # 1792 for TP=8
+    fused_gate_up = 2 * mlp_dim  # 3584 for TP=8
+    mlp_dims = {mlp_dim, fused_gate_up}
+
+    # Flatten all shape dimensions
+    all_dims: set[int] = set()
+    for shape in shapes:
+        all_dims.update(shape)
+
+    # Check for vocab dimension first (embed / lm_head)
+    if all_dims & vocab_dims:
+        return EMBED_LM_HEAD
+
+    # MLP dimensions are more distinctive than attention; check first
+    if all_dims & mlp_dims:
+        return GEMM_MLP
+
+    # Attention dimensions
+    if all_dims & attn_dims:
+        return GEMM_ATTENTION
+
+    return OTHER
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def classify_kernel(
+    name: str,
+    shapes: list[list[int]] | None = None,
+    tp: int = 8,
+) -> str:
+    """Classify a single kernel into a component bucket.
+
+    Args:
+        name: CUDA kernel name from the profiler trace.
+        shapes: Optional list of tensor shapes (from ``record_shapes=True``).
+        tp: Tensor-parallelism degree (default 8).
+
+    Returns:
+        Component bucket string (one of :data:`ALL_COMPONENTS`).
+    """
+    # Non-GEMM patterns first (cheaper regex, unambiguous)
+    for pattern, component in _COMPILED_PATTERNS:
+        if pattern.search(name):
+            return component
+
+    # GEMM kernels need shape-based disambiguation
+    if _is_gemm_kernel(name):
+        return _classify_gemm_by_shape(shapes, tp)
+
+    return OTHER
+
+
+@dataclass
+class ComponentBudget:
+    """Aggregated latency budget for one component."""
+
+    component: str
+    total_us: float = 0.0
+    kernel_count: int = 0
+    kernel_names: set[str] = field(default_factory=set)
+
+    @property
+    def total_ms(self) -> float:
+        return self.total_us / 1000.0
+
+    def pct_of(self, total_us: float) -> float:
+        """Return this component's share of *total_us* as a percentage."""
+        if total_us <= 0:
+            return 0.0
+        return 100.0 * self.total_us / total_us
+
+
+def classify_trace_events(
+    events: list[dict[str, Any]],
+    tp: int = 8,
+) -> dict[str, ComponentBudget]:
+    """Classify a list of Chrome-trace events into component budgets.
+
+    Each event dict must have ``name`` (str) and ``dur`` (µs, float).
+    Optional ``args.shapes`` enables GEMM disambiguation.
+
+    Returns:
+        Dict mapping component name → :class:`ComponentBudget`.
+    """
+    budgets = {comp: ComponentBudget(component=comp) for comp in ALL_COMPONENTS}
+
+    for ev in events:
+        name = ev.get("name", "")
+        dur_us = ev.get("dur", 0.0)
+        shapes = None
+        if "args" in ev and "shapes" in ev["args"]:
+            shapes = ev["args"]["shapes"]
+
+        comp = classify_kernel(name, shapes=shapes, tp=tp)
+        budgets[comp].total_us += dur_us
+        budgets[comp].kernel_count += 1
+        budgets[comp].kernel_names.add(name)
+
+    return budgets
+
+
+def format_budget_table(
+    budgets: dict[str, ComponentBudget],
+    phase: str,
+) -> str:
+    """Format component budgets as a Markdown table sorted by descending time."""
+    total_us = sum(b.total_us for b in budgets.values())
+    lines = [
+        f"## {phase}",
+        "",
+        "| Component | Kernel time (ms) | % of total | ARK target | Q-item |",
+        "|-----------|-----------------|------------|------------|--------|",
+    ]
+    sorted_budgets = sorted(
+        budgets.values(), key=lambda b: b.total_us, reverse=True
+    )
+    for b in sorted_budgets:
+        lines.append(
+            f"| {b.component} | {b.total_ms:.2f} "
+            f"| {b.pct_of(total_us):.1f}% | TBD | TBD |"
+        )
+    lines.append(
+        f"| **Total** | **{total_us / 1000:.2f}** | **100%** | | |"
+    )
+    lines.append("")
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Load a Chrome-trace JSON, classify kernel events, and print a budget table."""
+    import argparse
+    import json
+
+    parser = argparse.ArgumentParser(
+        description="Classify GPU kernels from a Chrome-trace JSON into component buckets.",
+    )
+    parser.add_argument("trace", help="Path to Chrome-trace JSON file")
+    parser.add_argument("--tp", type=int, default=8, help="Tensor-parallelism degree (default 8)")
+    args = parser.parse_args(argv)
+
+    with open(args.trace) as f:
+        trace = json.load(f)
+
+    events = [
+        e
+        for e in trace.get("traceEvents", [])
+        if e.get("ph") == "X" and e.get("cat", "") == "kernel"
+    ]
+
+    budgets = classify_trace_events(events, tp=args.tp)
+    print(format_budget_table(budgets, "GPU Kernel Budget"))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/qwen3/bench/profile_sglang.py b/examples/qwen3/bench/profile_sglang.py
new file mode 100644
index 00000000..de21d947
--- /dev/null
+++ b/examples/qwen3/bench/profile_sglang.py
@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+"""Profile SGLang Qwen3-8B with torch.profiler for prefill and decode phases.
+
+Uses SGLang's ``/start_profile`` / ``/stop_profile`` server endpoints to
+trigger server-side ``torch.profiler``, then sends HTTP requests to exercise
+prefill (prompt=2048, max_new_tokens=1) and decode (prompt=2048,
+max_new_tokens=128) phases.  Chrome-trace JSON files are saved on the server's
+filesystem (default ``/tmp/sglang_profile/``).
+
+Profiler configuration:
+    schedule:       wait=2, warmup=1, active=3
+    record_shapes:  True
+    with_stack:     True
+    output:         tensorboard_trace_handler
+
+Usage (against a running SGLang server — see reproduce_profile.md)::
+
+    python profile_sglang.py --port 30000 --output-dir /tmp/sglang_profile
+    python profile_sglang.py --phase prefill --trials 5
+    python profile_sglang.py --phase decode  --trials 5
+
+Prerequisites:
+    - SGLang server running (see reproduce_profile.md)
+    - ``requests`` Python package
+    - Pinned SGLang image: lmsysorg/sglang:v0.4.6.post1-cu124
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from typing import Any
+
+try:
+    import torch.profiler as _torch_profiler  # noqa: F401 — used by helpers
+except ImportError:
+    _torch_profiler = None  # type: ignore[assignment]
+
+try:
+    import requests as _requests
+except ImportError:
+    _requests = None  # type: ignore[assignment]
+
+
+# ---------------------------------------------------------------------------
+# Profiler configuration helpers (testable without GPU)
+# ---------------------------------------------------------------------------
+
+# Default schedule parameters
+SCHEDULE_WAIT = 2
+SCHEDULE_WARMUP = 1
+SCHEDULE_ACTIVE = 3
+DEFAULT_OUTPUT_DIR = "/tmp/sglang_profile"
+DEFAULT_PORT = 30000
+DEFAULT_PROMPT_LEN = 2048
+DEFAULT_DECODE_TOKENS = 128
+DEFAULT_TRIALS = 5
+
+# Pinned SGLang image (matches Q1 pinned image)
+SGLANG_IMAGE_TAG = "lmsysorg/sglang:v0.4.6.post1-cu124"
+
+
+# Canonical schedule constructor — used server-side or by external tooling.
+# Tested when torch is available to keep the specification in sync with the constants.
+def build_profiler_schedule() -> Any:
+    """Build a ``torch.profiler.schedule`` with the canonical parameters.
+
+    Returns:
+        A callable schedule suitable for ``torch.profiler.profile(schedule=...)``.
+
+    Raises:
+        RuntimeError: If ``torch`` is not installed.
+    """
+    if _torch_profiler is None:
+        raise RuntimeError("torch is required to build the profiler schedule")
+    return _torch_profiler.schedule(
+        wait=SCHEDULE_WAIT,
+        warmup=SCHEDULE_WARMUP,
+        active=SCHEDULE_ACTIVE,
+    )
+
+
+def build_profiler_config(output_dir: str = DEFAULT_OUTPUT_DIR) -> dict[str, Any]:
+    """Return the ``torch.profiler.profile`` keyword arguments as a dict.
+
+    This dict is the *specification*; it can be passed directly to
+    ``torch.profiler.profile(**config)`` on the server side, or serialised
+    for documentation.
+
+    Returns:
+        Dict with keys matching ``torch.profiler.profile`` kwargs.
+    """
+    config: dict[str, Any] = {
+        "activities": ["cpu", "cuda"],
+        "schedule": {
+            "wait": SCHEDULE_WAIT,
+            "warmup": SCHEDULE_WARMUP,
+            "active": SCHEDULE_ACTIVE,
+        },
+        "record_shapes": True,
+        "with_stack": True,
+        "output_dir": output_dir,
+    }
+    return config
+
+
+# ---------------------------------------------------------------------------
+# Prompt construction (reuses Q1 pattern)
+# ---------------------------------------------------------------------------
+
+
+def build_prompt(num_tokens: int) -> str:
+    """Build a prompt that is approximately *num_tokens* tokens long.
+
+    Uses a repeating word pattern.  Actual count depends on the tokenizer;
+    ``"hello "`` ≈ 1 token for most BPE tokenizers.  Over-generates by 1.2×
+    to compensate for BPE variance.
+    """
+    return "hello " * int(num_tokens * 1.2)
+
+
+# ---------------------------------------------------------------------------
+# SGLang HTTP helpers
+# ---------------------------------------------------------------------------
+
+
+def _require_requests() -> None:
+    if _requests is None:
+        raise ImportError("'requests' package required. Install: pip install requests")
+
+
+def start_server_profile(base_url: str) -> None:
+    """POST /start_profile to begin server-side torch.profiler."""
+    _require_requests()
+    resp = _requests.post(f"{base_url}/start_profile")
+    resp.raise_for_status()
+
+
+def stop_server_profile(base_url: str) -> None:
+    """POST /stop_profile to stop server-side torch.profiler and flush trace."""
+    _require_requests()
+    resp = _requests.post(f"{base_url}/stop_profile")
+    resp.raise_for_status()
+
+
+def send_generate(
+    base_url: str,
+    prompt: str,
+    max_new_tokens: int,
+) -> dict[str, Any]:
+    """Send a /generate request and return the JSON response."""
+    _require_requests()
+    payload = {
+        "text": prompt,
+        "sampling_params": {
+            "max_new_tokens": max_new_tokens,
+            "temperature": 0.0,
+            "ignore_eos": True,
+        },
+    }
+    start = time.perf_counter()
+    resp = _requests.post(f"{base_url}/generate", json=payload, timeout=300)
+    elapsed_ms = (time.perf_counter() - start) * 1000
+    resp.raise_for_status()
+    result = resp.json()
+    result["elapsed_ms"] = elapsed_ms
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Profiling workflow
+# ---------------------------------------------------------------------------
+
+
+def run_phase(
+    base_url: str,
+    phase: str,
+    prompt_len: int,
+    max_new_tokens: int,
+    trials: int,
+) -> list[dict[str, Any]]:
+    """Run *trials* requests for a single phase (prefill or decode).
+
+    Starts/stops server-side profiling around the request batch.
+
+    Returns:
+        List of per-trial result dicts.
+    """
+    prompt = build_prompt(prompt_len)
+    results: list[dict[str, Any]] = []
+
+    print(f"[{phase}] Starting server-side profiler ...")
+    start_server_profile(base_url)
+
+    for i in range(trials):
+        print(f"[{phase}] Trial {i + 1}/{trials} "
+              f"(prompt≈{prompt_len}, max_new_tokens={max_new_tokens})")
+        result = send_generate(base_url, prompt, max_new_tokens)
+        results.append(result)
+        print(f"  elapsed: {result['elapsed_ms']:.1f} ms")
+
+    print(f"[{phase}] Stopping server-side profiler ...")
+    stop_server_profile(base_url)
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Argument parsing
+# ---------------------------------------------------------------------------
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Profile SGLang Qwen3-8B prefill/decode with torch.profiler",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=DEFAULT_PORT,
+        help=f"SGLang server port (default {DEFAULT_PORT})",
+    )
+    parser.add_argument(
+        "--host",
+        default="localhost",
+        help="SGLang server host (default localhost)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default=DEFAULT_OUTPUT_DIR,
+        help=f"Server-side trace output directory (default {DEFAULT_OUTPUT_DIR})",
+    )
+    parser.add_argument(
+        "--phase",
+        choices=["prefill", "decode", "both"],
+        default="both",
+        help="Which phase to profile (default both)",
+    )
+    parser.add_argument(
+        "--prompt-len",
+        type=int,
+        default=DEFAULT_PROMPT_LEN,
+        help=f"Prompt length in tokens (default {DEFAULT_PROMPT_LEN})",
+    )
+    parser.add_argument(
+        "--decode-tokens",
+        type=int,
+        default=DEFAULT_DECODE_TOKENS,
+        help=f"Number of tokens to generate in decode phase (default {DEFAULT_DECODE_TOKENS})",
+    )
+    parser.add_argument(
+        "--trials",
+        type=int,
+        default=DEFAULT_TRIALS,
+        help=f"Number of trials per phase (default {DEFAULT_TRIALS})",
+    )
+    return parser.parse_args(argv)
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    args = parse_args()
+    base_url = f"http://{args.host}:{args.port}"
+
+    print(f"SGLang server: {base_url}")
+    print(f"Profiler config: {json.dumps(build_profiler_config(args.output_dir), indent=2)}")
+    print()
+
+    if args.phase in ("prefill", "both"):
+        run_phase(
+            base_url,
+            phase="prefill",
+            prompt_len=args.prompt_len,
+            max_new_tokens=1,
+            trials=args.trials,
+        )
+        print()
+
+    if args.phase in ("decode", "both"):
+        run_phase(
+            base_url,
+            phase="decode",
+            prompt_len=args.prompt_len,
+            max_new_tokens=args.decode_tokens,
+            trials=args.trials,
+        )
+        print()
+
+    print("Done. Traces saved on server at:", args.output_dir)
+    print("Copy traces locally, then run: bash analyze_profile.sh <trace.json>")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/qwen3/bench/reproduce_profile.md b/examples/qwen3/bench/reproduce_profile.md
new file mode 100644
index 00000000..39b788cb
--- /dev/null
+++ b/examples/qwen3/bench/reproduce_profile.md
@@ -0,0 +1,75 @@
+# Reproducing the Qwen3-8B Profile
+
+## Prerequisites
+
+- 8×A100-80GB node (e.g., `mscclpp-a100-dev`)
+- Docker with NVIDIA runtime
+- HuggingFace cache with `Qwen/Qwen3-8B` weights (or network access to pull)
+
+## Steps
+
+### 1. Launch SGLang server
+
+Start the SGLang server manually with the pinned image
+(`lmsysorg/sglang:v0.4.6.post1-cu124`), TP=8, and the `Qwen/Qwen3-8B` model.
+The `launch_sglang.sh` script is not included in this changeset.
+
+### 2. Run the profiling harness
+
+```bash
+# Profile both prefill and decode (5 trials each)
+python profile_sglang.py --port 30000 --phase both --trials 5
+
+# Or profile each phase separately
+python profile_sglang.py --port 30000 --phase prefill --trials 5
+python profile_sglang.py --port 30000 --phase decode  --trials 5
+```
+
+Traces are saved server-side at `/tmp/sglang_profile/` (configurable via
+`--output-dir`).
+
+### 3. Copy traces from the container
+
+```bash
+CONTAINER=sglang-qwen3-bench
+docker cp "${CONTAINER}:/tmp/sglang_profile/" ./traces/
+```
+
+### 4. Analyze traces
+
+```bash
+# Full analysis (trace_analyzer.py + per-component classifier)
+bash analyze_profile.sh ./traces/<trace_file>.json --tp 8
+
+# Or run the classifier directly
+python3 classify_kernels.py traces/<trace_file>.json --tp 8
+```
+
+### 5. Fill PROFILE.md
+
+Copy the per-component budget numbers into `../PROFILE.md`.  Re-rank Q4–Q8
+by descending total kernel time.
+
+## Expected output structure
+
+```
+traces/
+├── analysis/
+│   ├── kernels.txt           # Top-N kernel breakdown
+│   ├── comm.txt              # NCCL communication analysis
+│   ├── gaps.txt              # GPU idle gap analysis
+│   ├── full_analysis.md      # Complete trace_analyzer output
+│   └── component_budget.md   # Per-component latency budget
+└── *.pt.trace.json           # Raw Chrome-trace files
+```
+
+## Notes
+
+- The profiler adds overhead; do not compare profiled latencies with the
+  Q1 baseline numbers.  The profile is for *relative component breakdown*,
+  not absolute latency.
+- `record_shapes=True` increases trace size.  Profile rank 0 only for
+  manageable file sizes.
+- SGLang's `/start_profile` and `/stop_profile` endpoints control the
+  server-side `torch.profiler`.  The profiler schedule (wait=2, warmup=1,
+  active=3) is configured server-side.
diff --git a/examples/qwen3/bench/test_profile_sglang.py b/examples/qwen3/bench/test_profile_sglang.py
new file mode 100644
index 00000000..191f18f4
--- /dev/null
+++ b/examples/qwen3/bench/test_profile_sglang.py
@@ -0,0 +1,412 @@
+#!/usr/bin/env python3
+"""Unit tests for profile_sglang.py and classify_kernels.py.
+
+CPU-only — no GPU, no SGLang server, no network required.
+Tests: kernel classifier correctness, profiler schedule construction,
+argument parsing, and module imports.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+# Add bench directory to path so we can import the modules under test.
+_BENCH_DIR = str(Path(__file__).resolve().parent)
+if _BENCH_DIR not in sys.path:
+    sys.path.insert(0, _BENCH_DIR)
+
+import classify_kernels  # noqa: E402
+import profile_sglang  # noqa: E402
+
+
+# =========================================================================
+# classify_kernels tests
+# =========================================================================
+
+
+class TestClassifyKernel:
+    """Test classify_kernel() with known kernel name patterns."""
+
+    # --- Attention kernels ---
+
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "flash_fwd_splitkv_kernel",
+            "fmha_v2_flash_attention_fp16_128_64_S_128_kernel",
+            "flashinfer_batch_prefill_ragged",
+            "cudnn_sdpa_fwd_fp16",
+            "sdpa_backward_kernel",
+        ],
+    )
+    def test_attention_kernels(self, name: str) -> None:
+        assert classify_kernels.classify_kernel(name) == classify_kernels.ATTENTION
+
+    # --- NCCL kernels ---
+
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "ncclDevKernel_AllReduce_Sum_f16",
+            "nccl_allreduce_ring_ll_bf16",
+            "ncclKernel_ReduceScatter_fp32",
+        ],
+    )
+    def test_nccl_kernels(self, name: str) -> None:
+        assert classify_kernels.classify_kernel(name) == classify_kernels.NCCL
+
+    # --- Norms / RoPE / activations ---
+
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "rms_norm_kernel_fp16",
+            "rmsnorm_fwd",
+            "layernorm_forward_cuda",
+            "layer_norm_kernel",
+            "fused_rms_norm_kernel",
+            "rotary_embedding_kernel",
+            "rope_forward_kernel",
+            "silu_and_mul_kernel",
+            "gelu_forward",
+            "elementwise_kernel_add",
+        ],
+    )
+    def test_norms_rope_kernels(self, name: str) -> None:
+        assert classify_kernels.classify_kernel(name) == classify_kernels.NORMS_ROPE
+
+    # --- GEMM kernels with shape disambiguation ---
+
+    def test_gemm_mlp_gate_up(self) -> None:
+        """GEMM with MLP gate/up dimension (1792 for TP=8)."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_s16816gemm_f16_256x128_32x3_nn",
+            shapes=[[2048, 4096], [4096, 1792]],
+            tp=8,
+        )
+        assert result == classify_kernels.GEMM_MLP
+
+    def test_gemm_mlp_fused_gate_up(self) -> None:
+        """GEMM with fused gate+up dimension (3584 for TP=8)."""
+        result = classify_kernels.classify_kernel(
+            "cublas_GemmEx_f16",
+            shapes=[[2048, 4096], [4096, 3584]],
+            tp=8,
+        )
+        assert result == classify_kernels.GEMM_MLP
+
+    def test_gemm_mlp_down(self) -> None:
+        """GEMM with MLP down dimension (1792 input for TP=8)."""
+        result = classify_kernels.classify_kernel(
+            "sm80_xmma_gemm_f16f16_f32_f32_tn_n_tilesize128x128x32",
+            shapes=[[2048, 1792], [1792, 4096]],
+            tp=8,
+        )
+        assert result == classify_kernels.GEMM_MLP
+
+    def test_gemm_attention_qkv(self) -> None:
+        """GEMM with fused QKV dimension (768 for TP=8)."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[2048, 4096], [4096, 768]],
+            tp=8,
+        )
+        assert result == classify_kernels.GEMM_ATTENTION
+
+    def test_gemm_attention_q_proj(self) -> None:
+        """GEMM with Q projection dimension (512 for TP=8)."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[2048, 4096], [4096, 512]],
+            tp=8,
+        )
+        assert result == classify_kernels.GEMM_ATTENTION
+
+    def test_gemm_attention_kv_proj(self) -> None:
+        """GEMM with K/V projection dimension (128 for TP=8)."""
+        result = classify_kernels.classify_kernel(
+            "cublas_GemmEx_bf16",
+            shapes=[[2048, 4096], [4096, 128]],
+            tp=8,
+        )
+        assert result == classify_kernels.GEMM_ATTENTION
+
+    def test_gemm_embed_lm_head(self) -> None:
+        """GEMM with vocab dimension (151936)."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[1, 4096], [4096, 151936]],
+            tp=8,
+        )
+        assert result == classify_kernels.EMBED_LM_HEAD
+
+    def test_gemm_embed_lm_head_sharded(self) -> None:
+        """GEMM with sharded vocab dimension (151936 / 8 = 18992)."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[1, 4096], [4096, 18992]],
+            tp=8,
+        )
+        assert result == classify_kernels.EMBED_LM_HEAD
+
+    def test_gemm_no_shapes_falls_to_other(self) -> None:
+        """GEMM without shapes cannot be disambiguated → OTHER."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=None,
+            tp=8,
+        )
+        assert result == classify_kernels.OTHER
+
+    def test_gemm_unknown_shapes_falls_to_other(self) -> None:
+        """GEMM with unrecognised shapes → OTHER."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[32, 32], [32, 32]],
+            tp=8,
+        )
+        assert result == classify_kernels.OTHER
+
+    # --- Unknown kernels ---
+
+    def test_unknown_kernel(self) -> None:
+        assert classify_kernels.classify_kernel("some_random_kernel") == classify_kernels.OTHER
+
+    def test_empty_name(self) -> None:
+        assert classify_kernels.classify_kernel("") == classify_kernels.OTHER
+
+    # --- TP=1 shapes ---
+
+    def test_gemm_mlp_tp1(self) -> None:
+        """MLP gate dimension at TP=1 is 14336."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[2048, 4096], [4096, 14336]],
+            tp=1,
+        )
+        assert result == classify_kernels.GEMM_MLP
+
+
+class TestClassifyTraceEvents:
+    """Test classify_trace_events() aggregation."""
+
+    def test_aggregation(self) -> None:
+        events = [
+            {"name": "flash_fwd_kernel", "dur": 100.0},
+            {"name": "flash_fwd_kernel", "dur": 200.0},
+            {"name": "ncclDevKernel_AllReduce", "dur": 50.0},
+            {"name": "rms_norm_kernel", "dur": 30.0},
+            {"name": "unknown_kernel_xyz", "dur": 10.0},
+        ]
+        budgets = classify_kernels.classify_trace_events(events, tp=8)
+
+        assert budgets[classify_kernels.ATTENTION].total_us == pytest.approx(300.0)
+        assert budgets[classify_kernels.ATTENTION].kernel_count == 2
+        assert budgets[classify_kernels.NCCL].total_us == pytest.approx(50.0)
+        assert budgets[classify_kernels.NORMS_ROPE].total_us == pytest.approx(30.0)
+        assert budgets[classify_kernels.OTHER].total_us == pytest.approx(10.0)
+
+    def test_empty_events(self) -> None:
+        budgets = classify_kernels.classify_trace_events([])
+        for comp in classify_kernels.ALL_COMPONENTS:
+            assert budgets[comp].total_us == 0.0
+            assert budgets[comp].kernel_count == 0
+
+    def test_shapes_in_events(self) -> None:
+        events = [
+            {
+                "name": "cutlass_80_gemm",
+                "dur": 500.0,
+                "args": {"shapes": [[2048, 4096], [4096, 1792]]},
+            },
+        ]
+        budgets = classify_kernels.classify_trace_events(events, tp=8)
+        assert budgets[classify_kernels.GEMM_MLP].total_us == pytest.approx(500.0)
+
+
+class TestComponentBudget:
+    """Test ComponentBudget dataclass helpers."""
+
+    def test_total_ms(self) -> None:
+        b = classify_kernels.ComponentBudget(component="test", total_us=1500.0)
+        assert b.total_ms == pytest.approx(1.5)
+
+    def test_pct_of(self) -> None:
+        b = classify_kernels.ComponentBudget(component="test", total_us=250.0)
+        assert b.pct_of(1000.0) == pytest.approx(25.0)
+
+    def test_pct_of_zero(self) -> None:
+        b = classify_kernels.ComponentBudget(component="test", total_us=100.0)
+        assert b.pct_of(0.0) == pytest.approx(0.0)
+
+
+class TestFormatBudgetTable:
+    """Test format_budget_table() output structure."""
+
+    def test_markdown_table(self) -> None:
+        budgets = {
+            comp: classify_kernels.ComponentBudget(component=comp)
+            for comp in classify_kernels.ALL_COMPONENTS
+        }
+        budgets[classify_kernels.ATTENTION].total_us = 1000.0
+        budgets[classify_kernels.GEMM_MLP].total_us = 2000.0
+
+        table = classify_kernels.format_budget_table(budgets, "Test Phase")
+        assert "## Test Phase" in table
+        assert "| Component |" in table
+        assert "gemm_mlp" in table
+        assert "attention" in table
+        assert "**Total**" in table
+
+
+# =========================================================================
+# profile_sglang tests
+# =========================================================================
+
+
+class TestParseArgs:
+    """Test profile_sglang.parse_args()."""
+
+    def test_defaults(self) -> None:
+        args = profile_sglang.parse_args([])
+        assert args.port == 30000
+        assert args.host == "localhost"
+        assert args.phase == "both"
+        assert args.prompt_len == 2048
+        assert args.decode_tokens == 128
+        assert args.trials == 5
+        assert args.output_dir == "/tmp/sglang_profile"
+
+    def test_custom_args(self) -> None:
+        args = profile_sglang.parse_args([
+            "--port", "8080",
+            "--host", "gpu-node",
+            "--phase", "prefill",
+            "--prompt-len", "1024",
+            "--decode-tokens", "64",
+            "--trials", "3",
+            "--output-dir", "/data/traces",
+        ])
+        assert args.port == 8080
+        assert args.host == "gpu-node"
+        assert args.phase == "prefill"
+        assert args.prompt_len == 1024
+        assert args.decode_tokens == 64
+        assert args.trials == 3
+        assert args.output_dir == "/data/traces"
+
+
+class TestBuildProfilerConfig:
+    """Test profile_sglang.build_profiler_config()."""
+
+    def test_config_structure(self) -> None:
+        config = profile_sglang.build_profiler_config()
+        assert config["record_shapes"] is True
+        assert config["with_stack"] is True
+        assert config["schedule"]["wait"] == 2
+        assert config["schedule"]["warmup"] == 1
+        assert config["schedule"]["active"] == 3
+        assert "activities" in config
+
+    def test_custom_output_dir(self) -> None:
+        config = profile_sglang.build_profiler_config("/custom/dir")
+        assert config["output_dir"] == "/custom/dir"
+
+
+class TestBuildProfilerSchedule:
+    """Test profile_sglang.build_profiler_schedule()."""
+
+    def test_returns_callable(self) -> None:
+        pytest.importorskip("torch")
+        schedule = profile_sglang.build_profiler_schedule()
+        assert callable(schedule)
+
+    def test_schedule_phases(self) -> None:
+        """Verify the schedule produces the expected action sequence."""
+        torch_profiler = pytest.importorskip("torch.profiler")
+
+        schedule = profile_sglang.build_profiler_schedule()
+        # wait=2 → NONE, NONE; warmup=1 → WARMUP; active=3 → RECORD x3
+        expected = [
+            torch_profiler.ProfilerAction.NONE,        # step 0 (wait)
+            torch_profiler.ProfilerAction.NONE,        # step 1 (wait)
+            torch_profiler.ProfilerAction.WARMUP,      # step 2 (warmup)
+            torch_profiler.ProfilerAction.RECORD,      # step 3 (active)
+            torch_profiler.ProfilerAction.RECORD,      # step 4 (active)
+            torch_profiler.ProfilerAction.RECORD_AND_SAVE,  # step 5 (last active)
+        ]
+        for step, exp_action in enumerate(expected):
+            assert schedule(step) == exp_action
+
+
+class TestBuildPrompt:
+    """Test profile_sglang.build_prompt()."""
+
+    def test_prompt_length(self) -> None:
+        prompt = profile_sglang.build_prompt(100)
+        # Overgenerate by 1.2×, each "hello " is one word
+        words = prompt.strip().split()
+        assert len(words) == 120  # 100 * 1.2
+
+    def test_prompt_not_empty(self) -> None:
+        prompt = profile_sglang.build_prompt(1)
+        assert len(prompt) > 0
+
+
+class TestRequireRequests:
+    """Test _require_requests() raises ImportError when requests is missing."""
+
+    def test_raises_when_requests_missing(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(profile_sglang, "_requests", None)
+        with pytest.raises(ImportError, match="requests"):
+            profile_sglang._require_requests()
+
+
+class TestClassifyKernelsMain:
+    """Test classify_kernels.main() CLI entry point."""
+
+    def test_main_prints_budget_table(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+        """main() loads a trace JSON, classifies events, and prints a budget table."""
+        trace = {
+            "traceEvents": [
+                {"ph": "X", "cat": "kernel", "name": "flash_fwd_kernel", "dur": 100.0},
+            ]
+        }
+        trace_file = tmp_path / "trace.json"
+        import json
+        trace_file.write_text(json.dumps(trace))
+
+        classify_kernels.main([str(trace_file), "--tp", "8"])
+
+        captured = capsys.readouterr().out
+        assert "| Component |" in captured
+        assert "attention" in captured
+
+
+class TestBuildProfilerScheduleNoTorch:
+    """Test build_profiler_schedule() when torch is absent."""
+
+    def test_raises_runtime_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(profile_sglang, "_torch_profiler", None)
+        with pytest.raises(RuntimeError, match="torch is required"):
+            profile_sglang.build_profiler_schedule()
+
+
+class TestConstants:
+    """Verify module-level constants are consistent."""
+
+    def test_sglang_image_tag(self) -> None:
+        assert profile_sglang.SGLANG_IMAGE_TAG == "lmsysorg/sglang:v0.4.6.post1-cu124"
+
+    def test_all_components_complete(self) -> None:
+        assert len(classify_kernels.ALL_COMPONENTS) == 7
+        assert classify_kernels.ATTENTION in classify_kernels.ALL_COMPONENTS
+        assert classify_kernels.GEMM_ATTENTION in classify_kernels.ALL_COMPONENTS
+        assert classify_kernels.GEMM_MLP in classify_kernels.ALL_COMPONENTS
+        assert classify_kernels.NCCL in classify_kernels.ALL_COMPONENTS
+        assert classify_kernels.NORMS_ROPE in classify_kernels.ALL_COMPONENTS
+        assert classify_kernels.EMBED_LM_HEAD in classify_kernels.ALL_COMPONENTS
+        assert classify_kernels.OTHER in classify_kernels.ALL_COMPONENTS

From 0c1ab08eca7871fb8e609d197508bbefa562b5be Mon Sep 17 00:00:00 2001
From: ark-dev <ark-dev@local>
Date: Fri, 12 Jun 2026 05:48:40 +0000
Subject: [PATCH 2/4] =?UTF-8?q?ark-dev:=20Implement=20Q2:=20profile=20SGLa?=
 =?UTF-8?q?ng=20Qwen3-8B=20forward=20with=20torch.profiler=20for=20both=20?=
 =?UTF-8?q?prefill=20and=20decode=20phases;=20write=20PROFILE.md=20with=20?=
 =?UTF-8?q?a=20ranked=20per-component=20latency=20budget=20that=20re-ranks?=
 =?UTF-8?q?=20Q4=E2=80=93Q8.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 examples/qwen3/bench/classify_kernels.py    | 11 ++-
 examples/qwen3/bench/profile_sglang.py      | 18 ++--
 examples/qwen3/bench/test_profile_sglang.py | 93 +++++++++++++++------
 3 files changed, 86 insertions(+), 36 deletions(-)

diff --git a/examples/qwen3/bench/classify_kernels.py b/examples/qwen3/bench/classify_kernels.py
index 3b882b05..2cad5141 100644
--- a/examples/qwen3/bench/classify_kernels.py
+++ b/examples/qwen3/bench/classify_kernels.py
@@ -276,9 +276,7 @@ def format_budget_table(
             f"| {b.component} | {b.total_ms:.2f} "
             f"| {b.pct_of(total_us):.1f}% | TBD | TBD |"
         )
-    lines.append(
-        f"| **Total** | **{total_us / 1000:.2f}** | **100%** | | |"
-    )
+    lines.append(f"| **Total** | **{total_us / 1000:.2f}** | **100%** | | |")
     lines.append("")
     return "\n".join(lines)
 
@@ -297,7 +295,12 @@ def main(argv: list[str] | None = None) -> None:
         description="Classify GPU kernels from a Chrome-trace JSON into component buckets.",
     )
     parser.add_argument("trace", help="Path to Chrome-trace JSON file")
-    parser.add_argument("--tp", type=int, default=8, help="Tensor-parallelism degree (default 8)")
+    parser.add_argument(
+        "--tp",
+        type=int,
+        default=8,
+        help="Tensor-parallelism degree (default 8)",
+    )
     args = parser.parse_args(argv)
 
     with open(args.trace) as f:
diff --git a/examples/qwen3/bench/profile_sglang.py b/examples/qwen3/bench/profile_sglang.py
index de21d947..27b4ed56 100644
--- a/examples/qwen3/bench/profile_sglang.py
+++ b/examples/qwen3/bench/profile_sglang.py
@@ -81,7 +81,9 @@ def build_profiler_schedule() -> Any:
     )
 
 
-def build_profiler_config(output_dir: str = DEFAULT_OUTPUT_DIR) -> dict[str, Any]:
+def build_profiler_config(
+    output_dir: str = DEFAULT_OUTPUT_DIR,
+) -> dict[str, Any]:
     """Return the ``torch.profiler.profile`` keyword arguments as a dict.
 
     This dict is the *specification*; it can be passed directly to
@@ -127,7 +129,9 @@ def build_prompt(num_tokens: int) -> str:
 
 def _require_requests() -> None:
     if _requests is None:
-        raise ImportError("'requests' package required. Install: pip install requests")
+        raise ImportError(
+            "'requests' package required. Install: pip install requests"
+        )
 
 
 def start_server_profile(base_url: str) -> None:
@@ -194,8 +198,10 @@ def run_phase(
     start_server_profile(base_url)
 
     for i in range(trials):
-        print(f"[{phase}] Trial {i + 1}/{trials} "
-              f"(prompt≈{prompt_len}, max_new_tokens={max_new_tokens})")
+        print(
+            f"[{phase}] Trial {i + 1}/{trials} "
+            f"(prompt≈{prompt_len}, max_new_tokens={max_new_tokens})"
+        )
         result = send_generate(base_url, prompt, max_new_tokens)
         results.append(result)
         print(f"  elapsed: {result['elapsed_ms']:.1f} ms")
@@ -268,7 +274,9 @@ def main() -> None:
     base_url = f"http://{args.host}:{args.port}"
 
     print(f"SGLang server: {base_url}")
-    print(f"Profiler config: {json.dumps(build_profiler_config(args.output_dir), indent=2)}")
+    print(
+        f"Profiler config: {json.dumps(build_profiler_config(args.output_dir), indent=2)}"
+    )
     print()
 
     if args.phase in ("prefill", "both"):
diff --git a/examples/qwen3/bench/test_profile_sglang.py b/examples/qwen3/bench/test_profile_sglang.py
index 191f18f4..efc7d694 100644
--- a/examples/qwen3/bench/test_profile_sglang.py
+++ b/examples/qwen3/bench/test_profile_sglang.py
@@ -21,7 +21,6 @@
 import classify_kernels  # noqa: E402
 import profile_sglang  # noqa: E402
 
-
 # =========================================================================
 # classify_kernels tests
 # =========================================================================
@@ -43,7 +42,9 @@ class TestClassifyKernel:
         ],
     )
     def test_attention_kernels(self, name: str) -> None:
-        assert classify_kernels.classify_kernel(name) == classify_kernels.ATTENTION
+        assert (
+            classify_kernels.classify_kernel(name) == classify_kernels.ATTENTION
+        )
 
     # --- NCCL kernels ---
 
@@ -76,7 +77,10 @@ def test_nccl_kernels(self, name: str) -> None:
         ],
     )
     def test_norms_rope_kernels(self, name: str) -> None:
-        assert classify_kernels.classify_kernel(name) == classify_kernels.NORMS_ROPE
+        assert (
+            classify_kernels.classify_kernel(name)
+            == classify_kernels.NORMS_ROPE
+        )
 
     # --- GEMM kernels with shape disambiguation ---
 
@@ -173,7 +177,10 @@ def test_gemm_unknown_shapes_falls_to_other(self) -> None:
     # --- Unknown kernels ---
 
     def test_unknown_kernel(self) -> None:
-        assert classify_kernels.classify_kernel("some_random_kernel") == classify_kernels.OTHER
+        assert (
+            classify_kernels.classify_kernel("some_random_kernel")
+            == classify_kernels.OTHER
+        )
 
     def test_empty_name(self) -> None:
         assert classify_kernels.classify_kernel("") == classify_kernels.OTHER
@@ -203,10 +210,14 @@ def test_aggregation(self) -> None:
         ]
         budgets = classify_kernels.classify_trace_events(events, tp=8)
 
-        assert budgets[classify_kernels.ATTENTION].total_us == pytest.approx(300.0)
+        assert budgets[classify_kernels.ATTENTION].total_us == pytest.approx(
+            300.0
+        )
         assert budgets[classify_kernels.ATTENTION].kernel_count == 2
         assert budgets[classify_kernels.NCCL].total_us == pytest.approx(50.0)
-        assert budgets[classify_kernels.NORMS_ROPE].total_us == pytest.approx(30.0)
+        assert budgets[classify_kernels.NORMS_ROPE].total_us == pytest.approx(
+            30.0
+        )
         assert budgets[classify_kernels.OTHER].total_us == pytest.approx(10.0)
 
     def test_empty_events(self) -> None:
@@ -224,7 +235,9 @@ def test_shapes_in_events(self) -> None:
             },
         ]
         budgets = classify_kernels.classify_trace_events(events, tp=8)
-        assert budgets[classify_kernels.GEMM_MLP].total_us == pytest.approx(500.0)
+        assert budgets[classify_kernels.GEMM_MLP].total_us == pytest.approx(
+            500.0
+        )
 
 
 class TestComponentBudget:
@@ -281,15 +294,24 @@ def test_defaults(self) -> None:
         assert args.output_dir == "/tmp/sglang_profile"
 
     def test_custom_args(self) -> None:
-        args = profile_sglang.parse_args([
-            "--port", "8080",
-            "--host", "gpu-node",
-            "--phase", "prefill",
-            "--prompt-len", "1024",
-            "--decode-tokens", "64",
-            "--trials", "3",
-            "--output-dir", "/data/traces",
-        ])
+        args = profile_sglang.parse_args(
+            [
+                "--port",
+                "8080",
+                "--host",
+                "gpu-node",
+                "--phase",
+                "prefill",
+                "--prompt-len",
+                "1024",
+                "--decode-tokens",
+                "64",
+                "--trials",
+                "3",
+                "--output-dir",
+                "/data/traces",
+            ]
+        )
         assert args.port == 8080
         assert args.host == "gpu-node"
         assert args.phase == "prefill"
@@ -331,11 +353,11 @@ def test_schedule_phases(self) -> None:
         schedule = profile_sglang.build_profiler_schedule()
         # wait=2 → NONE, NONE; warmup=1 → WARMUP; active=3 → RECORD x3
         expected = [
-            torch_profiler.ProfilerAction.NONE,        # step 0 (wait)
-            torch_profiler.ProfilerAction.NONE,        # step 1 (wait)
-            torch_profiler.ProfilerAction.WARMUP,      # step 2 (warmup)
-            torch_profiler.ProfilerAction.RECORD,      # step 3 (active)
-            torch_profiler.ProfilerAction.RECORD,      # step 4 (active)
+            torch_profiler.ProfilerAction.NONE,  # step 0 (wait)
+            torch_profiler.ProfilerAction.NONE,  # step 1 (wait)
+            torch_profiler.ProfilerAction.WARMUP,  # step 2 (warmup)
+            torch_profiler.ProfilerAction.RECORD,  # step 3 (active)
+            torch_profiler.ProfilerAction.RECORD,  # step 4 (active)
             torch_profiler.ProfilerAction.RECORD_AND_SAVE,  # step 5 (last active)
         ]
         for step, exp_action in enumerate(expected):
@@ -359,7 +381,9 @@ def test_prompt_not_empty(self) -> None:
 class TestRequireRequests:
     """Test _require_requests() raises ImportError when requests is missing."""
 
-    def test_raises_when_requests_missing(self, monkeypatch: pytest.MonkeyPatch) -> None:
+    def test_raises_when_requests_missing(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
         monkeypatch.setattr(profile_sglang, "_requests", None)
         with pytest.raises(ImportError, match="requests"):
             profile_sglang._require_requests()
@@ -368,15 +392,23 @@ def test_raises_when_requests_missing(self, monkeypatch: pytest.MonkeyPatch) ->
 class TestClassifyKernelsMain:
     """Test classify_kernels.main() CLI entry point."""
 
-    def test_main_prints_budget_table(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+    def test_main_prints_budget_table(
+        self, tmp_path: Path, capsys: pytest.CaptureFixture[str]
+    ) -> None:
         """main() loads a trace JSON, classifies events, and prints a budget table."""
         trace = {
             "traceEvents": [
-                {"ph": "X", "cat": "kernel", "name": "flash_fwd_kernel", "dur": 100.0},
+                {
+                    "ph": "X",
+                    "cat": "kernel",
+                    "name": "flash_fwd_kernel",
+                    "dur": 100.0,
+                },
             ]
         }
         trace_file = tmp_path / "trace.json"
         import json
+
         trace_file.write_text(json.dumps(trace))
 
         classify_kernels.main([str(trace_file), "--tp", "8"])
@@ -389,7 +421,9 @@ def test_main_prints_budget_table(self, tmp_path: Path, capsys: pytest.CaptureFi
 class TestBuildProfilerScheduleNoTorch:
     """Test build_profiler_schedule() when torch is absent."""
 
-    def test_raises_runtime_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
+    def test_raises_runtime_error(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
         monkeypatch.setattr(profile_sglang, "_torch_profiler", None)
         with pytest.raises(RuntimeError, match="torch is required"):
             profile_sglang.build_profiler_schedule()
@@ -399,12 +433,17 @@ class TestConstants:
     """Verify module-level constants are consistent."""
 
     def test_sglang_image_tag(self) -> None:
-        assert profile_sglang.SGLANG_IMAGE_TAG == "lmsysorg/sglang:v0.4.6.post1-cu124"
+        assert (
+            profile_sglang.SGLANG_IMAGE_TAG
+            == "lmsysorg/sglang:v0.4.6.post1-cu124"
+        )
 
     def test_all_components_complete(self) -> None:
         assert len(classify_kernels.ALL_COMPONENTS) == 7
         assert classify_kernels.ATTENTION in classify_kernels.ALL_COMPONENTS
-        assert classify_kernels.GEMM_ATTENTION in classify_kernels.ALL_COMPONENTS
+        assert (
+            classify_kernels.GEMM_ATTENTION in classify_kernels.ALL_COMPONENTS
+        )
         assert classify_kernels.GEMM_MLP in classify_kernels.ALL_COMPONENTS
         assert classify_kernels.NCCL in classify_kernels.ALL_COMPONENTS
         assert classify_kernels.NORMS_ROPE in classify_kernels.ALL_COMPONENTS

From 6388a19a9144dffb8b2e47bf4593f6e5374bf23c Mon Sep 17 00:00:00 2001
From: Changho Hwang <changhohwang@microsoft.com>
Date: Fri, 12 Jun 2026 08:16:43 +0000
Subject: [PATCH 3/4] ci: use --no-install-recommends for lcov to prevent OOM
 (exit 137)

The CUDA UnitTest job OOM-kills during 'apt-get install lcov' because
fontconfig-config and its recommended packages exhaust container memory.
Adding --no-install-recommends avoids pulling those heavy deps.
---
 .github/workflows/ut.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ut.yml b/.github/workflows/ut.yml
index 4e60fde7..1be09faf 100644
--- a/.github/workflows/ut.yml
+++ b/.github/workflows/ut.yml
@@ -51,7 +51,7 @@ jobs:
 
       - name: Build
         run: |
-          apt-get update && apt-get install -y lcov
+          apt-get update && apt-get install -y --no-install-recommends lcov
           mkdir build && cd build
           CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug"
           if [ "${{ matrix.platform }}" = "rocm" ]; then

From 427d7d15b867d0bd86e9a113c2e0d96efd5de3ca Mon Sep 17 00:00:00 2001
From: ark-dev <ark-dev@local>
Date: Fri, 12 Jun 2026 08:40:06 +0000
Subject: [PATCH 4/4] ark-dev: Add SGLang Qwen3-8B torch.profiler harness:
 capture per-component latency budget for prefill and decode, write PROFILE.md
 with ranked breakdown and targets

---
 examples/qwen3/bench/analyze_profile.sh     |  2 +-
 examples/qwen3/bench/classify_kernels.py    |  9 ++++++++-
 examples/qwen3/bench/profile_sglang.py      |  7 ++++---
 examples/qwen3/bench/test_profile_sglang.py | 18 ++++++++++++++++++
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/examples/qwen3/bench/analyze_profile.sh b/examples/qwen3/bench/analyze_profile.sh
index 982b90a7..b9a77212 100644
--- a/examples/qwen3/bench/analyze_profile.sh
+++ b/examples/qwen3/bench/analyze_profile.sh
@@ -30,7 +30,7 @@ shift
 TP=8
 while [ $# -gt 0 ]; do
     case "$1" in
-        --tp) TP="$2"; shift 2 ;;
+        --tp) if [ $# -lt 2 ]; then echo "Error: --tp requires a value" >&2; exit 1; fi; TP="$2"; shift 2 ;;
         *) echo "Unknown argument: $1" >&2; exit 1 ;;
     esac
 done
diff --git a/examples/qwen3/bench/classify_kernels.py b/examples/qwen3/bench/classify_kernels.py
index 2cad5141..6d33856d 100644
--- a/examples/qwen3/bench/classify_kernels.py
+++ b/examples/qwen3/bench/classify_kernels.py
@@ -147,8 +147,15 @@ def _classify_gemm_by_shape(
     # shared with MLP and therefore non-distinctive)
     q_dim = HIDDEN // tp  # 512 for TP=8
     kv_dim = (NUM_KV_HEADS * HEAD_DIM) // tp  # 128 for TP=8
+    # NOTE: kv_dim == 128 at TP=8, which collides with common tile/block
+    # sizes.  The classification order (vocab → MLP → attention) limits
+    # false positives: 128 only triggers attention when no more-distinctive
+    # dimension is present.  Accept this as a known limitation.
     fused_qkv = q_dim + 2 * kv_dim  # 768 for TP=8
     attn_dims = {q_dim, kv_dim, fused_qkv}
+    # Remove HIDDEN — it appears in both attention and MLP projections,
+    # so it is non-distinctive.
+    attn_dims.discard(HIDDEN)
 
     # MLP projection dimensions (per-GPU)
     mlp_dim = INTERMEDIATE // tp  # 1792 for TP=8
@@ -243,7 +250,7 @@ def classify_trace_events(
 
     for ev in events:
         name = ev.get("name", "")
-        dur_us = ev.get("dur", 0.0)
+        dur_us = float(ev.get("dur", 0.0))
         shapes = None
         if "args" in ev and "shapes" in ev["args"]:
             shapes = ev["args"]["shapes"]
diff --git a/examples/qwen3/bench/profile_sglang.py b/examples/qwen3/bench/profile_sglang.py
index 27b4ed56..bd335e0c 100644
--- a/examples/qwen3/bench/profile_sglang.py
+++ b/examples/qwen3/bench/profile_sglang.py
@@ -86,9 +86,10 @@ def build_profiler_config(
 ) -> dict[str, Any]:
     """Return the ``torch.profiler.profile`` keyword arguments as a dict.
 
-    This dict is the *specification*; it can be passed directly to
-    ``torch.profiler.profile(**config)`` on the server side, or serialised
-    for documentation.
+    This dict is a *serialisable specification* for logging and documentation.
+    Values are JSON-friendly (strings / plain dicts) rather than live
+    ``torch.profiler`` objects, so it cannot be passed directly to
+    ``torch.profiler.profile()``.
 
     Returns:
         Dict with keys matching ``torch.profiler.profile`` kwargs.
diff --git a/examples/qwen3/bench/test_profile_sglang.py b/examples/qwen3/bench/test_profile_sglang.py
index efc7d694..45189d81 100644
--- a/examples/qwen3/bench/test_profile_sglang.py
+++ b/examples/qwen3/bench/test_profile_sglang.py
@@ -196,6 +196,24 @@ def test_gemm_mlp_tp1(self) -> None:
         )
         assert result == classify_kernels.GEMM_MLP
 
+    def test_gemm_tp1_hidden_is_not_attention(self) -> None:
+        """At TP=1 q_dim == HIDDEN; HIDDEN is discarded so this is 'other'."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[1, 4096], [4096, 4096]],
+            tp=1,
+        )
+        assert result == classify_kernels.OTHER
+
+    def test_gemm_tp1_fused_qkv_is_attention(self) -> None:
+        """At TP=1 fused_qkv = 4096 + 2*1024 = 6144, a distinctive dim."""
+        result = classify_kernels.classify_kernel(
+            "cutlass_80_tensorop_f16_gemm",
+            shapes=[[1, 4096], [4096, 6144]],
+            tp=1,
+        )
+        assert result == classify_kernels.GEMM_ATTENTION
+
 
 class TestClassifyTraceEvents:
     """Test classify_trace_events() aggregation."""