diff --git a/src/code/issue5/benchmark/benchmark_concurrency.py b/src/code/issue5/benchmark/benchmark_concurrency.py new file mode 100644 index 0000000..49cdbf9 --- /dev/null +++ b/src/code/issue5/benchmark/benchmark_concurrency.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +背景: 端到端命中路径被 write_through kernel 卡死, 无法在真 server 上跑并发 +命中。用实测数值驱动的离散事件重放(replay): 每个逻辑请求的服务时间由 + - 命中: T_lookup + T_restore(hN)/B_retrieve + T_prefill((1-h)N) [远端 retrieve + 后缀现算] + - 未命中: T_prefill(N) [全量重算] +其中 T_prefill(N) 用步骤5 从 L3 prefill_local.csv 拟合的 a·N+b·N² (ms), +T_restore 用 L2 kv_tensor 的 zerocopy get 带宽(TCP~1.1 / RDMA~20.7 GB/s)。 + +排队模型: c 个 prefill worker 的 FIFO 队列(GPU 算力受限, 默认 c=1 即算力串行), +Poisson 到达(率 λ req/s)。扫 λ 直到 P95 TTFT 超过 SLO -> λ_max; QPM=60·λ_max; +成本/query ∝ 固定节点机时 / QPM (归一到 no-cache 基线)。 + +SLO 对 TCP/RDMA/nocache **三者相同**(取 no-cache 单请求全量重算服务时间 × slo-factor), +保证公平对比。对每 context 各扫一遍。 + +用法 : + python3 benchmark/benchmark_concurrency.py \ + --calib results/raw/prefill_local.csv \ + --transfer-tcp results/raw/kv_tensor_tcp.csv \ + --transfer-rdma results/raw/kv_tensor_rdma.csv \ + --context 16k,24k --hit-prefix 0.5 --phit 0.5 --slo-factor 1.5 \ + --out-prefix results/raw/conc +""" +import argparse, csv, math, os, random, statistics, sys + +GB = 1 << 30 +S_TOKEN_V32 = 61 * (512 + 64) * 2 # 70272 B = 68.6 KiB/token (V3.2 MLA 单 latent) +PAGE = 64 + + +def align(n, page=PAGE): + return max(0, (n // page) * page) + + +def parse_ctx(s): + out = [] + for t in s.replace(" ", "").lower().split(","): + if not t: + continue + out.append(int(float(t[:-1]) * 1024) if t.endswith("k") else int(t)) + return out + + +def fit_tprefill(path): + """从 L3 CSV 最小二乘拟合 T_prefill(N)=a*N+b*N^2 (过原点, ms)。""" + import numpy as np + byN = {} + with open(path) as f: + for r in csv.DictReader(f): + byN.setdefault(int(r["context_length"]), []).append(float(r["recompute_ms"])) + pts = sorted((N, statistics.median(v)) for N, v in byN.items()) + Ns = np.array([p[0] for p in pts], float) + Ts = np.array([p[1] for p in pts], float) + A = np.vstack([Ns, Ns ** 2]).T + coef, *_ = np.linalg.lstsq(A, Ts, rcond=None) + return float(coef[0]), float(coef[1]), pts + + +def load_retrieve_bw(path, want="zerocopy", op="get"): + bws = [] + with open(path) as f: + for r in csv.DictReader(f): + if r.get("path") == want and r.get("op") == op and r.get("success") == "1": + try: + bws.append(float(r["effective_bandwidth_gbps"])) + except (ValueError, KeyError): + pass + return statistics.median(bws) if bws else None + + +def t_prefill(N, a, b): + return a * N + b * N * N + + +def service_ms(N, is_hit, h, a, b, b_ret_gbps, lookup_ms): + """单请求服务时间(ms)。命中: lookup+restore+后缀现算; 未命中: 全量重算。""" + if not is_hit: + return t_prefill(N, a, b) + hN = align(int(round(h * N))) + uniq = N - hN + s_kv_gb = hN * S_TOKEN_V32 / GB + t_restore = (s_kv_gb / b_ret_gbps) * 1e3 if b_ret_gbps else 0.0 + return lookup_ms + t_restore + t_prefill(uniq, a, b) + + +def simulate(lam_rps, n_req, workers, svc_fn, rng, warmup_frac=0.1): + """c-worker FIFO 队列 + Poisson 到达的离散事件重放, 返回 TTFT 列表(ms, 去 warmup)。 + FIFO+c 同构 server: 每到达分配给最早空闲的 worker。""" + free = [0.0] * workers # 各 worker 空闲时刻(ms) + t = 0.0 + ttfts = [] + for i in range(n_req): + t += rng.expovariate(lam_rps) * 1e3 # 到达间隔(ms) + w = min(range(workers), key=lambda k: free[k]) + start = max(t, free[w]) + svc = svc_fn() + finish = start + svc + free[w] = finish + ttfts.append(finish - t) # TTFT = 排队等待 + 服务 + cut = int(n_req * warmup_frac) + return ttfts[cut:] + + +def pctl(xs, p): + import numpy as np + return float(np.percentile(xs, p)) if xs else float("inf") + + +def make_svc_fn(N, phit, h, a, b, b_ret, lookup_ms, rng): + """返回一个无参函数: 每次按 phit 概率抽命中/未命中, 给出服务时间(ms)。""" + def fn(): + is_hit = (rng.random() < phit) + return service_ms(N, is_hit, h, a, b, b_ret, lookup_ms) + return fn + + +def sweep_lambda(N, phit, h, a, b, b_ret, lookup_ms, slo_ms, n_req, workers, + lam_grid, seed): + """扫 λ, 返回 (λ_max_rps, per_lambda_rows)。λ_max = P95 TTFT ≤ SLO 的最大 λ。""" + rows, lam_max = [], 0.0 + for lam in lam_grid: + rng = random.Random(seed) # 同 seed 保证各 λ 间可比 + svc = make_svc_fn(N, phit, h, a, b, b_ret, lookup_ms, rng) + ttfts = simulate(lam, n_req, workers, svc, rng) + p50, p95, p99 = pctl(ttfts, 50), pctl(ttfts, 95), pctl(ttfts, 99) + ok = p95 <= slo_ms + if ok: + lam_max = lam + rows.append(dict(lambda_rps=round(lam, 4), p50_ttft_ms=round(p50, 1), + p95_ttft_ms=round(p95, 1), p99_ttft_ms=round(p99, 1), + slo_ms=round(slo_ms, 1), within_slo=int(ok))) + if not ok and lam_max > 0: + break # 已越过 SLO, 后续更高 λ 无意义 + return lam_max, rows + + +def build_grid(base_svc_ms, workers): + """λ 网格: 以单 worker 饱和率 (workers*1000/base_svc) 为量程, 细扫到 ~1.3×。""" + sat = workers * 1000.0 / base_svc_ms # 名义饱和到达率(req/s) + return [sat * f for f in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.75, 0.8, + 0.85, 0.9, 0.95, 1.0, 1.05, 1.1, 1.2, 1.3]] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--calib", default="results/raw/prefill_local.csv") + ap.add_argument("--transfer-tcp", default="results/raw/kv_tensor_tcp.csv") + ap.add_argument("--transfer-rdma", default="results/raw/kv_tensor_rdma.csv") + ap.add_argument("--context", default="8k,16k,24k") + ap.add_argument("--hit-prefix", type=float, default=0.5, help="命中请求复用的前缀比例 h") + ap.add_argument("--phit", type=float, default=0.5, help="请求命中缓存的概率") + ap.add_argument("--workers", type=int, default=1, help="并行 prefill worker 数 (算力串行=1)") + ap.add_argument("--lookup-ms", type=float, default=5.0, help="远端查表固定开销") + ap.add_argument("--slo-factor", type=float, default=1.5, help="SLO = nocache 单请求服务 × 此") + ap.add_argument("--n-req", type=int, default=4000) + ap.add_argument("--seed", type=int, default=2026) + ap.add_argument("--out-prefix", default="results/raw/conc") + args = ap.parse_args() + + a, b, pts = fit_tprefill(args.calib) + bw = {"nocache": None, + "tcp": load_retrieve_bw(args.transfer_tcp), + "rdma": load_retrieve_bw(args.transfer_rdma)} + print(f"[fit] T_prefill(N)={a:.5f}·N+{b:.3e}·N² | " + f"B_retrieve tcp={bw['tcp']:.2f} rdma={bw['rdma']:.2f} GB/s", flush=True) + print(f"[cfg] hit_prefix h={args.hit_prefix} phit={args.phit} workers={args.workers} " + f"lookup={args.lookup_ms}ms slo_factor={args.slo_factor}", flush=True) + + ctxs = parse_ctx(args.context) + all_rows, summary = [], [] + for N in ctxs: + # SLO 由 no-cache 全量重算服务时间定, 三种传输共用同一 SLO (公平) + base = t_prefill(N, a, b) + slo = base * args.slo_factor + # 平均服务时间 (用于 λ 网格量程): 取 nocache 与 hit 的期望 + for transport, b_ret in bw.items(): + # nocache = 缓存关闭 => 每请求都全量重算 (phit_eff=0); 有缓存 => 用真实 phit + phit_eff = 0.0 if transport == "nocache" else args.phit + mean_svc = (phit_eff * service_ms(N, True, args.hit_prefix, a, b, + b_ret, args.lookup_ms) + + (1 - phit_eff) * t_prefill(N, a, b)) + grid = build_grid(mean_svc, args.workers) + lam_max, rows = sweep_lambda(N, phit_eff, args.hit_prefix, a, b, b_ret, + args.lookup_ms, slo, args.n_req, args.workers, + grid, args.seed) + qpm = 60.0 * lam_max + for r in rows: + r.update(transport=transport, context_length=N, hit_prefix=args.hit_prefix, + phit=phit_eff, workers=args.workers) + all_rows.append(r) + summary.append(dict(context_length=N, transport=transport, + mean_service_ms=round(mean_svc, 1), slo_ms=round(slo, 1), + lambda_max_rps=round(lam_max, 4), qpm=round(qpm, 2))) + print(f" N={N//1024}k {transport:8} mean_svc={mean_svc:7.0f}ms " + f"SLO={slo:7.0f}ms λ_max={lam_max:.4f}rps QPM={qpm:.2f}", flush=True) + + # 每 context 打印 TCP/RDMA 相对 no-cache 的 QPM 增益 + RDMA vs TCP 成本比 + print("\n[对比] QPM 增益 (相对 no-cache) 与 成本/query (∝ 1/QPM, 归一 nocache):") + byN = {} + for s in summary: + byN.setdefault(s["context_length"], {})[s["transport"]] = s + for N, d in sorted(byN.items()): + base_qpm = d["nocache"]["qpm"] or 1e-9 + line = f" N={N//1024}k:" + for tp in ("nocache", "tcp", "rdma"): + q = d[tp]["qpm"] + gain = q / base_qpm + cost = base_qpm / q if q > 0 else float("inf") + line += f" {tp}=QPM {q:.1f}(×{gain:.2f}, cost×{cost:.2f})" + # RDMA vs TCP + if d["tcp"]["qpm"] > 0: + line += f" || rdma/tcp QPM={d['rdma']['qpm']/d['tcp']['qpm']:.2f}×" + print(line, flush=True) + + _write(args.out_prefix + "_sweep.csv", all_rows, + ["transport", "context_length", "hit_prefix", "phit", "workers", + "lambda_rps", "p50_ttft_ms", "p95_ttft_ms", "p99_ttft_ms", "slo_ms", "within_slo"]) + _write(args.out_prefix + "_summary.csv", summary, + ["context_length", "transport", "mean_service_ms", "slo_ms", "lambda_max_rps", "qpm"]) + print(f"\n[out] {args.out_prefix}_sweep.csv ({len(all_rows)} rows) + " + f"{args.out_prefix}_summary.csv ({len(summary)} rows)", flush=True) + + +def _write(path, rows, cols): + if not rows: + return + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for r in rows: + w.writerow(r) + + +if __name__ == "__main__": + main() + + diff --git a/src/code/issue5/benchmark/benchmark_kv_tensor.py b/src/code/issue5/benchmark/benchmark_kv_tensor.py new file mode 100644 index 0000000..77147b9 --- /dev/null +++ b/src/code/issue5/benchmark/benchmark_kv_tensor.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Issue5 真实 KV Tensor Transfer Microbench + +在 L1(随机 buffer) 之上, 按真实模型 config 造 **每 token KV 的 shape/dtype/分层布局**, +走同一个 mooncake store, 但**镜像 sglang hicache 的真实零拷贝路径**: + register_buffer(整块 KV buffer) -> batch_put_from / batch_get_into (逐 page, 每 page 2 个 key) +对照一条 **naive bytes 路径**(put_batch/get_batch + python bytes), 复现 L1 发现的 GET 瓶颈, +并证明零拷贝能否让 retrieve 突破 ~1.9 GB/s。仍**不跑 attention**, 因此绕开两个昆仑算子 blocker。 + +真实布局参考 sglang mooncake_store.py: + - MHA/GQA: 每 page 2 个 key ({k}, {v}); bytes_per_page = ksize_per_token * page_size + - MLA: 每 page 1 个 key; latent = (kv_lora_rank + qk_rope_head_dim) + - register_buffer(data_ptr, nbytes) 一次, 逐 page 给 (ptr+offset, size) 做 batch_put_from/get_into + - batch_put_from 返回码: 每元素 0 成功; batch_get_into 返回码: 每元素 >0(读到的字节数) 成功 + +用法 (prefill pod 内, conda env python310_torch29_cuda): + export PATH=/root/miniconda/envs/python310_torch29_cuda/bin:$PATH + TRANSPORT=tcp python3 benchmark_kv_tensor.py --model qwen3-32b --out kv_tcp.csv + TRANSPORT=rdma python3 benchmark_kv_tensor.py --model qwen3-32b --device mlx5_3 --out kv_rdma.csv +""" +import argparse, os, time, uuid, statistics, csv, sys + +GB = 1 << 30 +MiB = 1 << 20 +KiB = 1 << 10 + +# 模型 config 预设 (每 TP rank 视角): 逐层 KV, K 与 V 各一份 +# S_token(整模型/rank) = 2 * L * H_kv * D_h * b (GQA/MHA) +# = L * (kv_lora_rank + qk_rope) * b, 单 latent (MLA, 无独立 V) +MODELS = { + # Qwen3-32B: L=64, H_kv=8, D_h=128, bf16; 整模型 256 KiB/token, TP8 -> 32 KiB/token/rank + "qwen3-32b": dict(kind="mha", num_layers=64, num_kv_heads=8, head_dim=128, dtype_bytes=2), + # DeepSeek-V3.2 (MLA): L=61, kv_lora_rank=512, qk_rope_head_dim=64, bf16; 单 latent, 无独立 V + "deepseek-v32": dict(kind="mla", num_layers=61, kv_lora_rank=512, qk_rope=64, dtype_bytes=2), +} + + +def human(n): + if n >= GB: return f"{n/GB:.1f}GB" + if n >= MiB: return f"{n/MiB:.0f}MB" + return f"{n}B" + + +def parse_ctx(s): + out = [] + for tok in s.replace(" ", "").lower().split(","): + if not tok: continue + if tok.endswith("k"): out.append(int(float(tok[:-1]) * 1024)) + else: out.append(int(tok)) + return out + + +def s_token_per_rank(cfg, tp): + """每 token / 每 rank 的 KV 字节数。""" + if cfg["kind"] == "mha": + hkv = cfg["num_kv_heads"] // tp + assert hkv >= 1, "num_kv_heads 必须 >= tp" + return 2 * cfg["num_layers"] * hkv * cfg["head_dim"] * cfg["dtype_bytes"] + else: # mla: 单 latent, 逐层, 不随 TP 切分 (MLA latent 各 rank 复制/或 tp=1 视角) + return cfg["num_layers"] * (cfg["kv_lora_rank"] + cfg["qk_rope"]) * cfg["dtype_bytes"] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--master", default=os.environ.get("KV_IP", "192.0.2.10") + ":50051") + ap.add_argument("--metadata", default="http://" + os.environ.get("KV_IP", "192.0.2.10") + ":18080/metadata") + ap.add_argument("--protocol", default=os.environ.get("TRANSPORT", "tcp")) + ap.add_argument("--device", default=os.environ.get("MC_DEVICE", "")) + ap.add_argument("--local-hostname", default=os.environ.get("HOST_IP", "")) + ap.add_argument("--model", default="qwen3-32b", choices=list(MODELS)) + ap.add_argument("--tp", type=int, default=8) + ap.add_argument("--context", default="8k,32k,128k") + ap.add_argument("--page-size", type=int, default=64) + ap.add_argument("--paths", default="bytes,zerocopy") # 逗号: bytes / zerocopy + ap.add_argument("--iters", type=int, default=5) + ap.add_argument("--local-buffer-gb", type=float, default=1.0) + ap.add_argument("--out", default="/root/kv_tensor.csv") + args = ap.parse_args() + + if not args.local_hostname: + import socket + args.local_hostname = socket.gethostbyname(socket.gethostname()) + + global torch + import torch as torch + from mooncake.store import MooncakeDistributedStore + store = MooncakeDistributedStore() + ret = store.setup(args.local_hostname, args.metadata, 0, + int(args.local_buffer_gb * GB), args.protocol, + args.device, args.master) + if ret: + print(f"[FATAL] store.setup ret={ret}", file=sys.stderr); sys.exit(1) + print(f"[ok] setup protocol={args.protocol} device='{args.device}' model={args.model} " + f"tp={args.tp} local={args.local_hostname}", flush=True) + + cfg = MODELS[args.model] + st = s_token_per_rank(cfg, args.tp) + print(f"[cfg] S_token/rank = {st} B ({human(st)}), page={args.page_size} " + f"-> {human(st*args.page_size)}/page", flush=True) + + ctxs = parse_ctx(args.context) + paths = [p for p in args.paths.split(",") if p.strip()] + rows, tag = [], uuid.uuid4().hex[:8] + + try: # 清空可能的历史残留, 保证干净起点 + n = store.remove_all(); print(f"[clean] remove_all removed {n} objects", flush=True) + except Exception as e: + print(f"[clean] remove_all skip: {e}", flush=True) + + for N in ctxs: + for path in paths: + _bench(store, cfg, args, N, st, path, tag, rows) + + _write_csv(args.out, rows) + print(f"[done] wrote {len(rows)} rows -> {args.out}", flush=True) + + +def _layout(cfg, N, st, page_size): + """返回 (num_pages, keys_per_page, key_size_bytes, kv_bytes)。 + mha: 每 page 2 key(k,v), 各 (st/2)*page; mla: 每 page 1 key, st*page。""" + num_pages = N // page_size + if cfg["kind"] == "mha": + keys_per_page = 2 + key_size = (st // 2) * page_size + else: + keys_per_page = 1 + key_size = st * page_size + kv_bytes = num_pages * keys_per_page * key_size + return num_pages, keys_per_page, key_size, kv_bytes + + +def _median_bw(times, kv_bytes): + if not times: return 0.0, 0.0 + med = statistics.median(times) + bw = (kv_bytes / med / GB) if med > 0 else 0.0 + return med * 1e3, bw # ms, GB/s + + +def _bench(store, cfg, args, N, st, path, tag, rows): + ps = args.page_size + num_pages, kpp, key_size, kv_bytes = _layout(cfg, N, st, ps) + numel = kv_bytes // cfg["dtype_bytes"] # bf16 元素数 + prefix = f"kv_{tag}_{args.model}_{N}_{path}" + + # 造真实 KV: 逐元素 bf16, 值域 ~ N(0,0.1) 贴近真实激活尺度; read 端清零 + torch.manual_seed(1234) + write = (torch.randn(numel, dtype=torch.float32) * 0.1).to(torch.bfloat16).contiguous() + read = torch.zeros(numel, dtype=torch.bfloat16).contiguous() + + # 每 iter 唯一 key (含 it): 必须如此才能测到真实 PUT 传输带宽。 + # 关键坑 (见 doc/步骤3): TCP 模式下 mooncake put/batch_put_from 对**已存在的 key 是 + # no-op(按 key 去重, 直接跳过传输)** -> 若复用同一批 key, iter>=1 的 put 只有 ~3ms + # (假带宽 ~90GB/s)。故逐 iter 用唯一 key 做真插入, 并在每轮结束后 remove 释放 working + # set(对未被覆盖写的新 key, TCP/RDMA 下 remove 均返回 0), 避免 handle 累积耗尽。 + def page_keys(it): + ks = [] + for p in range(num_pages): + ks.append(f"{prefix}_{it}_{p}_k") + if kpp == 2: + ks.append(f"{prefix}_{it}_{p}_v") + return ks + + put_t, get_t, max_err, ok = [], [], -1.0, 1 + + if path == "zerocopy": + # 镜像 sglang: register_buffer 整块, batch_put_from/batch_get_into 逐 key(ptr+offset,size) + rc1 = store.register_buffer(write.data_ptr(), write.numel() * cfg["dtype_bytes"]) + rc2 = store.register_buffer(read.data_ptr(), read.numel() * cfg["dtype_bytes"]) + if rc1 or rc2: + print(f"[warn] register_buffer rc=({rc1},{rc2}) path={path} N={N}", flush=True) + wptrs = [write.data_ptr() + i * key_size for i in range(num_pages * kpp)] + rptrs = [read.data_ptr() + i * key_size for i in range(num_pages * kpp)] + sizes = [key_size] * (num_pages * kpp) + for it in range(args.iters): + ks = page_keys(it) + t = time.perf_counter(); pr = store.batch_put_from(ks, wptrs, sizes) + put_t.append(time.perf_counter() - t) + if any(r != 0 for r in pr): ok = 0 + t = time.perf_counter(); gr = store.batch_get_into(ks, rptrs, sizes) + get_t.append(time.perf_counter() - t) + if any(r <= 0 for r in gr): ok = 0 + for k in ks: # 逐轮清理 (untimed), 释放 handle + try: store.remove(k) + except Exception: pass + max_err = float((read.to(torch.float32) - write.to(torch.float32)).abs().max()) + store.unregister_buffer(write.data_ptr()); store.unregister_buffer(read.data_ptr()) + else: + max_err, ok = _bench_bytes(store, write, page_keys, key_size, + num_pages, kpp, args.iters, put_t, get_t) + # 兜底: 清掉本 context 可能残留的全部 key (remove_all 在 TCP/RDMA 下均可用) + try: store.remove_all() + except Exception: pass + + put_ms, put_bw = _median_bw(put_t, kv_bytes) + get_ms, get_bw = _median_bw(get_t, kv_bytes) + for op, ms, bw, me in (("put", put_ms, put_bw, -1.0), ("get", get_ms, get_bw, max_err)): + rows.append(dict( + transport=args.protocol, model=args.model, path=path, op=op, + context=N, tp=args.tp, page_size=ps, num_pages=num_pages, + num_keys=num_pages * kpp, kv_bytes=kv_bytes, kv_size=human(kv_bytes), + iters=args.iters, transfer_ms=round(ms, 3), + effective_bandwidth_gbps=round(bw, 3), + max_abs_error=("" if me < 0 else round(me, 6)), success=ok)) + print(f" {args.protocol:4} {path:8} {op} N={N:<7} {human(kv_bytes):>7} " + f"keys={num_pages*kpp:<5} BW={bw:.2f} GB/s {ms:.1f}ms " + f"err={'' if me<0 else round(me,5)} ok={ok}", flush=True) + del write, read + + +def _bench_bytes(store, write, page_keys, key_size, num_pages, kpp, iters, put_t, get_t): + """naive bytes 路径: put_batch/get_batch, 复现 L1 的 python bytes 拷贝瓶颈。""" + u8 = write.view(torch.uint8).contiguous() + mv = memoryview(u8.numpy()) + vals = [bytes(mv[i * key_size:(i + 1) * key_size]) for i in range(num_pages * kpp)] + max_err, ok = -1.0, 1 + for it in range(iters): + ks = page_keys(it) + t = time.perf_counter(); rc = store.put_batch(ks, vals) + put_t.append(time.perf_counter() - t) + if rc != 0: ok = 0 + t = time.perf_counter(); got = store.get_batch(ks) + get_t.append(time.perf_counter() - t) + if it == iters - 1: + bad = sum(1 for g in got if (g is None or len(g) != key_size)) + if bad: ok = 0 + max_err = 0.0 if (not bad and all(bytes(got[i]) == vals[i] + for i in range(len(vals)))) else 1.0 + for k in ks: + try: store.remove(k) + except Exception: pass + return max_err, ok + + +def _write_csv(path, rows): + if not rows: return + cols = ["transport", "model", "path", "op", "context", "tp", "page_size", + "num_pages", "num_keys", "kv_bytes", "kv_size", "iters", "transfer_ms", + "effective_bandwidth_gbps", "max_abs_error", "success"] + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for r in rows: w.writerow(r) + + +if __name__ == "__main__": + main() diff --git a/src/code/issue5/benchmark/benchmark_prefill.py b/src/code/issue5/benchmark/benchmark_prefill.py new file mode 100644 index 0000000..52c07d3 --- /dev/null +++ b/src/code/issue5/benchmark/benchmark_prefill.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Issue5 prefill_only end to end microbench + +prefill_only(max_new_tokens=0)在本地 device radix cache 下: + - 全量重算 T_prefill(N) = 冷 prefill 长度 N。 -> recompute 基线, 喂 B_crit 分母。 + - 前缀本地命中 total(N,h) = 先发 shared_prefix(hN) 预热进 radix, 不 flush, 再发 + shared_prefix + unique_suffix((1-h)N); cached_tokens=hN, + prefix 的 KV 直接复用(device 本地, restore≈0), 只现算后缀。 + - saved(N,h) = T_prefill(N) - total(N,h) = 复用 hN 前缀 KV 省下的**计算**时间。 +这是「以存代算」的**算力节省侧**真实标定(传输侧由 L2 给); 本地命中 => transfer/restore≈0。 + +命中率 h: shared_prefix 长度 = align(round(h*N), page); unique_suffix = N - prefix。 +JIT/无 cuda graph => 每个 shape 首次请求有编译毛刺, 故每 shape 先 warmup 丢弃, 再取 median。 + +用法(prefill pod 内, server 已 ready, 用 start_prefill_v32_nohicache.sh 起): + export PATH=/root/miniconda/envs/python310_torch29_cuda/bin:$PATH + python3 benchmark_prefill.py --base http://$(hostname -i):8000 --mode local \ + --transport local_radix --context 8k,32k,128k --hit 0,0.25,0.5,0.75,1.0 \ + --iters 3 --warmup 1 --out /root/prefill_local.csv +""" +import argparse, os, json, time, statistics, csv, sys, random, urllib.request + +GB = 1 << 30 +KiB = 1 << 10 +# DeepSeek-V3.2 MLA: 单 latent, 每 token KV = L*(kv_lora_rank+qk_rope)*b +V32 = dict(num_layers=61, kv_lora_rank=512, qk_rope=64, dtype_bytes=2) + + +def s_token_v32(): + return V32["num_layers"] * (V32["kv_lora_rank"] + V32["qk_rope"]) * V32["dtype_bytes"] + + +def parse_ctx(s): + out = [] + for tok in s.replace(" ", "").lower().split(","): + if not tok: + continue + out.append(int(float(tok[:-1]) * 1024) if tok.endswith("k") else int(tok)) + return out + + +def post(base, path, obj, timeout=1800): + data = json.dumps(obj).encode() + req = urllib.request.Request(base + path, data=data, + headers={"Content-Type": "application/json"}) + t = time.perf_counter() + with urllib.request.urlopen(req, timeout=timeout) as r: + body = r.read() + dt = time.perf_counter() - t + try: + return dt, json.loads(body) + except Exception: + return dt, {"_raw": body[:300].decode(errors="replace")} + + +def flush(base): + # /flush_cache 清本地 radix(host+device)。注意: 有在跑的请求时返回 400, 故只在 idle 调。 + try: + urllib.request.urlopen(urllib.request.Request(base + "/flush_cache", method="POST"), + timeout=60).read() + except Exception: + pass + time.sleep(0.4) + + +def cached_tokens(meta): + m = meta.get("meta_info", meta) if isinstance(meta, dict) else {} + return int(m.get("cached_tokens", -1)), int(m.get("prompt_tokens", -1)) + + +def align(n, page): + return max(0, (n // page) * page) + + +def make_ids(n, seed, vocab_lo=10, vocab_hi=100000): + rng = random.Random(seed) + return [rng.randint(vocab_lo, vocab_hi) for _ in range(n)] + + +def gen_prefill(base, ids, timeout=1800): + """发一个 prefill_only 请求 (max_new_tokens=0), 返回 (e2e_s, cached, prompt)。""" + body = {"input_ids": ids, "sampling_params": {"max_new_tokens": 0, "temperature": 0.0}, + "stream": False, "return_logprob": False} + dt, resp = post(base, "/generate", body, timeout=timeout) + c, p = cached_tokens(resp) + return dt, c, p, resp + + +def median_fresh(base, n, page, iters, seed0): + """冷全量 prefill 长度 n 的 median 墙钟(ms)。每次 flush + 唯一内容 => 真现算; shape 已预热。""" + ts = [] + for it in range(iters): + flush(base) + dt, c, p, _ = gen_prefill(base, make_ids(n, seed0 + it)) + ts.append(dt * 1e3) + return statistics.median(ts) + + +def warm_shape(base, n, seed): + """对某个长度 n 先发一个 throwaway 请求, 触发 JIT 编译(--disable-cuda-graph 首次有毛刺)。""" + flush(base) + try: + gen_prefill(base, make_ids(n, seed)) + except Exception: + pass + + +def bench_one(base, N, h, page, iters, warmup, st, rows, transport, mode): + Lhit = align(N, page) if h >= 1.0 else align(int(round(h * N)), page) + Luniq = N - Lhit + prefix = make_ids(Lhit, seed=42) if Lhit > 0 else [] + + # 预热 shape N (JIT), 然后测 recompute 基线 = 冷全量 T_prefill(N) + if warmup: + warm_shape(base, N, seed=90000) + cold_full_ms = median_fresh(base, N, page, iters, seed0=90001) + + # 冷 unique 部分 (1-h)N: 命中时仍需现算的后缀, 作为下界参考 + if Luniq > 0: + if warmup: + warm_shape(base, Luniq, seed=91000) + cold_uniq_ms = median_fresh(base, Luniq, page, iters, seed0=91001) + else: + cold_uniq_ms = 0.0 + + hit_ms_list, cached_list, prompt_list = [], [], [] + if Lhit > 0: + # 预热前缀进本地 radix (不 flush), 再预热一次 prefix+suffix 的完整 shape + flush(base) + gen_prefill(base, prefix) # warm prefix KV into radix (untimed) + if warmup: + gen_prefill(base, prefix + (make_ids(Luniq, seed=69999) if Luniq > 0 else [])) + for it in range(iters): + # 每 iter 唯一后缀; **不 flush** => prefix 一直在本地 radix, cached=Lhit + suffix = make_ids(Luniq, seed=70000 + it) if Luniq > 0 else [] + dt, c, p, _ = gen_prefill(base, prefix + suffix) + hit_ms_list.append(dt * 1e3) + cached_list.append(c) + prompt_list.append(p) + else: + # h=0: 命中即 cold_full 自身 (无前缀可复用) + if warmup: + warm_shape(base, N, seed=79999) + for it in range(iters): + flush(base) + dt, c, p, _ = gen_prefill(base, make_ids(N, seed=80000 + it)) + hit_ms_list.append(dt * 1e3) + cached_list.append(c) + prompt_list.append(p) + + hit_ms = statistics.median(hit_ms_list) + cached = int(statistics.median(cached_list)) if cached_list else -1 + prompt = int(statistics.median(prompt_list)) if prompt_list else -1 + kv_bytes = st * max(cached, 0) + # 本地命中: prefix KV 已在 device, restore≈0; saved = 全量重算 - 命中总时 = 省下的算力时间 + restore_ms = 0.0 + saved_ms = max(cold_full_ms - hit_ms, 0.0) + # 本地命中传输为 0 => 有效带宽不适用(留空); 远端 retrieve 带宽由 L2 给 + eff_bw = "" + + rows.append(dict( + transport=transport, context_length=N, hit_ratio=h, concurrency=1, + prefix_tokens=Lhit, uniq_tokens=Luniq, cached_tokens=cached, prompt_tokens=prompt, + kv_bytes=kv_bytes, s_token=st, + lookup_ms="", transfer_ms=0.0, restore_ms=restore_ms, + cold_uniq_ms=round(cold_uniq_ms, 2), recompute_ms=round(cold_full_ms, 2), + saved_ms=round(saved_ms, 2), total_ms=round(hit_ms, 2), + effective_bandwidth_gbps=eff_bw, max_abs_error="", success=1)) + print(f" {transport:10} N={N:<7} h={h:<4} prefix={Lhit:<7} cached={cached:<7} " + f"total={hit_ms:.0f}ms recompute={cold_full_ms:.0f}ms saved={saved_ms:.0f}ms " + f"cold_uniq={cold_uniq_ms:.0f}ms", flush=True) + + +def write_csv(path, rows): + if not rows: + return + cols = ["transport", "context_length", "hit_ratio", "concurrency", "prefix_tokens", + "uniq_tokens", "cached_tokens", "prompt_tokens", "kv_bytes", "s_token", + "lookup_ms", "transfer_ms", "restore_ms", "cold_uniq_ms", "recompute_ms", + "saved_ms", "total_ms", "effective_bandwidth_gbps", "max_abs_error", "success"] + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for r in rows: + w.writerow(r) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base", default="http://127.0.0.1:8000") + ap.add_argument("--mode", default="local", choices=["local"], + help="local: 本地 device radix 命中(远端 write_through 被昆仑 kernel blocker 卡死)") + ap.add_argument("--transport", default="local_radix") + ap.add_argument("--context", default="8k,32k,128k") + ap.add_argument("--hit", default="0,0.25,0.5,0.75,1.0") + ap.add_argument("--page-size", type=int, default=64) + ap.add_argument("--iters", type=int, default=3) + ap.add_argument("--warmup", type=int, default=1, help="每 shape 先发几个 throwaway 触发 JIT") + ap.add_argument("--out", default="/root/prefill_local.csv") + args = ap.parse_args() + + st = s_token_v32() + print(f"[cfg] V3.2 MLA S_token/token = {st} B ({st/KiB:.1f} KiB), mode={args.mode}, base={args.base}", flush=True) + ctxs = parse_ctx(args.context) + hits = [float(x) for x in args.hit.replace(" ", "").split(",") if x] + rows = [] + for N in ctxs: + for h in hits: + try: + bench_one(args.base, N, h, args.page_size, args.iters, args.warmup, + st, rows, args.transport, args.mode) + except Exception as e: + print(f" [err] N={N} h={h}: {e}", flush=True) + write_csv(args.out, rows) + print(f"[done] wrote {len(rows)} rows -> {args.out}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/code/issue5/benchmark/benchmark_transfer.py b/src/code/issue5/benchmark/benchmark_transfer.py new file mode 100644 index 0000000..479f09c --- /dev/null +++ b/src/code/issue5/benchmark/benchmark_transfer.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Issue5 步骤2 / Layer1 —— Mooncake primitive microbench (不进模型) + +直接用 mooncake python client 连已起的远端 store (kvstore 节点), put/get 不同大小 buffer, +测传输层本身的带宽/延迟。TCP vs RDMA 用 TRANSPORT 环境变量或 --protocol 切换。 + +用法 (在 prefill pod 内, conda env python310_torch29_cuda): + export PATH=/root/miniconda/envs/python310_torch29_cuda/bin:$PATH + TRANSPORT=tcp python3 benchmark_transfer.py --out /root/transfer_tcp.csv + TRANSPORT=rdma python3 benchmark_transfer.py --out /root/transfer_rdma.csv --device eth1 + +setup(local_hostname, metadata_server, global_segment_size, local_buffer_size, + protocol, device_name, master_server_address) # 7 位置参数, 见 sglang mooncake_store.py +""" +import argparse, os, time, uuid, statistics, csv, sys +from concurrent.futures import ThreadPoolExecutor + +GB = 1 << 30 +MB = 1 << 20 + + +def human(n): + if n >= GB: return f"{n/GB:.0f}GB" + if n >= MB: return f"{n/MB:.0f}MB" + return f"{n}B" + + +def parse_sizes(s): + out = [] + for tok in s.split(","): + tok = tok.strip().lower() + if not tok: continue + if tok.endswith("gb"): out.append(int(float(tok[:-2]) * GB)) + elif tok.endswith("mb"): out.append(int(float(tok[:-2]) * MB)) + elif tok.endswith("kb"): out.append(int(float(tok[:-2]) * 1024)) + else: out.append(int(tok)) + return out + + +def pctl(vals, p): + if not vals: return 0.0 + xs = sorted(vals) + k = max(0, min(len(xs) - 1, int(round((p / 100.0) * (len(xs) - 1))))) + return xs[k] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--master", default=os.environ.get("KV_IP", "192.0.2.10") + ":50051") + ap.add_argument("--metadata", default="http://" + os.environ.get("KV_IP", "192.0.2.10") + ":18080/metadata") + ap.add_argument("--protocol", default=os.environ.get("TRANSPORT", "tcp")) + ap.add_argument("--device", default=os.environ.get("MC_DEVICE", "")) # rdma NIC 名, tcp 时留空 + ap.add_argument("--local-hostname", default=os.environ.get("HOST_IP", "")) + ap.add_argument("--sizes", default="64mb,256mb,1gb") + ap.add_argument("--concurrency", default="1,2,4,8") + ap.add_argument("--iters", type=int, default=20) + ap.add_argument("--local-buffer-gb", type=float, default=5.0) + ap.add_argument("--mode", default="simple", choices=["simple", "zcget", "zcput"], + help="simple=原有 put+bytes-get; zcget=零拷贝 batch_get_into 取回; " + "zcput=零拷贝 batch_put_from 写入 (均镜像 hicache 真实路径)") + ap.add_argument("--out", default="/root/transfer.csv") + args = ap.parse_args() + + if not args.local_hostname: + import socket + args.local_hostname = socket.gethostbyname(socket.gethostname()) + + from mooncake.store import MooncakeDistributedStore + store = MooncakeDistributedStore() + # global_segment_size=0: client 不贡献存储, 所有数据落到远端 kvstore 段 -> 真跨网传输 + ret = store.setup(args.local_hostname, args.metadata, 0, + int(args.local_buffer_gb * GB), args.protocol, + args.device, args.master) + if ret: + print(f"[FATAL] store.setup ret={ret}", file=sys.stderr); sys.exit(1) + print(f"[ok] setup protocol={args.protocol} device='{args.device}' " + f"local={args.local_hostname} master={args.master}", flush=True) + + sizes = parse_sizes(args.sizes) + concs = [int(x) for x in args.concurrency.split(",") if x.strip()] + rows = [] + run_tag = uuid.uuid4().hex[:8] + + for size in sizes: + payload = os.urandom(min(size, 64 * MB)) + # 大 value 用重复拼接避免 os.urandom 太慢, 但仍保持 size 字节 + if size > len(payload): + payload = (payload * (size // len(payload) + 1))[:size] + for conc in concs: + if args.mode == "zcget": + _bench_zc_get(store, size, conc, args.iters, payload, run_tag, rows, args.protocol) + elif args.mode == "zcput": + _bench_zc_put(store, size, conc, args.iters, payload, run_tag, rows, args.protocol) + else: + _bench_one(store, size, conc, args.iters, payload, run_tag, rows, args.protocol) + + _write_csv(args.out, rows) + print(f"[done] wrote {len(rows)} rows -> {args.out}", flush=True) + try: store.close() + except Exception: pass + + +def _bench_one(store, size, conc, iters, payload, run_tag, rows, protocol): + keys = [f"bench_{run_tag}_{human(size)}_{conc}_{i}" for i in range(iters)] + + def do_put(i): + t = time.perf_counter() + rc = store.put(keys[i], payload) + return (time.perf_counter() - t, rc) + + def do_get(i): + t = time.perf_counter() + v = store.get(keys[i]) + dt = time.perf_counter() - t + return (dt, 0 if (v is not None and len(v) == size) else 1) + + # ---- PUT ---- + put_lat, put_ok, put_wall = _run_parallel(do_put, iters, conc) + # ---- GET ---- + get_lat, get_ok, get_wall = _run_parallel(do_get, iters, conc) + + for op, lats, oks, wall in (("put", put_lat, put_ok, put_wall), + ("get", get_lat, get_ok, get_wall)): + # 有效聚合带宽 = 总字节 / 实测批次墙钟时间 (并发下反映真实吞吐) + total_bytes = size * len(lats) + ok = int(sum(oks) == 0) + bw = (total_bytes / wall / GB) if (wall > 0 and ok) else 0.0 + row = dict(transport=protocol, op=op, size_bytes=size, size=human(size), + concurrency=conc, iters=len(lats), total_bytes=total_bytes, + wall_s=round(wall, 4), bandwidth_gbps=round(bw, 3), + lat_p50_ms=round(pctl(lats, 50) * 1e3, 3), + lat_p95_ms=round(pctl(lats, 95) * 1e3, 3), + lat_p99_ms=round(pctl(lats, 99) * 1e3, 3), + lat_mean_ms=round(statistics.mean(lats) * 1e3, 3) if lats else 0, + success=ok) + rows.append(row) + print(f" {protocol:4} {op} {human(size):>6} c={conc:<2} " + f"BW={row['bandwidth_gbps']:.2f} GB/s p50={row['lat_p50_ms']:.1f}ms " + f"p99={row['lat_p99_ms']:.1f}ms ok={row['success']}", flush=True) + + # 清理 keys + for k in keys: + try: store.remove(k) + except Exception: pass + + +def _bench_zc_put(store, size, conc, iters, payload, run_tag, rows, protocol): + """零拷贝 PUT: 镜像 sglang hicache 的 register_buffer + batch_put_from 真实 write_through 路径。 + 对照 _bench_one 的 simple bytes put(store.put 传 python bytes), 这里一次 register_buffer 整块 + write 缓冲, 一批 batch_put_from(keys, ptrs, sizes) 从预注册张量直 DMA 出去, 无 python 中间对象。 + 并发 = 单批次内并行写 conc 个 buffer, BW = conc*size / 批次中位墙钟时间。 + ⚠️ 关键坑(见 doc/步骤3): TCP 下 put/batch_put_from 对**已存在的 key 是 no-op(按 key 去重, + 直接跳过传输)** -> 必须每 iter 用唯一 key(含 it) 做真插入, 否则 iter>=1 只花 ~3ms(假带宽)。""" + global torch + import torch as torch + # 整块 write 缓冲 conc*size 字节, 用 payload 填充(避免全零), 注册一次 + write = torch.frombuffer(bytearray(payload * conc), dtype=torch.uint8).clone().contiguous() + rc = store.register_buffer(write.data_ptr(), conc * size) + if rc: + print(f"[warn] register_buffer rc={rc} zcput size={human(size)} c={conc}", flush=True) + wptrs = [write.data_ptr() + i * size for i in range(conc)] + sizes = [size] * conc + put_t, ok = [], 1 + for it in range(iters): + keys = [f"zcput_{run_tag}_{human(size)}_{conc}_{it}_{i}" for i in range(conc)] + t = time.perf_counter() + pr = store.batch_put_from(keys, wptrs, sizes) + put_t.append(time.perf_counter() - t) + if any(r != 0 for r in pr): ok = 0 + for k in keys: # 逐轮清理(untimed), 释放 handle 并避免下轮去重 + try: store.remove(k) + except Exception: pass + med = statistics.median(put_t) if put_t else 0.0 + total_bytes = size * conc + bw = (total_bytes / med / GB) if (med > 0 and ok) else 0.0 + row = dict(transport=protocol, op="put_zc", size_bytes=size, size=human(size), + concurrency=conc, iters=len(put_t), total_bytes=total_bytes, + wall_s=round(med, 4), bandwidth_gbps=round(bw, 3), + lat_p50_ms=round(pctl(put_t, 50) * 1e3, 3), + lat_p95_ms=round(pctl(put_t, 95) * 1e3, 3), + lat_p99_ms=round(pctl(put_t, 99) * 1e3, 3), + lat_mean_ms=round(statistics.mean(put_t) * 1e3, 3) if put_t else 0, + success=ok) + rows.append(row) + print(f" {protocol:4} put_zc {human(size):>6} c={conc:<2} " + f"BW={row['bandwidth_gbps']:.2f} GB/s p50={row['lat_p50_ms']:.1f}ms " + f"p99={row['lat_p99_ms']:.1f}ms ok={row['success']}", flush=True) + store.unregister_buffer(write.data_ptr()) + try: store.remove_all() + except Exception: pass + + +def _bench_zc_get(store, size, conc, iters, payload, run_tag, rows, protocol): + """零拷贝 GET: 镜像 sglang hicache 的 register_buffer + batch_get_into 真实 retrieve 路径。 + 对照 _bench_one 的 simple bytes get(每 key 新建 python bytes + memcpy + GIL, 卡 ~1.9GB/s), + 这里一次 register_buffer 整块 read 缓冲, 逐 key 直 DMA 进预注册张量, 无 python 中间对象。 + 并发用「一次 batch_get_into 取 conc 个 key」表达(与 simple 的 ThreadPoolExecutor 并发对齐语义: + 同一批次内并行取 conc 个 buffer), BW = conc*size / 批次中位墙钟时间。""" + global torch + import torch as torch + keys = [f"zcget_{run_tag}_{human(size)}_{conc}_{i}" for i in range(conc)] + # 预置数据 (untimed setup): 每 key 存 size 字节 payload + for k in keys: + store.put(k, payload) + # 整块 read 缓冲 conc*size 字节, 注册一次 + read = torch.zeros(conc * size, dtype=torch.uint8).contiguous() + rc = store.register_buffer(read.data_ptr(), conc * size) + if rc: + print(f"[warn] register_buffer rc={rc} zcget size={human(size)} c={conc}", flush=True) + rptrs = [read.data_ptr() + i * size for i in range(conc)] + sizes = [size] * conc + get_t, ok = [], 1 + for _ in range(iters): + t = time.perf_counter() + gr = store.batch_get_into(keys, rptrs, sizes) + get_t.append(time.perf_counter() - t) + if any(r <= 0 for r in gr): ok = 0 + med = statistics.median(get_t) if get_t else 0.0 + total_bytes = size * conc + bw = (total_bytes / med / GB) if (med > 0 and ok) else 0.0 + row = dict(transport=protocol, op="get_zc", size_bytes=size, size=human(size), + concurrency=conc, iters=len(get_t), total_bytes=total_bytes, + wall_s=round(med, 4), bandwidth_gbps=round(bw, 3), + lat_p50_ms=round(pctl(get_t, 50) * 1e3, 3), + lat_p95_ms=round(pctl(get_t, 95) * 1e3, 3), + lat_p99_ms=round(pctl(get_t, 99) * 1e3, 3), + lat_mean_ms=round(statistics.mean(get_t) * 1e3, 3) if get_t else 0, + success=ok) + rows.append(row) + print(f" {protocol:4} get_zc {human(size):>6} c={conc:<2} " + f"BW={row['bandwidth_gbps']:.2f} GB/s p50={row['lat_p50_ms']:.1f}ms " + f"p99={row['lat_p99_ms']:.1f}ms ok={row['success']}", flush=True) + store.unregister_buffer(read.data_ptr()) + for k in keys: + try: store.remove(k) + except Exception: pass + + +def _run_parallel(fn, iters, conc): + lats, oks = [], [] + t0 = time.perf_counter() + if conc <= 1: + for i in range(iters): + dt, rc = fn(i); lats.append(dt); oks.append(rc) + else: + with ThreadPoolExecutor(max_workers=conc) as ex: + for dt, rc in ex.map(fn, range(iters)): + lats.append(dt); oks.append(rc) + wall = time.perf_counter() - t0 + return lats, oks, wall + + +def _write_csv(path, rows): + if not rows: return + cols = ["transport", "op", "size", "size_bytes", "concurrency", "iters", + "total_bytes", "wall_s", "bandwidth_gbps", "lat_mean_ms", + "lat_p50_ms", "lat_p95_ms", "lat_p99_ms", "success"] + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for r in rows: w.writerow(r) + + +if __name__ == "__main__": + main() diff --git a/src/code/issue5/configs/rdma.yaml b/src/code/issue5/configs/rdma.yaml new file mode 100644 index 0000000..f78f96c --- /dev/null +++ b/src/code/issue5/configs/rdma.yaml @@ -0,0 +1,47 @@ +# ============================================================================= +# Issue5 RDMA (RoCEv2) 传输配置 +# ----------------------------------------------------------------------------- +# Mooncake transfer engine 的 RDMA 协议档。相对 tcp.yaml 的**唯一实现改动**就是 +# protocol 切成 rdma + 指定 RDMA NIC(device_name), 其余端口/段大小不变。 +# store 端与 client 端 protocol 必须一致。被 scripts/run_benchmark.sh 读取。 +# +# device→netdev 映射两机不一致: +# RDMA 数据 NIC = eth1(198.51.100.x/27 子网) +# kvstore: eth1 = mlx5_2; prefill: eth1 = mlx5_3 +# ============================================================================= + +transport: rdma # Mooncake protocol = rdma (RoCEv2, verbs, 零拷贝直 DMA) + +# ---- store 端 (kvstore pod, MC_PROTOCOL=rdma bash start_kvstore.sh) ---- +store: + protocol: rdma + device_name: "" # store 侧自动发现 OK (Topology discovery: Found 6 HCAs) + auto_discovery: true # MC_MS_AUTO_DISC=1 + global_segment_size: 200gb + local_buffer_size: 0 + metadata_port: 18080 # 握手/元数据走已放通端口; RDMA 数据面另走 verbs + store_port: 18081 + master_rpc_port: 50051 + te_port: 15964 # transfer engine 端口 (跨机安全组白名单内) + +# ---- client 端 (prefill pod, TRANSPORT=rdma MC_DEVICE=mlx5_3 ...) ---- +client: + protocol: rdma + device_name: mlx5_3 # 必填: prefill 的 eth1(RDMA NIC)=mlx5_3; 空会 No available RNIC + local_buffer_size_gb: 6 + global_segment_size: 0 + +# ---- RDMA 硬件事实 (doc/步骤1-rdma.md 实测, 非假设) ---- +hardware: + link_layer: Ethernet # RoCEv2 (非 IB) + gid_index: 3 # RoCEv2 gid=3, MTU 4096 + hca_count: {kvstore: 6, prefill: 7} + rdma_subnet: {kvstore: "198.51.100.0/27", prefill: "198.51.100.32/27"} # 不同 /27, RoCEv2 可路由 + peermem: kunlun_peermem # 支持 GPUDirect RDMA (XPU 显存直通) + loopback_ib_write_bw_gbps: 18.3 # 146.73 Gb/s @ 65536B, gid3, 证明 verbs 全链路健康 + +# ---- 期望性能量级 (来自 L1/L2 实测) ---- +expect: + l1_put_gbps: "4.9 - 16.0" # 随并发 scale, transfer_rdma.csv (~13.9x TCP) + l1_get_gbps: "1.5 - 2.1" # simple-API get 卡在 client python bytes 拷贝(非网络) + l2_zerocopy_get_gbps: "20.68" # ★ kv_tensor_rdma.csv zerocopy get 中位, ~18.8x TCP-zc diff --git a/src/code/issue5/configs/tcp.yaml b/src/code/issue5/configs/tcp.yaml new file mode 100644 index 0000000..5b3bca0 --- /dev/null +++ b/src/code/issue5/configs/tcp.yaml @@ -0,0 +1,32 @@ +# ============================================================================= +# Issue5 TCP 传输配置 (基线) +# ----------------------------------------------------------------------------- +# Mooncake transfer engine 的 TCP 协议档。store 端与 client 端的 protocol 必须一致。 +# 被 scripts/run_benchmark.sh 读取, 落成 store 的 --config JSON 与 client 的 +# TRANSPORT/MC_DEVICE 环境变量。字节级值(如 segment)由脚本换算。 +# ============================================================================= + +transport: tcp # Mooncake protocol = tcp (走内核 socket, 无 verbs) + +# ---- store 端 (kvstore pod, MC_PROTOCOL=tcp bash start_kvstore.sh) ---- +store: + protocol: tcp + device_name: "" # TCP 不用 RDMA NIC, 留空 + global_segment_size: 200gb # 贡献给全局 KV 池的本机内存 + local_buffer_size: 0 # 纯存储节点不发起 get/put + metadata_port: 18080 # http metadata (8080 常被同宿主机其它 hostNetwork pod 占) + store_port: 18081 + master_rpc_port: 50051 + +# ---- client 端 (prefill pod 内跑 benchmark, 跨机连 store) ---- +client: + protocol: tcp + device_name: "" # TCP client 无需指定网卡 + local_buffer_size_gb: 6 # bytes 路径需 >= 单次 kv_bytes (128k=4GB) + global_segment_size: 0 # client 不贡献内存 + +# ---- 期望性能量级 (来自 L1/L2 实测, 仅作 sanity, 非硬编码) ---- +expect: + l1_put_gbps: "0.77 - 2.88" # 随并发, transfer_tcp.csv + l1_get_gbps: "0.52 - 0.94" # 卡在内核 socket 拷贝 + l2_zerocopy_get_gbps: "1.10" # kv_tensor_tcp.csv zerocopy get 中位 diff --git a/src/code/issue5/kvstore_mooncake.yaml b/src/code/issue5/kvstore_mooncake.yaml new file mode 100644 index 0000000..556460e --- /dev/null +++ b/src/code/issue5/kvstore_mooncake.yaml @@ -0,0 +1,171 @@ +apiVersion: example.com/v1 +kind: FedDeployment +metadata: + labels: + example.com/managed-by: deploytask + example.com/feddeploy-name: issue5-kvstore-by-fed-deploy + example.com/inference-service-name: issue5-kvstore-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + name: issue5-kvstore-mooncake + namespace: default +spec: + minReadySeconds: 30 + replicas: 1 + selector: + matchLabels: + example.com/feddeploy-name: issue5-kvstore-by-fed-deploy + example.com/inference-service-name: issue5-kvstore-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + strategy: + canary: + maxSurge: 100% + maxUnavailable: 100% + steps: + - pause: {} + setWeight: 100 + template: + metadata: + annotations: + example.com/update-by-job: "0" + + labels: + example.com/feddeploy-name: issue5-kvstore-by-fed-deploy + example.com/inference-service-name: issue5-kvstore-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + spec: + replicas: 1 + template: + metadata: + labels: + example.com/feddeploy-name: issue5-kvstore-by-fed-deploy + example.com/inference-service-name: issue5-kvstore-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + annotations: + scheduling.volcano.sh/queue-name: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - 192.0.2.101 + - 192.0.2.102 + - 192.0.2.103 + - 192.0.2.104 + - 192.0.2.105 + - 192.0.2.106 + - 192.0.2.107 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rdma-test + operator: In + values: + - "true" + topologyKey: "kubernetes.io/hostname" + + containers: + # 容器只拉起并 idle, 不在启动时跑存储服务。 + # 实际启动逻辑放在 scripts/start_kvstore.sh, 进 pod 后手动执行: + # kubectl exec -it -- bash + # MC_PROTOCOL=rdma MC_SEGMENT_SIZE=200gb bash /mnt/cluster/issue5/start_kvstore.sh + - args: + - -c + - | + . ~/.bashrc + echo "[issue5-kvstore] Pod ready. Exec in and run scripts/start_kvstore.sh manually." + tail -f /dev/null + command: + - /bin/sh + # 容器只 idle, 存储服务(mooncake_master + store_service)由 + # scripts/start_kvstore.sh 手动执行, 该脚本 export 自己的 MC_* / PATH / HOST_IP, + # 且本节点不跑模型, 故此处删除全部 sglang/MoE/kunlun 调优变量, + # 仅保留 downward-API pod 元信息, 方便 exec 进 pod 调试。 + env: + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: inference-sglang: + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - | + pkill -9 -f 'mooncake' || true + name: model-server + ports: + - containerPort: 6677 + protocol: TCP + - containerPort: 8000 + protocol: TCP + resources: + limits: + kunlunxin.com/xpu: "8" + rdma/hca: "1" + requests: + kunlunxin.com/xpu: "8" + rdma/hca: "1" + securityContext: + capabilities: + add: + - SYS_PTRACE + - SYS_ADMIN + runAsUser: 0 + volumeMounts: + - mountPath: /dev/shm + name: shm-cache + - mountPath: /mnt/cluster + name: pvc-model + dnsPolicy: ClusterFirst + hostNetwork: true + nodeSelector: + example.com/dedicated-pool: + restartPolicy: Always + schedulerName: volcano + securityContext: {} + shareProcessNamespace: true + terminationGracePeriodSeconds: 600 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 32Gi + name: local-cache + - emptyDir: + medium: Memory + sizeLimit: 800Gi + name: shm-cache + - emptyDir: {} + name: shared-volume + - name: pvc-model + persistentVolumeClaim: + claimName: model-pvc + # topologyKey: example.com/dcn-tor + # topologyKeyExclusive: true \ No newline at end of file diff --git a/src/code/issue5/model/bandwidth_model.py b/src/code/issue5/model/bandwidth_model.py new file mode 100644 index 0000000..87b288a --- /dev/null +++ b/src/code/issue5/model/bandwidth_model.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Issue5 带宽临界值模型 B_crit + 用 L1/L2/L3 实测标定 + +闭式解 (PLAN 附录 A): + T_full(N) = T_prefill(N) + T_cache(N,h,B) = T_0 + S_KV(hN)/B + T_restore(hN) + T_prefill((1-h)N) + 令 T_cache < T_full 解出临界带宽: + B_crit(N,h) = S_KV(hN) / ( T_prefill(N) - T_prefill((1-h)N) - T_lookup - T_restore - L_net ) + 划算条件: 实际 retrieve 带宽 B_retrieve > B_crit <=> 以存代算比全量重算快。 + +本脚本做四件事: + 1) 用 L3 prefill_local.csv 拟合算力侧 T_prefill(N)=a*N+b*N^2 (prefill 前向墙钟 ms, 过原点)。 + b 项吸收 attention 的超线性; NSA 稀疏(topk=2048)会压平, 故 128k 外推给区间上界(见 doc)。 + 2) 每 (N,h) 用实测 recompute/cold_uniq 直接算 B_crit(本地命中 restore/lookup≈0), + 再用拟合模型算一遍 -> 标定误差 (model vs measured)。 + 3) 用 L1/L2 的 B_retrieve(TCP/RDMA zerocopy) 算 T_restore、净收益、是否划算。 + 4) 外推线上 128K/30% 命中: 预测 B_crit, 对比线上 TCP~2.5GB/s / 实测 TCP / RDMA。 + 出 results/model_fit.csv + 拟合图 results/figures/。 + +用法: + python3 model/bandwidth_model.py \ + --calib results/raw/prefill_local.csv \ + --transfer-tcp results/raw/kv_tensor_tcp.csv \ + --transfer-rdma results/raw/kv_tensor_rdma.csv \ + --report results/model_fit.csv --figdir results/figures +""" +import argparse, csv, os, sys, statistics + +GB = 1 << 30 +KiB = 1 << 10 +# V3.2 MLA 单 latent: 每 token KV = L*(kv_lora_rank+qk_rope)*b (与 L2/L3 脚本一致) +S_TOKEN_V32 = 61 * (512 + 64) * 2 # = 70272 B = 68.6 KiB/token +PAGE = 64 + + +def s_kv_bytes(n_tok): + return n_tok * S_TOKEN_V32 + + +def align(n, page=PAGE): + return max(0, (n // page) * page) + + +def load_prefill(path): + """读 L3 CSV -> list[dict(N,h,cached,recompute_ms,cold_uniq_ms,total_ms,saved_ms)]。""" + rows = [] + with open(path) as f: + for r in csv.DictReader(f): + rows.append(dict( + N=int(r["context_length"]), h=float(r["hit_ratio"]), + cached=int(r["cached_tokens"]), + recompute_ms=float(r["recompute_ms"]), + cold_uniq_ms=float(r["cold_uniq_ms"]), + total_ms=float(r["total_ms"]), + saved_ms=float(r["saved_ms"]))) + return rows + + +def fit_tprefill(rows): + """最小二乘拟合 T_prefill(N)=a*N+b*N^2 (过原点)。每个 N 用 h=0 的 recompute 均值做标定点。 + 返回 (a[ms/tok], b[ms/tok^2], r2, points)。""" + import numpy as np + # 每个 N 的全量重算基线 = 该 N 所有行的 recompute_ms 中位数 (h 无关, 都是全量现算) + byN = {} + for r in rows: + byN.setdefault(r["N"], []).append(r["recompute_ms"]) + pts = sorted((N, statistics.median(v)) for N, v in byN.items()) + Ns = np.array([p[0] for p in pts], dtype=float) + Ts = np.array([p[1] for p in pts], dtype=float) + # 设计矩阵 [N, N^2], 无截距 (N=0 -> 0 prefill) + A = np.vstack([Ns, Ns ** 2]).T + coef, *_ = np.linalg.lstsq(A, Ts, rcond=None) + a, b = float(coef[0]), float(coef[1]) + pred = A @ coef + ss_res = float(((Ts - pred) ** 2).sum()) + ss_tot = float(((Ts - Ts.mean()) ** 2).sum()) + r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 1.0 + return a, b, r2, pts + + +def t_prefill(N, a, b): + return a * N + b * N * N + + +def load_retrieve_bw(path, want_path="zerocopy", op="get"): + """从 L2 kv_tensor CSV 取 retrieve 有效带宽 (GB/s) 的中位数。""" + bws = [] + with open(path) as f: + for r in csv.DictReader(f): + if r.get("path") == want_path and r.get("op") == op and r.get("success") == "1": + try: + bws.append(float(r["effective_bandwidth_gbps"])) + except (ValueError, KeyError): + pass + return statistics.median(bws) if bws else None + + +def b_crit_gbps(s_kv_B, denom_ms): + """B_crit = S_KV / 分母(转秒). 分母<=0 => 不可能划算 (返回 inf)。""" + if denom_ms <= 0: + return float("inf") + return (s_kv_B / GB) / (denom_ms / 1e3) + + +def compute_cells(rows, a, b, retrieves): + """对每 (N,h) 生成: 实测 B_crit + 模型 B_crit + 每种传输的 T_restore/净收益/是否划算。""" + out = [] + for r in rows: + N, h, cached = r["N"], r["h"], r["cached"] + if cached <= 0: + continue # h=0 无前缀可复用, B_crit 无意义 + s_kv = s_kv_bytes(cached) + # 实测分母 = 前缀现算时间 = 全量重算 - 冷唯一后缀 (本地命中 lookup/restore≈0) + denom_meas = r["recompute_ms"] - r["cold_uniq_ms"] + bcrit_meas = b_crit_gbps(s_kv, denom_meas) + # 模型分母 = T_prefill(N) - T_prefill((1-h)N) + uniq = N - align(int(round(h * N))) + denom_model = t_prefill(N, a, b) - t_prefill(uniq, a, b) + bcrit_model = b_crit_gbps(s_kv, denom_model) + base = dict(N=N, h=h, cached=cached, s_kv_gb=round(s_kv / GB, 4), + tprefill_N_ms=round(t_prefill(N, a, b), 1), + denom_meas_ms=round(denom_meas, 1), + denom_model_ms=round(denom_model, 1), + bcrit_meas_gbps=round(bcrit_meas, 4), + bcrit_model_gbps=round(bcrit_model, 4), + measured_saved_ms=r["saved_ms"]) + for tname, bw in retrieves.items(): + t_restore = (s_kv / GB) / bw * 1e3 # ms + net = denom_meas - t_restore # 净省下的时间 (算侧收益 - restore 成本) + row = dict(base, transport=tname, b_retrieve_gbps=round(bw, 3), + t_restore_ms=round(t_restore, 1), + net_saved_ms=round(net, 1), + profitable=int(bw > bcrit_meas)) + out.append(row) + return out + + +def extrapolate(a, b, retrieves, N=131072, h=0.30, online_tcp=2.5): + """外推线上场景 (默认 128K / 30% 命中)。返回 dict。""" + hN = align(int(round(h * N))) + uniq = N - hN + s_kv = s_kv_bytes(hN) + denom = t_prefill(N, a, b) - t_prefill(uniq, a, b) + bcrit = b_crit_gbps(s_kv, denom) + res = dict(N=N, h=h, hit_tokens=hN, s_kv_gb=round(s_kv / GB, 3), + tprefill_N_ms=round(t_prefill(N, a, b), 0), + tprefill_uniq_ms=round(t_prefill(uniq, a, b), 0), + denom_ms=round(denom, 0), bcrit_gbps=round(bcrit, 4)) + bws = dict(retrieves); bws["online_tcp_hypo"] = online_tcp + for tname, bw in bws.items(): + res[f"Trestore_{tname}_ms"] = round((s_kv / GB) / bw * 1e3, 0) + res[f"margin_{tname}_x"] = round(bw / bcrit, 1) if bcrit > 0 else float("inf") + return res + + +def write_csv(path, cells): + cols = ["N", "h", "cached", "s_kv_gb", "tprefill_N_ms", "denom_meas_ms", + "denom_model_ms", "bcrit_meas_gbps", "bcrit_model_gbps", "transport", + "b_retrieve_gbps", "t_restore_ms", "net_saved_ms", "profitable", + "measured_saved_ms"] + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore") + w.writeheader() + for c in cells: + w.writerow(c) + + +def make_plots(figdir, rows, cells, a, b, retrieves, pts): + import numpy as np + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + os.makedirs(figdir, exist_ok=True) + + # 图1: T_prefill 拟合 vs 实测 + fig, ax = plt.subplots(figsize=(6, 4)) + Ns = np.array([p[0] for p in pts]); Ts = np.array([p[1] for p in pts]) + xx = np.linspace(0, 131072, 200) + ax.scatter(Ns / 1024, Ts / 1e3, c="C3", zorder=3, label="measured recompute") + ax.plot(xx / 1024, t_prefill(xx, a, b) / 1e3, "C0-", + label=f"fit a·N+b·N²\na={a:.4f}ms/tok\nb={b:.2e}ms/tok²") + ax.set_xlabel("context N (K tokens)"); ax.set_ylabel("T_prefill (s)") + ax.set_title("V3.2 prefill compute cost (fit)"); ax.legend(fontsize=8); ax.grid(alpha=.3) + fig.tight_layout(); fig.savefig(f"{figdir}/tprefill_fit.png", dpi=120); plt.close(fig) + + # 图2: B_crit vs h (各 N), 叠加传输带宽水平线 + fig, ax = plt.subplots(figsize=(6, 4)) + byN = {} + for c in cells: + if c["transport"] == list(retrieves)[0]: + byN.setdefault(c["N"], []).append((c["h"], c["bcrit_meas_gbps"])) + for N, hv in sorted(byN.items()): + hv.sort(); ax.plot([x[0] for x in hv], [x[1] for x in hv], + "o-", label=f"N={N//1024}k") + for tname, bw in retrieves.items(): + ax.axhline(bw, ls="--", alpha=.6, label=f"{tname} B_ret={bw:.1f}") + ax.set_yscale("log"); ax.set_xlabel("hit ratio h"); ax.set_ylabel("B_crit (GB/s, log)") + ax.set_title("B_crit vs hit ratio (all << transport BW)") + ax.legend(fontsize=7, ncol=2); ax.grid(alpha=.3, which="both") + fig.tight_layout(); fig.savefig(f"{figdir}/bcrit_vs_hit.png", dpi=120); plt.close(fig) + + # 图3: T_restore TCP vs RDMA (各 N,h=1.0) + fig, ax = plt.subplots(figsize=(6, 4)) + hit1 = [c for c in cells if c["h"] == 1.0] + Nl = sorted({c["N"] for c in hit1}) + xs = range(len(Nl)); w = 0.35 + for i, tname in enumerate(retrieves): + ys = [next(c["t_restore_ms"] for c in hit1 if c["N"] == N and c["transport"] == tname) + for N in Nl] + ax.bar([x + (i - .5) * w for x in xs], ys, w, label=tname) + ax.set_xticks(list(xs)); ax.set_xticklabels([f"{N//1024}k" for N in Nl]) + ax.set_xlabel("context N"); ax.set_ylabel("T_restore (ms), h=1.0") + ax.set_title("Restore latency: TCP vs RDMA"); ax.legend(); ax.grid(alpha=.3, axis="y") + fig.tight_layout(); fig.savefig(f"{figdir}/trestore_tcp_vs_rdma.png", dpi=120); plt.close(fig) + return ["tprefill_fit.png", "bcrit_vs_hit.png", "trestore_tcp_vs_rdma.png"] + + +def selftest(a, b, retrieves): + """闭式解一致性自检: T_cache < T_full <=> B_retrieve > B_crit。""" + N, h = 16384, 0.5 + hN = align(int(round(h * N))); uniq = N - hN + s_kv = s_kv_bytes(hN) + denom = t_prefill(N, a, b) - t_prefill(uniq, a, b) + bcrit = b_crit_gbps(s_kv, denom) + for bw in (bcrit * 0.5, bcrit * 2.0): # 一半 / 两倍临界带宽 + t_restore = (s_kv / GB) / bw * 1e3 + t_cache = t_prefill(uniq, a, b) + t_restore # T_0=lookup=Lnet=0 + t_full = t_prefill(N, a, b) + profitable = t_cache < t_full + assert profitable == (bw > bcrit), \ + f"selftest FAIL bw={bw:.3f} bcrit={bcrit:.3f} cache={t_cache:.0f} full={t_full:.0f}" + print("[selftest] closed-form B_crit <-> T_cache {args.report}", flush=True) + + # 标定误差: 模型 B_crit vs 实测 B_crit + errs = [abs(c["bcrit_model_gbps"] - c["bcrit_meas_gbps"]) / c["bcrit_meas_gbps"] + for c in cells if c["transport"] == list(retrieves)[0] and c["bcrit_meas_gbps"] > 0] + if errs: + print(f"[calib] 模型 vs 实测 B_crit 相对误差: " + f"median={statistics.median(errs)*100:.1f}% max={max(errs)*100:.1f}%", flush=True) + + print("\n[B_crit 实测范围] " + f"{min(c['bcrit_meas_gbps'] for c in cells):.3f} – " + f"{max(c['bcrit_meas_gbps'] for c in cells):.3f} GB/s", flush=True) + + ex = extrapolate(a, b, retrieves, N=131072, h=0.30, online_tcp=args.online_tcp) + print("\n[外推 128K / 30% 命中] (验收① 验证点)") + print(f" hit_tokens={ex['hit_tokens']} S_KV={ex['s_kv_gb']}GB " + f"T_prefill(128k)≈{ex['tprefill_N_ms']/1e3:.1f}s 分母≈{ex['denom_ms']/1e3:.1f}s") + print(f" => 预测 B_crit = {ex['bcrit_gbps']:.4f} GB/s") + for tname in list(retrieves) + ["online_tcp_hypo"]: + print(f" {tname:16} margin={ex[f'margin_{tname}_x']}× " + f"T_restore={ex[f'Trestore_{tname}_ms']/1e3:.2f}s") + + selftest(a, b, retrieves) + + if not args.no_plot: + try: + figs = make_plots(args.figdir, rows, cells, a, b, retrieves, pts) + print(f"[fig] wrote {figs} -> {args.figdir}", flush=True) + except Exception as e: + print(f"[fig] skip plot: {e}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/code/issue5/prefill_hicache.yaml b/src/code/issue5/prefill_hicache.yaml new file mode 100644 index 0000000..f7e9a28 --- /dev/null +++ b/src/code/issue5/prefill_hicache.yaml @@ -0,0 +1,174 @@ +apiVersion: example.com/v1 +kind: FedDeployment +metadata: + labels: + example.com/managed-by: deploytask + example.com/feddeploy-name: issue5-prefill-by-fed-deploy + example.com/inference-service-name: issue5-prefill-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + name: issue5-prefill-hicache + namespace: default +spec: + minReadySeconds: 30 + replicas: 1 + selector: + matchLabels: + example.com/feddeploy-name: issue5-prefill-by-fed-deploy + example.com/inference-service-name: issue5-prefill-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + strategy: + canary: + maxSurge: 100% + maxUnavailable: 100% + steps: + - pause: {} + setWeight: 100 + template: + metadata: + annotations: + example.com/update-by-job: "0" + + labels: + example.com/feddeploy-name: issue5-prefill-by-fed-deploy + example.com/inference-service-name: issue5-prefill-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + spec: + replicas: 1 + template: + metadata: + labels: + example.com/feddeploy-name: issue5-prefill-by-fed-deploy + example.com/inference-service-name: issue5-prefill-by-fed-deploy + example.com/model-name: issue5-model + example.com/platform: TEST + annotations: + scheduling.volcano.sh/queue-name: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - 192.0.2.101 + - 192.0.2.102 + - 192.0.2.103 + - 192.0.2.104 + - 192.0.2.105 + - 192.0.2.106 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: rdma-test + operator: In + values: + - "true" + topologyKey: "kubernetes.io/hostname" + + containers: + # 容器只拉起并 idle, 不在启动时跑 server。 + # 因为里面有需要运行时才能确定的参数(存储节点宿主机 IP 等), + # 实际启动逻辑放在 scripts/start_prefill.sh, 进 pod 后手动执行: + # kubectl exec -it -- bash + # # 把 scripts/start_prefill.sh 拷进 /mnt/cluster 或直接粘贴执行 + # MOONCAKE_STORAGE_NODE_IP=<存储节点IP> MC_PROTOCOL=rdma bash /mnt/cluster/issue5/start_prefill.sh + - args: + - -c + - | + . ~/.bashrc + echo "[issue5-prefill] Pod ready. Exec in and run scripts/start_prefill.sh manually." + tail -f /dev/null + command: + - /bin/sh + # 容器只 idle, 模型实际启动由 scripts/start_prefill_v32.sh 手动执行, + # 该脚本会 export 自己需要的全部 env(甚至覆盖 pod 值), 故此处不再重复注入 + # sglang/MoE/kunlun 调优变量, 仅保留 downward-API pod 元信息, 方便 exec 进 pod 调试。 + env: + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: inference-sglang: + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - | + mkdir -p /run/sglang && \ + echo "stop" > /run/sglang/stop && \ + sleep 120 && \ + pkill -9 -f '(sgl|python3)' + name: model-server + ports: + - containerPort: 6677 + protocol: TCP + - containerPort: 8000 + protocol: TCP + resources: + limits: + kunlunxin.com/xpu: "8" + rdma/hca: "1" + requests: + kunlunxin.com/xpu: "8" + rdma/hca: "1" + securityContext: + capabilities: + add: + - SYS_PTRACE + - SYS_ADMIN + runAsUser: 0 + volumeMounts: + - mountPath: /dev/shm + name: shm-cache + - mountPath: /mnt/cluster + name: pvc-model + dnsPolicy: ClusterFirst + hostNetwork: true + nodeSelector: + example.com/dedicated-pool: + restartPolicy: Always + schedulerName: volcano + securityContext: {} + shareProcessNamespace: true + terminationGracePeriodSeconds: 600 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 32Gi + name: local-cache + - emptyDir: + medium: Memory + sizeLimit: 800Gi + name: shm-cache + - emptyDir: {} + name: shared-volume + - name: pvc-model + persistentVolumeClaim: + claimName: model-pvc + # topologyKey: example.com/dcn-tor + # topologyKeyExclusive: true \ No newline at end of file diff --git a/src/code/issue5/report/figures/bw_vs_concurrency.png b/src/code/issue5/report/figures/bw_vs_concurrency.png new file mode 100644 index 0000000..a99d0f0 Binary files /dev/null and b/src/code/issue5/report/figures/bw_vs_concurrency.png differ diff --git a/src/code/issue5/report/figures/bw_vs_size.png b/src/code/issue5/report/figures/bw_vs_size.png new file mode 100644 index 0000000..6ab2b14 Binary files /dev/null and b/src/code/issue5/report/figures/bw_vs_size.png differ diff --git a/src/code/issue5/report/figures/l2_zerocopy_bw.png b/src/code/issue5/report/figures/l2_zerocopy_bw.png new file mode 100644 index 0000000..71eecc7 Binary files /dev/null and b/src/code/issue5/report/figures/l2_zerocopy_bw.png differ diff --git a/src/code/issue5/report/figures/protocol_summary.png b/src/code/issue5/report/figures/protocol_summary.png new file mode 100644 index 0000000..88371d1 Binary files /dev/null and b/src/code/issue5/report/figures/protocol_summary.png differ diff --git a/src/code/issue5/report/figures/qpm_compare.png b/src/code/issue5/report/figures/qpm_compare.png new file mode 100644 index 0000000..fc6f01e Binary files /dev/null and b/src/code/issue5/report/figures/qpm_compare.png differ diff --git a/src/code/issue5/report/figures/ttft_vs_load.png b/src/code/issue5/report/figures/ttft_vs_load.png new file mode 100644 index 0000000..acbf5d8 Binary files /dev/null and b/src/code/issue5/report/figures/ttft_vs_load.png differ diff --git a/src/code/issue5/report/report.md b/src/code/issue5/report/report.md new file mode 100644 index 0000000..e3ad10d --- /dev/null +++ b/src/code/issue5/report/report.md @@ -0,0 +1,537 @@ +# issue5 report + +1. 带宽临界值模型 + +输入：命中率，上下文长度，KV Cache 量，算力 + +输出：以存代算的临界带宽 + +2. 相同 TTFT 约束下 + +TCP 条件下， 带宽，延迟，可承载并发query数，QPM + +RDMA 条件下，带宽，延迟，可承载并发query数，QPM + +## 基本环境与性能 + +### 硬件环境与拓扑 + +硬件设备 + +| 项 | kvstore 192.0.2.10 | prefill 192.0.2.20 | +| :-------------- | :------------------------------------- | :---------------------- | +| HCA | mlx5_0..5（6 个） | mlx5_0..6（7 个） | +| link_layer | Ethernet（RoCEv2） | Ethernet（RoCEv2） | +| 端口状态 | eth0/eth1-4 PORT_ACTIVE | eth0/eth1-4 PORT_ACTIVE | +| /dev/infiniband | uverbs0-5 + rdma_cm | uverbs0-6 + rdma_cm | +| kmods | mlx5_ib/ib_core/rdma_cm/kunlun_peermem | 同 | +| ulimit -l | unlimited | unlimited | +| perftest | ib_write_bw/ib_read_bw/ib_send_lat | 同 | + +硬件拓扑 + +| netdev | kvstore | prefill | +| :------------------------------ | :------------- | :-------------------- | +| eth0（前端 192.0.2.x，管理/OOB） | mlx5_0 | mlx5_0 | +| eth1（RDMA，198.51.100.x/27） | mlx5_2 | mlx5_3 | +| eth2 | mlx5_3 | mlx5_4 | +| eth3 | mlx5_4 | mlx5_5 | +| eth4 | mlx5_5 | mlx5_6 | +| eth5/eth6 | mlx5_1（DOWN） | mlx5_1/mlx5_2（DOWN） | + +- conda env `python310_torch29_cuda`，预装 mooncake **0.3.5**（勿 pip 装，见 MEMORY）。 +- store 端口：master RPC `50051`、http metadata `18080`、store `18081`。 +- client（prefill）用 `global_segment_size=0` 不贡献内存 → put/get 全部真跨机落到远端 store 段。 + +- RDMA 数据 NIC = **eth1-eth4**（与 `start_prefill*.sh` 的 `BKCL_RDMA_NICS="eth1,eth1,eth2,eth2,eth3,eth3,eth4,eth4"` 一致）。 +- 同名 netdev 在两机对应的 mlx5 序号不同 → 配置里若按 `mlx5_N` 指定要分别写；按 `ethN` 指定更稳。 +- RDMA NIC 子网：kvstore `198.51.100.0/27`，prefill `198.51.100.32/27`（不同 /27，RoCEv2 可路由）。 + +### 基本性能 + +#### 设备性能 + +RDMA 本机数据路径已实测 146 Gb/s； + +跨机 RDMA 需走 Mooncake 自带传输引擎（握手端口已放通、已发现 6 HCA），独立 perftest 的跨机 OOB 被安全组挡住， 不影响项目，真正的跨机 RDMA 数据面测试在 L1 用 `MC_PROTOCOL=rdma` 完成 + +65536B -> BW average 146.73 Gb/sec (~18.3 GB/s) + +#### Mooncake put/get primitive microbench + +这里使用 零拷贝 `batch_get_into`，镜像 hicache 真实 retrieve 路径，一次 `register_buffer(整块 read 缓冲)`， 一批 `batch_get_into(keys, ptrs, sizes)` 直 DMA 进预注册张量（无 python bytes 中间对象），否则 RDMA get 平台化在 ~1.5–2.1 GB/s，远低于同协议 put 的 16 GB/s，瓶颈卡在 client 侧 `bytes` 分配 + memcpy + GIL + +PUT（写入远端 store，对应 write_through KV 落盘） + +| size | conc | TCP-zc | RDMA-zc | RDMA-zc / TCP-zc | +|---|---|---|---|---| +| 64MB | 1 | 0.58 | **19.65** | 33.9× | +| 64MB | 2 | 0.62 | **20.80** | 33.5× | +| 64MB | 4 | 0.66 | **22.07** | 33.4× | +| 64MB | 8 | 0.73 | **21.86** | 29.9× | +| 256MB | 1 | 0.65 | **18.38** | 28.3× | +| 256MB | 2 | 0.77 | **20.15** | 26.2× | +| 256MB | 4 | 1.22 | **20.72** | 17.0× | +| 256MB | 8 | 1.17 | **21.14** | 18.1× | +| 1GB | 1 | 1.21 | **18.07** | 14.9× | +| 1GB | 2 | 1.23 | **19.77** | 16.1× | +| 1GB | 4 | 1.06 | **21.09** | 19.9× | + +GET（读回远端 store，对应命中时的 KV retrieve） + +| size | conc | TCP-zc | RDMA-zc | RDMA-zc / TCP-zc | +| :---- | :--- | :----- | :-------- | :--------------- | +| 64MB | 1 | 1.20 | **10.20** | 8.5× | +| 64MB | 2 | 1.22 | **8.93** | 7.3× | +| 64MB | 4 | 1.23 | **10.48** | 8.5× | +| 64MB | 8 | 1.21 | **13.39** | 11.1× | +| 256MB | 1 | 1.19 | **10.43** | 8.8× | +| 256MB | 2 | 0.93 | **12.78** | 13.7× | +| 256MB | 4 | 1.44 | **13.20** | 9.2× | +| 256MB | 8 | 1.49 | **12.03** | 8.1× | +| 1GB | 1 | 1.31 | **13.92** | 10.6× | +| 1GB | 2 | 1.44 | **14.47** | 10.0× | +| 1GB | 4 | 1.58 | **14.61** | 9.2× | +| 1GB | 8 | 1.23 | **14.69** | 11.9× | + + + +## 验收1 + +临界值带宽模型 + +输入：命中率，上下文长度，KV Cache 量，算力 + +输出：以存代算的临界带宽 + +### 模型推导 + +全量重算路径 +$$ +T_{\text{full}}(N)=T_{\text{prefill}}(N) +$$ + +- $N$：完整输入的 token 数； +- $T_{\text{prefill}}(N)$：从头计算这 $N$ 个 token 的 prefill 时间。 + +以存代算路径 +$$ +T_{\text{cache}}(N,h,B) += +T_0 ++ +T_{\text{retrieve}} ++ +T_{\text{restore}}(hN) ++ +T_{\text{prefill}}((1-h)N) +$$ + +- $h$：前缀命中率。 + +**固定开销 $T_0$** ：$T_0=T_{\text{lookup}}+L_{\text{net}}$，如查询 metadata，判断前缀是否命中，发起 RPC，建立或调度传输请求，网络往返固定延迟等 + +**传输时间 $T_{\text{retrieve}}$**：$T_{\text{retrieve}}=\frac{S_{\text{KV}}(hN)}{B}$ + +- $S_{\text{KV}}(hN)$：命中前缀对应的 KV 字节数； +- $B$：实际 retrieve 有效带宽。 + +**恢复开销 $T_{\text{restore}}$**：数据通过网络到达后，不一定能立刻被模型使用，可能还需要 反序列化，从临时缓冲区复制到 KV pool，恢复 page table 或 slot 映射，layout 转换 等，因此把 $T_{\text{retrieve}}$ 和 $T_{\text{restore}}(hN)$ 区分开 + +**后缀现算$T_{\text{prefill}}$**： $T_{\text{prefill}}((1-h)N)$ 命中的前缀不再计算，但未命中的后缀仍要正常跑模型 + + + +对于值得使用远端 KV 的情况，令 +$$ +T_{\text{cache}}B_{\text{crit}}$ 则以存代算更快 +- $B=B_{\text{crit}}$ 刚好打平 +- $B0.85 MoE OOM / <0.83 权重 OOM，窗口极窄），每卡只剩 ~15GB 给 KV ⇒ 全局 KV 上限 **29120 token**，32k / 128k 单请求超 KV 容量：实测发一个 32768-token prefill 直接把 scheduler 挂死（detokenizer 20s 无响应、health check 连续失败、进程 wedge，需 kill 重起）。故 sweep 的上下文改为 **8k / 16k / 24k**（均 < 29120，24k 留 ~4.5k 余量）。因此本实验只能稳定实测 8k、16k 和 24k 上下文； + +32k 与 128k 这些长上下文的 KV 应从**远端 store retrieve** 回来（用 L2 的 `B_retrieve` 代入 §6 的 `B_crit` 外推），但远端 KV 存储本身只避免前缀重算，并不会自动突破本地活跃 KV 容量上限；真正支持超长上下文还需分层按需加载、KV 分片或更大本地 KV 池。 + + + +命中率 h 用「共享前缀 + 唯一后缀」控制：`prefix = align(round(h*N), page=64)`，`suffix = N-prefix`。 + +- **recompute 基线 `T_prefill(N)`**：每 iter `flush` + 唯一内容 → 真全量现算，取 median。 +- **命中 total(N, h)**：先发 `prefix` 预热进本地 radix（**不 flush**），再发 `prefix+唯一后缀` + → `cached_tokens=prefix`，前缀 KV device 本地复用（restore≈0），只现算后缀。 +- **`saved = T_prefill(N) − total(N,h)`** = 复用 hN 前缀省下的**计算**时间。 +- **JIT 毛刺**：`--disable-cuda-graph` 下每个 shape 首次请求有编译开销（首个 8k ~9.6s， + 稳态 ~3.7s），故每 shape 先 `warmup` 丢弃再取 median。 + + + +DeepSeek V3.2 MLA, S_token=68.6 KiB/token, TP8, page=64, 本地 radix + +`total`/`recompute`/`saved` 单位 ms（median, iters=3） + +| N | h | prefix | cached | recompute_ms | total_ms | **saved_ms** | +|---|---|---|---|---|---|---| +| 8k | 0.0 | 0 | 0 | 3680 | 3661 | 19 | +| 8k | 0.25 | 2048 | 2048 | 3654 | 3296 | 358 | +| 8k | 0.5 | 4096 | 4096 | 3638 | 2242 | 1397 | +| 8k | 0.75 | 6144 | 6144 | 3649 | 1196 | 2453 | +| 8k | 1.0 | 8192 | 8128 | 3659 | 381 | 3278 | +| 16k | 0.0 | 0 | 0 | 8672 | 8673 | 0 | +| 16k | 0.25 | 4096 | 4096 | 8643 | 7220 | 1423 | +| 16k | 0.5 | 8192 | 8192 | 8655 | 5039 | 3616 | +| 16k | 0.75 | 12288 | 12288 | 8917 | 2629 | 6288 | +| 16k | 1.0 | 16384 | 16320 | 8638 | 385 | 8253 | +| 24k | 0.0 | 0 | 0 | 14394 | 14341 | 53 | +| 24k | 0.25 | 6144 | 6144 | 14380 | 11830 | 2549 | +| 24k | 0.5 | 12288 | 12288 | 14429 | 8304 | 6124 | +| 24k | 0.75 | 18432 | 18432 | 14315 | 4467 | 9848 | +| 24k | 1.0 | 24512 | 24512 | 14410 | 405 | 14005 | + +- `saved_ms` 随 h 近线性增长，随 N 更陡（长上下文重算越贵，复用价值越大）。 +- h=1.0 时 `total≈380–405ms`（几乎纯命中，只剩最后一页现算 + 调度开销），`saved` 达全量的 ~96%。 +- `cached_tokens` 恒等于 page 对齐前缀（h=1.0 时 = N−64，末页留一个未命中 token 触发前向）。 + + + +理想算力侧临界带宽 +$$ +B_{\text{crit}}^{\text{ideal}} += +\frac{S_{\text{KV}}(hN)} +{T_{\text{prefill}}(N)-T_{\text{prefill}}((1-h)N)} +$$ + +- $N$：请求总 token 数； +- $h$：前缀命中率； +- $hN$：命中的 token 数； +- $S_{\text{KV}}(hN)$：这些命中 token 对应的 KV 数据量； +- $T_{\text{prefill}}(N)$：完整重算 N 个 token 的时间； +- $T_{\text{prefill}}((1-h)N)$：只计算未命中后缀的时间； +- 两者之差：复用前缀 KV 所节省的计算时间。 + +分母用 `recompute_ms − cold_uniq_ms`，即前缀对应的现算时间；本地命中 restore≈0，此项即纯算侧收益，$T_{restore}$ 把命中的 KV cache 从远端 store 搬回本地所需要的时间, `T_restore@RDMA 20` 即使用 RDMA，有效带宽为 20GB/s + +| N | h | cached KV | 前缀现算(=分母) | **B_crit** | T_restore@RDMA 20 | T_restore@TCP 1 | +|---|---|---|---|---|---|---| +| 8k | 0.5 | 0.268 GB | 2236 ms | **0.120 GB/s** | 13 ms | 268 ms | +| 8k | 1.0 | 0.532 GB | 3659 ms | **0.145 GB/s** | 27 ms | 532 ms | +| 16k | 0.5 | 0.536 GB | 5029 ms | **0.107 GB/s** | 27 ms | 536 ms | +| 16k | 1.0 | 1.068 GB | 8638 ms | **0.124 GB/s** | 53 ms | 1068 ms | +| 24k | 0.75 | 1.206 GB | 11830 ms | **0.102 GB/s** | 60 ms | 1206 ms | +| 24k | 1.0 | 1.604 GB | 14410 ms | **0.111 GB/s** | 80 ms | 1604 ms | + +1. **DeepSeek V3.2 的 `B_crit` 极低（~0.09–0.15 GB/s）** + + 根因是 MLA 单 latent KV 极省（68.6 KiB/token）而 671B MoE 的 prefill 算力极贵（~0.45–0.6 ms/token），每传 1 字节能省下的算力时间极大，打平所需带宽很低 + +2. **TCP（~1 GB/s）与 RDMA（~20 GB/s）都远高于 `B_crit`（分别高 ~8× / ~180×）**，即 + 以存代算对 V3.2 在两种传输下都稳赚 + + RDMA 的 restore 延迟：`T_restore` 从 TCP 的 268–1604 ms 降到 RDMA 的 13–80 ms（~16–20×），直接改善命中请求的TTFT（验收2 的 QPM/TTFT 差距来源） + +3. 分母（算力收益）巨大使 `B_crit` 很低，retrieve 带宽越高，即可让命中率还比较低时，远端 KV retrieve 的总收益就已经大于传输成本，开始比全量重算更快 + +### 模型实测标定 + +通过模型推导得到： +$$ +B_{\text{crit}}(N,h) += +\frac{S_{\text{KV}}(hN)} +{ +T_{\text{prefill}}(N) +- +T_{\text{prefill}}((1-h)N) +- +T_{\text{lookup}} +- +T_{\text{restore}} +- +L_{\text{net}} +} +$$ + +- **分子** `S_KV(hN)` = 命中前缀要 retrieve 的 KV 字节。V3.2 MLA 单 latent： `S_token = L·(kv_lora_rank+qk_rope)·b = 61·(512+64)·2 = 70272 B = 68.6 KiB/token`。 +- **分母** = 复用 hN 前缀省下的算力时间（全量减去只剩的后缀现算），再扣掉查表/反序列化/网络固定开销。 +- 分母 ≤ 0 永不划算；固定开销太大，本地命中时 `T_lookup≈T_restore≈L_net≈0`，分母即纯算侧收益 + + + +**prefill end to end microbench** 标定算力侧 `T_prefill(N)` + +prefill_local.csv 每个 N 的全量重算中位数做标定点， 最小二乘拟合过原点的 `T_prefill(N) = a·N + b·N²`（`b·N²` 吸收 attention 的超线性）： + +需要把标定点: $8k=3654ms$, $16k=8655ms$, $24k=14394ms$ 三个离散点拟合成一个连续函数： +$$ +T_{\text{prefill}}(N) += +0.39318N+7.886\times10^{-6}N^2 +\quad\text{ms} +$$ + +- `a≈0.393 ms/tok` 线性主项（FFN/MoE + NSA 稀疏 attention 的近线性部分）， `b≈7.9e-6 ms/tok²` 是残余二次项。R²=0.9997 + +- **外推警告**：V3.2 用 NSA 稀疏（`index_topk=2048`），attention 在 N≫2048 后应趋近**线性** （每 query 只算 2048 个 key），故 `b·N²` 会高估 128k 的算力。 + + 所以 128k 外推的 `T_prefill` 是区间上界，对应 `B_crit` 是下界（真实 B_crit 只会更高一点点，结论不变）。 + + + +**KV Tensor Transfer microbench** 标定 retrieve 带宽 `B_retrieve` + +`T_restore = S_KV / B_retrieve`。B_retrieve 是传输层带宽（GB/s），与模型无关，直接取kv_tensor_v32_{tcp,rdma}.csv 的 zerocopy `batch_get_into` get 中位： + +| 传输 | B_retrieve (zerocopy get) | 备注 | +| :------- | :------------------------ | :------------------------------------- | +| **TCP** | **1.19 GB/s** | 卡在内核 socket 拷贝，零拷贝几乎无增益 | +| **RDMA** | **20.51 GB/s** | 零拷贝直 DMA 进目标张量，~17.2× TCP | + + + +结果：每 (N,h) 的 B_crit + 是否划算 + +results/model_fit.csv（24 行 = 12 个 h>0 cell × {tcp,rdma}） + +`bcrit_meas` 用实测分母，`bcrit_model` 用拟合 `T_prefill`。全部 `profitable=1` + +| N | h | cached | S_KV | 分母(实测) | **B_crit(实测)** | B_crit(模型) | T_restore@TCP | T_restore@RDMA | +| :--- | :--- | :----- | :------- | :--------- | :--------------- | :----------- | :------------ | :------------- | +| 8k | 0.5 | 4096 | 0.268 GB | 2236 ms | **0.120 GB/s** | 0.134 | 225 ms | 13 ms | +| 8k | 1.0 | 8128 | 0.532 GB | 3659 ms | **0.145 GB/s** | 0.142 | 446 ms | 26 ms | +| 16k | 0.5 | 8192 | 0.536 GB | 5029 ms | **0.107 GB/s** | 0.112 | 450 ms | 26 ms | +| 16k | 1.0 | 16320 | 1.068 GB | 8638 ms | **0.124 GB/s** | 0.125 | 896 ms | 52 ms | +| 24k | 0.75 | 18432 | 1.206 GB | 11830 ms | **0.102 GB/s** | 0.103 | 1012 ms | 59 ms | +| 24k | 1.0 | 24512 | 1.604 GB | 14410 ms | **0.111 GB/s** | 0.111 | 1346 ms | | + +**标定误差（模型 B_crit vs 实测 B_crit）**：`median=2.6%, max=13.0%`。 最大误差落在 8k（短序列 JIT/调度固定开销占比大，拟合的纯二次模型没显式建模 T_0，把开销摊进 斜率 → 短序列略偏）；16k/24k 误差 <5% + +**B_crit 实测范围：0.092 – 0.147 GB/s** + + + +外推线上：128K / 30% 命中 + +用拟合 `T_prefill` 外推（128k 单节点放不下，故只能靠模型）： + +理论命中 token 数，page size 是 64，对齐到 64 整数倍： +$$ +\text{hN} = \text{align}(0.3\times131072)=39296 +$$ +MLA 每 token、每 TP rank 的 KV 大小是： +$$ +S_{\text{token}}=70272\text{ B/token} +$$ +所以命中 39296 token 对应的 KV 大小为： +$$ +S_{\text{KV}} += +39296\times70272 += +2.76\times10^9\text{ B} +\approx +2.572\text{ GiB} +$$ +前面 N 代入拟合公式得到： +$$ +T_{\text{prefill}}(N) += +0.39318N+7.886\times10^{-6}N^2 +\quad\text{ms} +$$ +得到：$T_{prefill}(128k)\approx187.0$ s， $T_{prefill}(70\% \approx 91776)\approx102.5$ + +得到 +$$ +\text{B\_crit} = 2.572 \text{GB} / 84.5\text{s} = 0.0304 \text{GB/s} +$$ + + + +| retrieve 带宽 | margin (B/B_crit) | T_restore | +| :-------------- | :---------------- | :-------- | +| RDMA 20.51 GB/s | 674× | 0.12 s | +| TCP 1.19 GB/s | 39× | 2.16 s | + + + + +结论 + +1. **V3.2 的 `B_crit` 极低（实测 0.09–0.15 GB/s，128k/30% 外推 0.03 GB/s）** + + 根因：MLA 单 latent KV 极省（68.6 KiB/token），而 671B MoE prefill 算力极贵（~0.45–0.6 ms/token）， 每传 1 字节能省的算力时间极大。 + +2. **TCP（~1.19）与 RDMA（~20.51 GB/s）都远高于 B_crit**（分别 ~8–39× / ~140–674×） + + 以存代算 对 V3.2 在两种传输下都稳赚，且在极低命中率下就转正。 + +3. **RDMA 的价值在 restore 延迟/TTFT**：`T_restore` 从 TCP 的 0.23–2.16 s 降到 RDMA 的 0.01–0.12 s（~17×），直接改善命中请求的 TTFT + +4. `selftest` 验证闭式解 `T_cacheB_crit`；模型 vs 实测 B_crit 标定误差 median 2.6%。 + + + +## 验收2 + +相同 TTFT 约束下 + +TCP 条件下， 带宽，延迟，可承载并发query数，QPM + +RDMA 条件下，带宽，延迟，可承载并发query数，QPM + + + +**相同 TTFT SLO** 下量出 no-cache / TCP / RDMA 三者的 **λ_max、QPM、成本/query**，给出 TCP→RDMA 的 QPM/成本对比 + +端到端命路径有 kernel bug，使用离散事件重放：每个逻辑请求的服务时间 由前几个 microbench 的实测数值合成 +$$ +T_{\text{hit}} += +T_{\text{lookup}} ++ +\frac{S_{\text{KV}}(hN)}{B_{\text{retrieve}}} ++ +T_{\text{prefill}}((1-h)N) +$$ + +- `T_prefill(N) = 0.39318·N + 7.886e-6·N²` ms +- `B_retrieve`：L2 zerocopy `batch_get_into` 中位，TCP 1.19 / RDMA 20.51 GB/s +- `T_lookup = 5 ms` 固定 + +**排队模型**：`c` 个 prefill worker 的 FIFO 队列（GPU 算力受限，默认 `c=1` 即算力串行）， Poisson 到达（率 λ req/s）。扫 λ 直到 P95 TTFT > SLO 得到 `λ_max`； + +计算 `QPM = 60·λ_max`； `Cost/Query` 正比于 `C_node / QPM`。 + +三种传输（no-cache/TCP/RDMA）在同一 context 下**共用同一 SLO** （`SLO = T_prefill(N) × slo_factor`，`slo_factor=2.5`，由 no-cache 单请求服务定，与传输无关） + + + +phit 命中概率，h 命中前缀比例 + +高命中（h=0.95, phit=0.8），retrieve 主导，RDMA 显著 + +| Context | 方案 | KV retrieve 带宽 | 命中请求 restore 延迟 | 平均服务时间 | P95 TTFT SLO | 最大可持续到达率 λ_max | QPM | QPM vs no-cache | RDMA/TCP QPM | +| ------- | -------- | ---------------- | --------------------- | ------------ | ------------ | ---------------------- | ----- | --------------- | ------------ | +| 8k | no-cache | — | — | 3.750 s | 9.375 s | 0.080 req/s | 4.80 | 1.00× | — | +| 8k | TCP | 1.19 GB/s | ≈0.43 s | 1.236 s | 9.375 s | 0.404 req/s | 24.26 | 5.05× | 1.00× | +| 8k | RDMA | 20.51 GB/s | ≈0.025 s | 0.916 s | 9.375 s | 0.546 req/s | 32.75 | 6.82× | **1.35×** | +| 16k | no-cache | — | — | 8.559 s | 21.397 s | 0.035 req/s | 2.10 | 1.00× | — | +| 16k | TCP | 1.19 GB/s | ≈0.86 s | 2.665 s | 21.397 s | 0.188 req/s | 11.26 | 5.36× | 1.00× | +| 16k | RDMA | 20.51 GB/s | ≈0.050 s | 2.022 s | 21.397 s | 0.247 req/s | 14.84 | 7.07× | **1.32×** | +| 24k | no-cache | — | — | 14.426 s | 36.065 s | 0.021 req/s | 1.25 | 1.00× | — | +| 24k | TCP | 1.19 GB/s | ≈1.28 s | 4.325 s | 36.065 s | 0.116 req/s | 6.94 | 5.55× | 1.00× | +| 24k | RDMA | 20.51 GB/s | ≈0.075 s | 3.362 s | 36.065 s | 0.149 req/s | 8.92 | 7.14× | **1.29×** | + +- 高命中下缓存把 QPM 提升 **~5–7×**、成本降到 **~0.14–0.20×**。 +- **此时 RDMA 比 TCP 高 ~1.29–1.35× QPM**（成本再降 ~22–26%）。因为 h=0.95 时后缀现算≈0， 服务时间由 **retrieve 主导**：TCP restore（8k=0.43s/16k=0.86s/24k=1.28s 摊进平均）明显拖慢， RDMA（0.02–0.07s）几乎不占时间 ⇒ 直接转成更高 λ_max。 + + + +中等命中（h=0.5, phit=0.5），算力主导，RDMA≈TCP + +| Context | 方案 | KV retrieve 带宽 | 命中请求 restore 延迟 | 平均服务时间 | P95 TTFT SLO | 最大可持续到达率 λ_max | QPM | QPM vs no-cache | RDMA/TCP QPM | +| ------- | -------- | ---------------- | --------------------- | ------------ | ------------ | ---------------------- | ---- | --------------- | ------------ | +| 8k | no-cache | — | — | 3.750 s | 9.375 s | 0.080 req/s | 4.80 | 1.00× | — | +| 8k | TCP | 1.19 GB/s | ≈0.225 s | 2.861 s | 9.375 s | 0.140 req/s | 8.39 | 1.75× | 1.00× | +| 8k | RDMA | 20.51 GB/s | ≈0.013 s | 2.755 s | 9.375 s | 0.145 req/s | 8.71 | 1.81× | **1.04×** | +| 16k | no-cache | — | — | 8.559 s | 21.397 s | 0.035 req/s | 2.10 | 1.00× | — | +| 16k | TCP | 1.19 GB/s | ≈0.450 s | 6.382 s | 21.397 s | 0.063 req/s | 3.76 | 1.79× | 1.00× | +| 16k | RDMA | 20.51 GB/s | ≈0.026 s | 6.170 s | 21.397 s | 0.065 req/s | 3.89 | 1.85× | **1.03×** | +| 24k | no-cache | — | — | 14.426 s | 36.065 s | 0.021 req/s | 1.25 | 1.00× | — | +| 24k | TCP | 1.19 GB/s | ≈0.675 s | 10.564 s | 36.065 s | 0.038 req/s | 2.27 | 1.82× | 1.00× | +| 24k | RDMA | 20.51 GB/s | ≈0.039 s | 10.246 s | 36.065 s | 0.039 req/s | 2.34 | 1.87× | **1.03×** | + + +- **以存代算（缓存）本身把 QPM 提升 ~1.8×、成本降到 ~0.55×**（相对全量重算），因为命中请求少算了 hN 前缀。 +- **但 RDMA 仅比 TCP 高 ~1.03–1.04×**：在 8k–24k、h=0.5 下，retrieve 时间（TCP 225–675 ms） 相对 prefill 算力（数秒~十几秒）占比很小，**服务时间被算力主导**，换 RDMA 省下的那点 retrieve 时间对 λ_max 影响甚微。⇒ **中短上下文 + 中等命中，RDMA 的收益不明显**。 + + + +结论： + +由于 DeepSeek V3.2 特性，以存代算在两种传输下都提升吞吐/降成本：中等命中 QPM ×1.8（成本 0.55×）， 高命中 QPM ×5–7（成本 0.14–0.20×）。 + +RDMA vs TCP 的增益强依赖 retrieve 占服务时间的比重： + +- 算力主导区（中短上下文 / 中等命中）：retrieve 占比小，RDMA≈TCP（~1.03–1.04×） +- retrieve 主导区（高命中 / 长上下文）：RDMA 比 TCP 高 ~1.29–1.35× QPM，成本再降 ~1/4。 + +进一步外推（长上下文）RDMA 优势更大：h=1.0、128k 时 restore 是唯一变量， TCP restore ≈ 8.6GB/1.19 ≈ 7.2 s、RDMA ≈ 0.42 s（~17×），命中请求 TTFT 差距直接放大到 ~17× （128k 单节点放不下、只能靠 §模型外推，见步骤4 §2 / 步骤5 §2 的外推上界说明）。 \ No newline at end of file diff --git a/src/code/issue5/results/model_fit.csv b/src/code/issue5/results/model_fit.csv new file mode 100644 index 0000000..a89928f --- /dev/null +++ b/src/code/issue5/results/model_fit.csv @@ -0,0 +1,25 @@ +N,h,cached,s_kv_gb,tprefill_N_ms,denom_meas_ms,denom_model_ms,bcrit_meas_gbps,bcrit_model_gbps,transport,b_retrieve_gbps,t_restore_ms,net_saved_ms,profitable,measured_saved_ms +8192,0.25,2048,0.134,3750.1,911.1,1036.8,0.1471,0.1293,tcp_zc,1.192,112.4,798.7,1,357.56 +8192,0.25,2048,0.134,3750.1,911.1,1036.8,0.1471,0.1293,rdma_zc,20.508,6.5,904.6,1,357.56 +8192,0.5,4096,0.2681,3750.1,2236.2,2007.4,0.1199,0.1335,tcp_zc,1.192,224.9,2011.3,1,1396.53 +8192,0.5,4096,0.2681,3750.1,2236.2,2007.4,0.1199,0.1335,rdma_zc,20.508,13.1,2223.1,1,1396.53 +8192,0.75,6144,0.4021,3750.1,3291.1,2911.8,0.1222,0.1381,tcp_zc,1.192,337.3,2953.8,1,2452.81 +8192,0.75,6144,0.4021,3750.1,3291.1,2911.8,0.1222,0.1381,rdma_zc,20.508,19.6,3271.5,1,2452.81 +8192,1.0,8128,0.5319,3750.1,3659.4,3750.1,0.1454,0.1418,tcp_zc,1.192,446.3,3213.2,1,3278.28 +8192,1.0,8128,0.5319,3750.1,3659.4,3750.1,0.1454,0.1418,rdma_zc,20.508,25.9,3633.5,1,3278.28 +16384,0.25,4096,0.2681,8558.7,2540.4,2536.6,0.1055,0.1057,tcp_zc,1.192,224.9,2315.5,1,1422.53 +16384,0.25,4096,0.2681,8558.7,2540.4,2536.6,0.1055,0.1057,rdma_zc,20.508,13.1,2527.3,1,1422.53 +16384,0.5,8192,0.5361,8558.7,5028.6,4808.6,0.1066,0.1115,tcp_zc,1.192,449.8,4578.8,1,3615.6 +16384,0.5,8192,0.5361,8558.7,5028.6,4808.6,0.1066,0.1115,rdma_zc,20.508,26.1,5002.4,1,3615.6 +16384,0.75,12288,0.8042,8558.7,7530.1,6816.0,0.1068,0.118,tcp_zc,1.192,674.7,6855.4,1,6288.46 +16384,0.75,12288,0.8042,8558.7,7530.1,6816.0,0.1068,0.118,rdma_zc,20.508,39.2,7490.9,1,6288.46 +16384,1.0,16320,1.0681,8558.7,8638.4,8558.7,0.1236,0.1248,tcp_zc,1.192,896.0,7742.4,1,8253.2 +16384,1.0,16320,1.0681,8558.7,8638.4,8558.7,0.1236,0.1248,rdma_zc,20.508,52.1,8586.3,1,8253.2 +24576,0.25,6144,0.4021,14425.8,4375.1,4499.5,0.0919,0.0894,tcp_zc,1.192,337.3,4037.8,1,2549.47 +24576,0.25,6144,0.4021,14425.8,4375.1,4499.5,0.0919,0.0894,rdma_zc,20.508,19.6,4355.5,1,2549.47 +24576,0.5,12288,0.8042,14425.8,8354.4,8403.7,0.0963,0.0957,tcp_zc,1.192,674.7,7679.7,1,6124.49 +24576,0.5,12288,0.8042,14425.8,8354.4,8403.7,0.0963,0.0957,rdma_zc,20.508,39.2,8315.1,1,6124.49 +24576,0.75,18432,1.2063,14425.8,11830.3,11712.5,0.102,0.103,tcp_zc,1.192,1012.0,10818.3,1,9848.02 +24576,0.75,18432,1.2063,14425.8,11830.3,11712.5,0.102,0.103,rdma_zc,20.508,58.8,11771.5,1,9848.02 +24576,1.0,24512,1.6042,14425.8,14409.5,14425.8,0.1113,0.1112,tcp_zc,1.192,1345.8,13063.7,1,14004.68 +24576,1.0,24512,1.6042,14425.8,14409.5,14425.8,0.1113,0.1112,rdma_zc,20.508,78.2,14331.3,1,14004.68 diff --git a/src/code/issue5/results/raw/conc_hihit_summary.csv b/src/code/issue5/results/raw/conc_hihit_summary.csv new file mode 100644 index 0000000..e402faf --- /dev/null +++ b/src/code/issue5/results/raw/conc_hihit_summary.csv @@ -0,0 +1,10 @@ +context_length,transport,mean_service_ms,slo_ms,lambda_max_rps,qpm +8192,nocache,3750.1,9375.3,0.08,4.8 +8192,tcp,1236.4,9375.3,0.4044,24.26 +8192,rdma,916.0,9375.3,0.5459,32.75 +16384,nocache,8558.7,21396.9,0.0351,2.1 +16384,tcp,2664.9,21396.9,0.1876,11.26 +16384,rdma,2021.5,21396.9,0.2473,14.84 +24576,nocache,14425.8,36064.6,0.0208,1.25 +24576,tcp,4325.4,36064.6,0.1156,6.94 +24576,rdma,3361.6,36064.6,0.1487,8.92 diff --git a/src/code/issue5/results/raw/conc_hihit_sweep.csv b/src/code/issue5/results/raw/conc_hihit_sweep.csv new file mode 100644 index 0000000..dd5ccf2 --- /dev/null +++ b/src/code/issue5/results/raw/conc_hihit_sweep.csv @@ -0,0 +1,49 @@ +transport,context_length,hit_prefix,phit,workers,lambda_rps,p50_ttft_ms,p95_ttft_ms,p99_ttft_ms,slo_ms,within_slo +nocache,8192,0.95,0.0,1,0.0267,3750.1,5905.9,7345.3,9375.3,1 +nocache,8192,0.95,0.0,1,0.0533,3750.1,7017.5,9420.9,9375.3,1 +nocache,8192,0.95,0.0,1,0.08,3750.1,7962.4,11567.0,9375.3,1 +nocache,8192,0.95,0.0,1,0.1067,3750.1,9766.5,14300.2,9375.3,0 +tcp,8192,0.95,0.8,1,0.0809,607.9,3750.1,5594.3,9375.3,1 +tcp,8192,0.95,0.8,1,0.1618,607.9,4089.9,6716.1,9375.3,1 +tcp,8192,0.95,0.8,1,0.2426,607.9,5108.5,7447.4,9375.3,1 +tcp,8192,0.95,0.8,1,0.3235,738.5,6538.6,8895.2,9375.3,1 +tcp,8192,0.95,0.8,1,0.4044,1149.7,7593.4,12874.3,9375.3,1 +tcp,8192,0.95,0.8,1,0.4853,1954.7,9676.1,18999.1,9375.3,0 +rdma,8192,0.95,0.8,1,0.1092,207.4,3750.1,5996.4,9375.3,1 +rdma,8192,0.95,0.8,1,0.2183,207.4,3920.1,6877.7,9375.3,1 +rdma,8192,0.95,0.8,1,0.3275,207.4,5657.3,7601.2,9375.3,1 +rdma,8192,0.95,0.8,1,0.4367,266.2,6972.7,10558.3,9375.3,1 +rdma,8192,0.95,0.8,1,0.5459,1114.5,8454.8,16237.8,9375.3,1 +rdma,8192,0.95,0.8,1,0.655,2297.7,11247.7,21308.4,9375.3,0 +nocache,16384,0.95,0.0,1,0.0117,8558.7,13478.8,16763.9,21396.9,1 +nocache,16384,0.95,0.0,1,0.0234,8558.7,16015.8,21501.0,21396.9,1 +nocache,16384,0.95,0.0,1,0.0351,8558.7,18172.1,26398.7,21396.9,1 +nocache,16384,0.95,0.0,1,0.0467,8558.7,22289.6,32636.7,21396.9,0 +tcp,16384,0.95,0.8,1,0.0375,1191.5,8558.7,12988.6,21396.9,1 +tcp,16384,0.95,0.8,1,0.075,1191.5,9226.6,15478.4,21396.9,1 +tcp,16384,0.95,0.8,1,0.1126,1191.5,11873.2,16814.4,21396.9,1 +tcp,16384,0.95,0.8,1,0.1501,1465.1,15148.3,20946.7,21396.9,1 +tcp,16384,0.95,0.8,1,0.1876,2315.9,17526.9,30722.5,21396.9,1 +tcp,16384,0.95,0.8,1,0.2251,4538.9,22343.6,44827.7,21396.9,0 +rdma,16384,0.95,0.8,1,0.0495,387.2,8558.7,13777.4,21396.9,1 +rdma,16384,0.95,0.8,1,0.0989,387.2,8940.1,15743.5,21396.9,1 +rdma,16384,0.95,0.8,1,0.1484,387.2,13026.4,17459.4,21396.9,1 +rdma,16384,0.95,0.8,1,0.1979,554.3,16171.5,24664.6,21396.9,1 +rdma,16384,0.95,0.8,1,0.2473,2663.5,19640.5,37673.1,21396.9,1 +rdma,16384,0.95,0.8,1,0.2968,5378.8,26499.7,49919.3,21396.9,0 +nocache,24576,0.95,0.0,1,0.0069,14425.8,22718.6,28255.6,36064.6,1 +nocache,24576,0.95,0.0,1,0.0139,14425.8,26994.8,36240.0,36064.6,1 +nocache,24576,0.95,0.0,1,0.0208,14425.8,30629.2,44495.2,36064.6,1 +nocache,24576,0.95,0.0,1,0.0277,14425.8,37569.3,55009.4,36064.6,0 +tcp,24576,0.95,0.8,1,0.0231,1800.2,14425.8,22104.7,36064.6,1 +tcp,24576,0.95,0.8,1,0.0462,1800.2,15487.9,26191.3,36064.6,1 +tcp,24576,0.95,0.8,1,0.0694,1800.2,20223.8,28452.4,36064.6,1 +tcp,24576,0.95,0.8,1,0.0925,2219.8,25835.0,35682.3,36064.6,1 +tcp,24576,0.95,0.8,1,0.1156,3594.7,29843.9,52554.3,36064.6,1 +tcp,24576,0.95,0.8,1,0.1387,7662.1,38482.2,76935.3,36064.6,0 +rdma,24576,0.95,0.8,1,0.0297,595.5,14425.8,23267.1,36064.6,1 +rdma,24576,0.95,0.8,1,0.0595,595.5,15078.5,26566.8,36064.6,1 +rdma,24576,0.95,0.8,1,0.0892,595.5,22097.3,29708.6,36064.6,1 +rdma,24576,0.95,0.8,1,0.119,907.4,27276.9,41921.6,36064.6,1 +rdma,24576,0.95,0.8,1,0.1487,4655.6,33390.3,63899.9,36064.6,1 +rdma,24576,0.95,0.8,1,0.1785,9237.0,44923.1,84292.2,36064.6,0 diff --git a/src/code/issue5/results/raw/conc_summary.csv b/src/code/issue5/results/raw/conc_summary.csv new file mode 100644 index 0000000..004ba31 --- /dev/null +++ b/src/code/issue5/results/raw/conc_summary.csv @@ -0,0 +1,10 @@ +context_length,transport,mean_service_ms,slo_ms,lambda_max_rps,qpm +8192,nocache,3750.1,9375.3,0.08,4.8 +8192,tcp,2861.4,9375.3,0.1398,8.39 +8192,rdma,2755.5,9375.3,0.1452,8.71 +16384,nocache,8558.7,21396.9,0.0351,2.1 +16384,tcp,6381.8,21396.9,0.0627,3.76 +16384,rdma,6170.0,21396.9,0.0648,3.89 +24576,nocache,14425.8,36064.6,0.0208,1.25 +24576,tcp,10563.8,36064.6,0.0379,2.27 +24576,rdma,10246.1,36064.6,0.039,2.34 diff --git a/src/code/issue5/results/raw/conc_sweep.csv b/src/code/issue5/results/raw/conc_sweep.csv new file mode 100644 index 0000000..47c95ee --- /dev/null +++ b/src/code/issue5/results/raw/conc_sweep.csv @@ -0,0 +1,43 @@ +transport,context_length,hit_prefix,phit,workers,lambda_rps,p50_ttft_ms,p95_ttft_ms,p99_ttft_ms,slo_ms,within_slo +nocache,8192,0.5,0.0,1,0.0267,3750.1,5905.9,7345.3,9375.3,1 +nocache,8192,0.5,0.0,1,0.0533,3750.1,7017.5,9420.9,9375.3,1 +nocache,8192,0.5,0.0,1,0.08,3750.1,7962.4,11567.0,9375.3,1 +nocache,8192,0.5,0.0,1,0.1067,3750.1,9766.5,14300.2,9375.3,0 +tcp,8192,0.5,0.5,1,0.0349,3750.1,4616.2,6741.4,9375.3,1 +tcp,8192,0.5,0.5,1,0.0699,3750.1,5743.8,7559.2,9375.3,1 +tcp,8192,0.5,0.5,1,0.1048,3750.1,6959.7,9242.7,9375.3,1 +tcp,8192,0.5,0.5,1,0.1398,3750.1,8102.1,11393.4,9375.3,1 +tcp,8192,0.5,0.5,1,0.1747,3750.1,9785.3,13876.1,9375.3,0 +rdma,8192,0.5,0.5,1,0.0363,3750.1,4503.4,6728.2,9375.3,1 +rdma,8192,0.5,0.5,1,0.0726,3750.1,5728.7,7431.3,9375.3,1 +rdma,8192,0.5,0.5,1,0.1089,3750.1,6909.1,9119.1,9375.3,1 +rdma,8192,0.5,0.5,1,0.1452,3750.1,7981.0,11158.4,9375.3,1 +rdma,8192,0.5,0.5,1,0.1815,3750.1,9596.0,13748.1,9375.3,0 +nocache,16384,0.5,0.0,1,0.0117,8558.7,13478.8,16763.9,21396.9,1 +nocache,16384,0.5,0.0,1,0.0234,8558.7,16015.8,21501.0,21396.9,1 +nocache,16384,0.5,0.0,1,0.0351,8558.7,18172.1,26398.7,21396.9,1 +nocache,16384,0.5,0.0,1,0.0467,8558.7,22289.6,32636.7,21396.9,0 +tcp,16384,0.5,0.5,1,0.0157,8558.7,10383.5,15308.6,21396.9,1 +tcp,16384,0.5,0.5,1,0.0313,8558.7,13075.0,17044.4,21396.9,1 +tcp,16384,0.5,0.5,1,0.047,8558.7,15764.8,20819.6,21396.9,1 +tcp,16384,0.5,0.5,1,0.0627,8558.7,18227.3,25605.7,21396.9,1 +tcp,16384,0.5,0.5,1,0.0783,8558.7,22170.2,31352.0,21396.9,0 +rdma,16384,0.5,0.5,1,0.0162,8544.7,10093.5,15463.5,21396.9,1 +rdma,16384,0.5,0.5,1,0.0324,8558.7,13116.8,16962.5,21396.9,1 +rdma,16384,0.5,0.5,1,0.0486,8558.7,15620.8,20800.5,21396.9,1 +rdma,16384,0.5,0.5,1,0.0648,8558.7,18075.6,25619.2,21396.9,1 +rdma,16384,0.5,0.5,1,0.081,8558.7,21782.6,31308.9,21396.9,0 +nocache,24576,0.5,0.0,1,0.0069,14425.8,22718.6,28255.6,36064.6,1 +nocache,24576,0.5,0.0,1,0.0139,14425.8,26994.8,36240.0,36064.6,1 +nocache,24576,0.5,0.0,1,0.0208,14425.8,30629.2,44495.2,36064.6,1 +nocache,24576,0.5,0.0,1,0.0277,14425.8,37569.3,55009.4,36064.6,0 +tcp,24576,0.5,0.5,1,0.0095,14425.8,17264.7,25940.2,36064.6,1 +tcp,24576,0.5,0.5,1,0.0189,14425.8,22025.8,28586.6,36064.6,1 +tcp,24576,0.5,0.5,1,0.0284,14425.8,26536.3,35105.3,36064.6,1 +tcp,24576,0.5,0.5,1,0.0379,14425.8,30634.6,42973.7,36064.6,1 +tcp,24576,0.5,0.5,1,0.0473,14425.8,36864.5,52848.2,36064.6,0 +rdma,24576,0.5,0.5,1,0.0098,14179.0,16793.4,26020.2,36064.6,1 +rdma,24576,0.5,0.5,1,0.0195,14425.8,21953.1,28594.3,36064.6,1 +rdma,24576,0.5,0.5,1,0.0293,14425.8,26307.8,35179.7,36064.6,1 +rdma,24576,0.5,0.5,1,0.039,14425.8,30298.1,42877.2,36064.6,1 +rdma,24576,0.5,0.5,1,0.0488,14425.8,36554.3,52420.4,36064.6,0 diff --git a/src/code/issue5/results/raw/kv_tensor_rdma.csv b/src/code/issue5/results/raw/kv_tensor_rdma.csv new file mode 100644 index 0000000..abb6afa --- /dev/null +++ b/src/code/issue5/results/raw/kv_tensor_rdma.csv @@ -0,0 +1,13 @@ +transport,model,path,op,context,tp,page_size,num_pages,num_keys,kv_bytes,kv_size,iters,transfer_ms,effective_bandwidth_gbps,max_abs_error,success +rdma,qwen3-32b,bytes,put,8192,8,64,128,256,268435456,256MB,5,93.054,2.687,,1 +rdma,qwen3-32b,bytes,get,8192,8,64,128,256,268435456,256MB,5,107.556,2.324,0.0,1 +rdma,qwen3-32b,zerocopy,put,8192,8,64,128,256,268435456,256MB,5,11.893,21.021,,1 +rdma,qwen3-32b,zerocopy,get,8192,8,64,128,256,268435456,256MB,5,12.197,20.497,0.0,1 +rdma,qwen3-32b,bytes,put,32768,8,64,512,1024,1073741824,1.0GB,5,320.343,3.122,,1 +rdma,qwen3-32b,bytes,get,32768,8,64,512,1024,1073741824,1.0GB,5,257.4,3.885,0.0,1 +rdma,qwen3-32b,zerocopy,put,32768,8,64,512,1024,1073741824,1.0GB,5,49.273,20.295,,1 +rdma,qwen3-32b,zerocopy,get,32768,8,64,512,1024,1073741824,1.0GB,5,47.461,21.07,0.0,1 +rdma,qwen3-32b,bytes,put,131072,8,64,2048,4096,4294967296,4.0GB,5,1020.6,3.92,,1 +rdma,qwen3-32b,bytes,get,131072,8,64,2048,4096,4294967296,4.0GB,5,962.0,4.16,0.0,1 +rdma,qwen3-32b,zerocopy,put,131072,8,64,2048,4096,4294967296,4.0GB,5,202.36,19.767,,1 +rdma,qwen3-32b,zerocopy,get,131072,8,64,2048,4096,4294967296,4.0GB,5,193.394,20.683,0.0,1 diff --git a/src/code/issue5/results/raw/kv_tensor_tcp.csv b/src/code/issue5/results/raw/kv_tensor_tcp.csv new file mode 100644 index 0000000..a825839 --- /dev/null +++ b/src/code/issue5/results/raw/kv_tensor_tcp.csv @@ -0,0 +1,13 @@ +transport,model,path,op,context,tp,page_size,num_pages,num_keys,kv_bytes,kv_size,iters,transfer_ms,effective_bandwidth_gbps,max_abs_error,success +tcp,qwen3-32b,bytes,put,8192,8,64,128,256,268435456,256MB,5,342.811,0.729,,1 +tcp,qwen3-32b,bytes,get,8192,8,64,128,256,268435456,256MB,5,295.74,0.845,0.0,1 +tcp,qwen3-32b,zerocopy,put,8192,8,64,128,256,268435456,256MB,5,233.864,1.069,,1 +tcp,qwen3-32b,zerocopy,get,8192,8,64,128,256,268435456,256MB,5,226.193,1.105,0.0,1 +tcp,qwen3-32b,bytes,put,32768,8,64,512,1024,1073741824,1.0GB,5,1273.358,0.785,,1 +tcp,qwen3-32b,bytes,get,32768,8,64,512,1024,1073741824,1.0GB,5,1439.864,0.695,0.0,1 +tcp,qwen3-32b,zerocopy,put,32768,8,64,512,1024,1073741824,1.0GB,5,1072.737,0.932,,1 +tcp,qwen3-32b,zerocopy,get,32768,8,64,512,1024,1073741824,1.0GB,5,913.954,1.094,0.0,1 +tcp,qwen3-32b,bytes,put,131072,8,64,2048,4096,4294967296,4.0GB,5,5173.846,0.773,,1 +tcp,qwen3-32b,bytes,get,131072,8,64,2048,4096,4294967296,4.0GB,5,4840.868,0.826,0.0,1 +tcp,qwen3-32b,zerocopy,put,131072,8,64,2048,4096,4294967296,4.0GB,5,3980.428,1.005,,1 +tcp,qwen3-32b,zerocopy,get,131072,8,64,2048,4096,4294967296,4.0GB,5,3404.958,1.175,0.0,1 diff --git a/src/code/issue5/results/raw/kv_tensor_v32_rdma.csv b/src/code/issue5/results/raw/kv_tensor_v32_rdma.csv new file mode 100644 index 0000000..1af5c81 --- /dev/null +++ b/src/code/issue5/results/raw/kv_tensor_v32_rdma.csv @@ -0,0 +1,13 @@ +transport,model,path,op,context,tp,page_size,num_pages,num_keys,kv_bytes,kv_size,iters,transfer_ms,effective_bandwidth_gbps,max_abs_error,success +rdma,deepseek-v32,bytes,put,8192,8,64,128,128,575668224,549MB,5,77.2,6.945,,1 +rdma,deepseek-v32,bytes,get,8192,8,64,128,128,575668224,549MB,5,203.946,2.629,0.0,1 +rdma,deepseek-v32,zerocopy,put,8192,8,64,128,128,575668224,549MB,5,24.649,21.751,,1 +rdma,deepseek-v32,zerocopy,get,8192,8,64,128,128,575668224,549MB,5,31.016,17.286,0.0,1 +rdma,deepseek-v32,bytes,put,32768,8,64,512,512,2302672896,2.1GB,5,305.875,7.011,,1 +rdma,deepseek-v32,bytes,get,32768,8,64,512,512,2302672896,2.1GB,5,291.117,7.367,0.0,1 +rdma,deepseek-v32,zerocopy,put,32768,8,64,512,512,2302672896,2.1GB,5,99.534,21.546,,1 +rdma,deepseek-v32,zerocopy,get,32768,8,64,512,512,2302672896,2.1GB,5,104.572,20.508,0.0,1 +rdma,deepseek-v32,bytes,put,131072,8,64,2048,2048,9210691584,8.6GB,5,1877.866,4.568,,1 +rdma,deepseek-v32,bytes,get,131072,8,64,2048,2048,9210691584,8.6GB,5,1230.088,6.974,0.0,1 +rdma,deepseek-v32,zerocopy,put,131072,8,64,2048,2048,9210691584,8.6GB,5,390.708,21.955,,1 +rdma,deepseek-v32,zerocopy,get,131072,8,64,2048,2048,9210691584,8.6GB,5,404.963,21.182,0.0,1 diff --git a/src/code/issue5/results/raw/kv_tensor_v32_tcp.csv b/src/code/issue5/results/raw/kv_tensor_v32_tcp.csv new file mode 100644 index 0000000..88f366c --- /dev/null +++ b/src/code/issue5/results/raw/kv_tensor_v32_tcp.csv @@ -0,0 +1,13 @@ +transport,model,path,op,context,tp,page_size,num_pages,num_keys,kv_bytes,kv_size,iters,transfer_ms,effective_bandwidth_gbps,max_abs_error,success +tcp,deepseek-v32,bytes,put,8192,8,64,128,128,575668224,549MB,5,623.595,0.86,,1 +tcp,deepseek-v32,bytes,get,8192,8,64,128,128,575668224,549MB,5,791.541,0.677,0.0,1 +tcp,deepseek-v32,zerocopy,put,8192,8,64,128,128,575668224,549MB,5,452.819,1.184,,1 +tcp,deepseek-v32,zerocopy,get,8192,8,64,128,128,575668224,549MB,5,460.377,1.165,0.0,1 +tcp,deepseek-v32,bytes,put,32768,8,64,512,512,2302672896,2.1GB,5,3541.427,0.606,,1 +tcp,deepseek-v32,bytes,get,32768,8,64,512,512,2302672896,2.1GB,5,2648.839,0.81,0.0,1 +tcp,deepseek-v32,zerocopy,put,32768,8,64,512,512,2302672896,2.1GB,5,1984.112,1.081,,1 +tcp,deepseek-v32,zerocopy,get,32768,8,64,512,512,2302672896,2.1GB,5,1754.665,1.222,0.0,1 +tcp,deepseek-v32,bytes,put,131072,8,64,2048,2048,9210691584,8.6GB,5,8382.252,1.023,,1 +tcp,deepseek-v32,bytes,get,131072,8,64,2048,2048,9210691584,8.6GB,5,10357.485,0.828,0.0,1 +tcp,deepseek-v32,zerocopy,put,131072,8,64,2048,2048,9210691584,8.6GB,5,8221.086,1.043,,1 +tcp,deepseek-v32,zerocopy,get,131072,8,64,2048,2048,9210691584,8.6GB,5,7194.248,1.192,0.0,1 diff --git a/src/code/issue5/results/raw/prefill_local.csv b/src/code/issue5/results/raw/prefill_local.csv new file mode 100644 index 0000000..a608983 --- /dev/null +++ b/src/code/issue5/results/raw/prefill_local.csv @@ -0,0 +1,16 @@ +transport,context_length,hit_ratio,concurrency,prefix_tokens,uniq_tokens,cached_tokens,prompt_tokens,kv_bytes,s_token,lookup_ms,transfer_ms,restore_ms,cold_uniq_ms,recompute_ms,saved_ms,total_ms,effective_bandwidth_gbps,max_abs_error,success +local_radix,8192,0.0,1,0,8192,0,8192,0,70272,,0.0,0.0,3677.36,3679.76,18.91,3660.85,,,1 +local_radix,8192,0.25,1,2048,6144,2048,8192,143917056,70272,,0.0,0.0,2742.75,3653.85,357.56,3296.29,,,1 +local_radix,8192,0.5,1,4096,4096,4096,8192,287834112,70272,,0.0,0.0,1402.2,3638.41,1396.53,2241.88,,,1 +local_radix,8192,0.75,1,6144,2048,6144,8192,431751168,70272,,0.0,0.0,357.61,3648.71,2452.81,1195.91,,,1 +local_radix,8192,1.0,1,8192,0,8128,8192,571170816,70272,,0.0,0.0,0.0,3659.45,3278.28,381.17,,,1 +local_radix,16384,0.0,1,0,16384,0,16384,0,70272,,0.0,0.0,8691.48,8672.48,0.0,8672.75,,,1 +local_radix,16384,0.25,1,4096,12288,4096,16384,287834112,70272,,0.0,0.0,6102.61,8643.03,1422.53,7220.49,,,1 +local_radix,16384,0.5,1,8192,8192,8192,16384,575668224,70272,,0.0,0.0,3626.44,8655.03,3615.6,5039.44,,,1 +local_radix,16384,0.75,1,12288,4096,12288,16384,863502336,70272,,0.0,0.0,1386.94,8917.03,6288.46,2628.57,,,1 +local_radix,16384,1.0,1,16384,0,16320,16384,1146839040,70272,,0.0,0.0,0.0,8638.42,8253.2,385.21,,,1 +local_radix,24576,0.0,1,0,24576,0,24576,0,70272,,0.0,0.0,14404.24,14393.74,52.75,14340.99,,,1 +local_radix,24576,0.25,1,6144,18432,6144,24576,431751168,70272,,0.0,0.0,10004.39,14379.5,2549.47,11830.04,,,1 +local_radix,24576,0.5,1,12288,12288,12288,24576,863502336,70272,,0.0,0.0,6074.62,14428.98,6124.49,8304.49,,,1 +local_radix,24576,0.75,1,18432,6144,18432,24576,1295253504,70272,,0.0,0.0,2484.43,14314.73,9848.02,4466.71,,,1 +local_radix,24576,1.0,1,24576,0,24512,24576,1722507264,70272,,0.0,0.0,0.0,14409.52,14004.68,404.83,,,1 diff --git a/src/code/issue5/results/raw/transfer_rdma.csv b/src/code/issue5/results/raw/transfer_rdma.csv new file mode 100644 index 0000000..f81e073 --- /dev/null +++ b/src/code/issue5/results/raw/transfer_rdma.csv @@ -0,0 +1,25 @@ +transport,op,size,size_bytes,concurrency,iters,total_bytes,wall_s,bandwidth_gbps,lat_mean_ms,lat_p50_ms,lat_p95_ms,lat_p99_ms,success +rdma,put,64MB,67108864,1,20,1342177280,0.2547,4.907,12.728,12.092,14.24,22.654,1 +rdma,get,64MB,67108864,1,20,1342177280,0.8003,1.562,35.514,35.445,36.026,36.766,1 +rdma,put,64MB,67108864,2,20,1342177280,0.1329,9.405,12.806,12.779,12.984,16.228,1 +rdma,get,64MB,67108864,2,20,1342177280,0.5945,2.103,55.548,54.995,57.729,61.265,1 +rdma,put,64MB,67108864,4,20,1342177280,0.09,13.885,16.81,16.059,22.016,24.349,1 +rdma,get,64MB,67108864,4,20,1342177280,0.6732,1.857,119.401,123.213,194.759,230.516,1 +rdma,put,64MB,67108864,8,20,1342177280,0.078,16.034,26.58,25.703,36.297,40.607,1 +rdma,get,64MB,67108864,8,20,1342177280,0.6933,1.803,243.583,227.258,355.241,515.412,1 +rdma,put,256MB,268435456,1,20,5368709120,0.9575,5.222,47.868,47.774,48.066,50.26,1 +rdma,get,256MB,268435456,1,20,5368709120,3.4146,1.464,151.929,152.023,154.575,154.733,1 +rdma,put,256MB,268435456,2,20,5368709120,0.6555,7.628,64.854,55.525,142.204,153.727,1 +rdma,get,256MB,268435456,2,20,5368709120,2.9961,1.669,273.096,276.756,300.763,314.669,1 +rdma,put,256MB,268435456,4,20,5368709120,0.5117,9.772,97.732,81.539,185.472,198.128,1 +rdma,get,256MB,268435456,4,20,5368709120,2.9295,1.707,448.482,429.7,688.954,719.137,1 +rdma,put,256MB,268435456,8,20,5368709120,0.3733,13.394,129.937,129.498,168.917,180.765,1 +rdma,get,256MB,268435456,8,20,5368709120,2.9399,1.701,787.786,721.769,1424.445,1461.856,1 +rdma,put,1GB,1073741824,1,20,21474836480,3.9354,5.082,196.761,196.846,197.357,199.433,1 +rdma,get,1GB,1073741824,1,20,21474836480,13.0334,1.535,589.252,590.406,595.992,599.725,1 +rdma,put,1GB,1073741824,2,20,21474836480,2.9717,6.73,294.8,239.306,496.735,543.557,1 +rdma,get,1GB,1073741824,2,20,21474836480,11.1724,1.79,1040.841,1068.227,1180.398,1184.151,1 +rdma,put,1GB,1073741824,4,20,21474836480,2.0099,9.951,386.958,261.293,924.457,978.151,1 +rdma,get,1GB,1073741824,4,20,21474836480,10.5258,1.9,1626.061,1642.964,2170.677,2177.04,1 +rdma,put,1GB,1073741824,8,20,21474836480,0.603,33.17,129.357,0.088,559.85,601.837,0 +rdma,get,1GB,1073741824,8,20,21474836480,2.8063,7.127,664.466,0.234,2708.532,2761.194,0 diff --git a/src/code/issue5/results/raw/transfer_tcp.csv b/src/code/issue5/results/raw/transfer_tcp.csv new file mode 100644 index 0000000..65a0130 --- /dev/null +++ b/src/code/issue5/results/raw/transfer_tcp.csv @@ -0,0 +1,25 @@ +transport,op,size,size_bytes,concurrency,iters,total_bytes,wall_s,bandwidth_gbps,lat_mean_ms,lat_p50_ms,lat_p95_ms,lat_p99_ms,success +tcp,put,64MB,67108864,1,16,1073741824,1.2916,0.774,80.705,78.458,81.257,134.332,1 +tcp,get,64MB,67108864,1,16,1073741824,1.9084,0.524,115.875,115.572,135.98,142.531,1 +tcp,put,64MB,67108864,2,16,1073741824,0.9393,1.065,116.811,115.969,138.063,145.889,1 +tcp,get,64MB,67108864,2,16,1073741824,1.4136,0.707,172.828,149.225,267.304,270.044,1 +tcp,put,64MB,67108864,4,16,1073741824,0.8492,1.178,211.809,207.647,236.131,245.139,1 +tcp,get,64MB,67108864,4,16,1073741824,1.0607,0.943,249.244,269.754,297.412,302.182,1 +tcp,put,64MB,67108864,8,16,1073741824,0.8689,1.151,409.866,407.34,472.026,472.309,1 +tcp,get,64MB,67108864,8,16,1073741824,1.7648,0.567,833.883,871.141,951.296,956.719,1 +tcp,put,256MB,268435456,1,16,4294967296,4.8048,0.833,300.278,289.83,381.702,395.084,1 +tcp,get,256MB,268435456,1,16,4294967296,6.4712,0.618,393.257,434.073,487.26,495.68,1 +tcp,put,256MB,268435456,2,16,4294967296,4.168,0.96,518.663,557.95,683.211,683.769,1 +tcp,get,256MB,268435456,2,16,4294967296,7.9041,0.506,970.946,975.865,1018.318,1028.147,1 +tcp,put,256MB,268435456,4,16,4294967296,2.6162,1.529,639.918,637.962,784.555,784.867,1 +tcp,get,256MB,268435456,4,16,4294967296,4.6508,0.86,1042.28,1071.21,1282.543,1293.222,1 +tcp,put,256MB,268435456,8,16,4294967296,2.8253,1.416,1390.093,1487.925,1614.796,1615.577,1 +tcp,get,256MB,268435456,8,16,4294967296,6.0885,0.657,2684.126,2592.065,3455.654,3471.826,1 +tcp,put,1GB,1073741824,1,16,17179869184,15.0903,1.06,943.121,941.371,1129.753,1202.295,1 +tcp,get,1GB,1073741824,1,16,17179869184,29.1459,0.549,1776.471,1811.315,2163.275,2371.168,1 +tcp,put,1GB,1073741824,2,16,17179869184,14.9079,1.073,1813.277,1746.386,2677.336,2677.6,1 +tcp,get,1GB,1073741824,2,16,17179869184,30.126,0.531,3702.628,3896.115,4333.536,4382.249,1 +tcp,put,1GB,1073741824,4,16,17179869184,17.085,0.936,4066.624,4018.78,5491.047,5491.388,1 +tcp,get,1GB,1073741824,4,16,17179869184,27.8982,0.574,6306.943,7350.064,8485.421,8560.655,1 +tcp,put,1GB,1073741824,8,16,17179869184,5.5563,2.88,1735.704,0.177,5554.313,5554.522,0 +tcp,get,1GB,1073741824,8,16,17179869184,8.9092,1.796,2565.32,0.313,8394.456,8857.657,0 diff --git a/src/code/issue5/results/raw/transfer_zcget_rdma.csv b/src/code/issue5/results/raw/transfer_zcget_rdma.csv new file mode 100644 index 0000000..2d3f51d --- /dev/null +++ b/src/code/issue5/results/raw/transfer_zcget_rdma.csv @@ -0,0 +1,13 @@ +transport,op,size,size_bytes,concurrency,iters,total_bytes,wall_s,bandwidth_gbps,lat_mean_ms,lat_p50_ms,lat_p95_ms,lat_p99_ms,success +rdma,get_zc,64MB,67108864,1,20,67108864,0.0061,10.203,6.816,6.132,13.033,13.258,1 +rdma,get_zc,64MB,67108864,2,20,134217728,0.014,8.933,14.111,14.039,14.701,18.114,1 +rdma,get_zc,64MB,67108864,4,20,268435456,0.0239,10.482,23.596,23.931,24.973,25.065,1 +rdma,get_zc,64MB,67108864,8,20,536870912,0.0373,13.392,37.926,37.352,41.094,41.844,1 +rdma,get_zc,256MB,268435456,1,20,268435456,0.024,10.431,23.292,23.981,26.623,31.083,1 +rdma,get_zc,256MB,268435456,2,20,536870912,0.0391,12.779,39.584,39.223,42.882,43.325,1 +rdma,get_zc,256MB,268435456,4,20,1073741824,0.0758,13.195,78.758,76.476,91.547,120.712,1 +rdma,get_zc,256MB,268435456,8,20,2147483648,0.1662,12.031,166.577,168.621,185.066,188.238,1 +rdma,get_zc,1GB,1073741824,1,20,1073741824,0.0718,13.919,73.292,72.342,81.119,85.008,1 +rdma,get_zc,1GB,1073741824,2,20,2147483648,0.1382,14.472,140.909,138.236,164.686,173.192,1 +rdma,get_zc,1GB,1073741824,4,20,4294967296,0.2737,14.614,273.774,273.717,281.222,283.406,1 +rdma,get_zc,1GB,1073741824,8,20,8589934592,0.5448,14.686,544.857,544.853,567.016,576.927,1 diff --git a/src/code/issue5/results/raw/transfer_zcget_tcp.csv b/src/code/issue5/results/raw/transfer_zcget_tcp.csv new file mode 100644 index 0000000..086ddb3 --- /dev/null +++ b/src/code/issue5/results/raw/transfer_zcget_tcp.csv @@ -0,0 +1,13 @@ +transport,op,size,size_bytes,concurrency,iters,total_bytes,wall_s,bandwidth_gbps,lat_mean_ms,lat_p50_ms,lat_p95_ms,lat_p99_ms,success +tcp,get_zc,64MB,67108864,1,20,67108864,0.0522,1.197,51.632,52.253,54.246,54.575,1 +tcp,get_zc,64MB,67108864,2,20,134217728,0.1025,1.22,102.551,102.937,106.213,106.538,1 +tcp,get_zc,64MB,67108864,4,20,268435456,0.2038,1.226,206.889,203.894,209.778,264.491,1 +tcp,get_zc,64MB,67108864,8,20,536870912,0.4133,1.21,476.683,413.478,566.942,961.212,1 +tcp,get_zc,256MB,268435456,1,20,268435456,0.2107,1.186,253.134,211.432,377.767,430.38,1 +tcp,get_zc,256MB,268435456,2,20,536870912,0.5377,0.93,523.614,538.936,612.795,710.163,1 +tcp,get_zc,256MB,268435456,4,20,1073741824,0.6962,1.436,696.98,705.929,807.603,934.502,1 +tcp,get_zc,256MB,268435456,8,20,2147483648,1.34,1.493,1520.469,1343.019,2372.013,2951.801,1 +tcp,get_zc,1GB,1073741824,1,20,1073741824,0.7639,1.309,758.393,770.223,868.6,907.464,1 +tcp,get_zc,1GB,1073741824,2,20,2147483648,1.3856,1.443,1531.89,1397.911,2507.008,3055.359,1 +tcp,get_zc,1GB,1073741824,4,20,4294967296,2.5331,1.579,2833.353,2543.211,4234.524,5601.75,1 +tcp,get_zc,1GB,1073741824,8,20,8589934592,6.5104,1.229,6885.025,6738.138,8548.463,12657.145,1 diff --git a/src/code/issue5/results/raw/transfer_zcput_rdma.csv b/src/code/issue5/results/raw/transfer_zcput_rdma.csv new file mode 100644 index 0000000..689af58 --- /dev/null +++ b/src/code/issue5/results/raw/transfer_zcput_rdma.csv @@ -0,0 +1,13 @@ +transport,op,size,size_bytes,concurrency,iters,total_bytes,wall_s,bandwidth_gbps,lat_mean_ms,lat_p50_ms,lat_p95_ms,lat_p99_ms,success +rdma,put_zc,64MB,67108864,1,20,67108864,0.0032,19.654,3.808,3.185,3.258,15.701,1 +rdma,put_zc,64MB,67108864,2,20,134217728,0.006,20.802,6.162,6.014,6.433,8.007,1 +rdma,put_zc,64MB,67108864,4,20,268435456,0.0113,22.07,11.35,11.329,11.467,11.597,1 +rdma,put_zc,64MB,67108864,8,20,536870912,0.0229,21.856,22.88,22.885,22.979,23.0,1 +rdma,put_zc,256MB,268435456,1,20,268435456,0.0136,18.375,13.824,13.609,13.897,17.557,1 +rdma,put_zc,256MB,268435456,2,20,536870912,0.0248,20.152,24.84,24.834,25.177,26.093,1 +rdma,put_zc,256MB,268435456,4,20,1073741824,0.0483,20.719,48.25,48.267,48.412,48.528,1 +rdma,put_zc,256MB,268435456,8,20,2147483648,0.0946,21.14,94.913,94.632,95.329,101.911,1 +rdma,put_zc,1GB,1073741824,1,20,1073741824,0.0553,18.073,55.445,55.338,55.988,58.262,1 +rdma,put_zc,1GB,1073741824,2,20,2147483648,0.1011,19.775,100.639,101.164,101.364,111.412,1 +rdma,put_zc,1GB,1073741824,4,20,4294967296,0.1896,21.095,190.754,189.682,203.879,206.61,1 +rdma,put_zc,1GB,1073741824,8,20,8589934592,0.3591,22.28,360.598,359.133,377.662,382.966,1 diff --git a/src/code/issue5/results/raw/transfer_zcput_tcp.csv b/src/code/issue5/results/raw/transfer_zcput_tcp.csv new file mode 100644 index 0000000..7436803 --- /dev/null +++ b/src/code/issue5/results/raw/transfer_zcput_tcp.csv @@ -0,0 +1,13 @@ +transport,op,size,size_bytes,concurrency,iters,total_bytes,wall_s,bandwidth_gbps,lat_mean_ms,lat_p50_ms,lat_p95_ms,lat_p99_ms,success +tcp,put_zc,64MB,67108864,1,20,67108864,0.1072,0.583,112.322,107.292,116.707,179.119,1 +tcp,put_zc,64MB,67108864,2,20,134217728,0.2009,0.622,201.746,201.093,256.049,271.939,1 +tcp,put_zc,64MB,67108864,4,20,268435456,0.3803,0.657,377.463,381.663,404.829,405.909,1 +tcp,put_zc,64MB,67108864,8,20,536870912,0.6837,0.731,674.838,690.787,830.46,1065.035,1 +tcp,put_zc,256MB,268435456,1,20,268435456,0.3868,0.646,432.223,387.972,425.065,1386.039,1 +tcp,put_zc,256MB,268435456,2,20,536870912,0.6539,0.765,658.928,662.376,777.822,790.043,1 +tcp,put_zc,256MB,268435456,4,20,1073741824,0.8219,1.217,925.995,831.905,1635.524,1882.998,1 +tcp,put_zc,256MB,268435456,8,20,2147483648,1.7081,1.171,1870.389,1713.598,2553.497,2958.894,1 +tcp,put_zc,1GB,1073741824,1,20,1073741824,0.8253,1.212,913.702,829.09,1492.525,1679.464,1 +tcp,put_zc,1GB,1073741824,2,20,2147483648,1.623,1.232,1810.009,1634.325,2507.256,4079.183,1 +tcp,put_zc,1GB,1073741824,4,20,4294967296,3.778,1.059,3980.553,3862.133,5261.083,7691.516,1 +tcp,put_zc,1GB,1073741824,8,20,8589934592,9.7038,0.824,9747.463,9791.47,11694.599,14900.413,1 diff --git a/src/code/issue5/scripts/plot_results.py b/src/code/issue5/scripts/plot_results.py new file mode 100644 index 0000000..6f6bb15 --- /dev/null +++ b/src/code/issue5/scripts/plot_results.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Issue5 汇总出图 (读 results/raw 的全部 CSV, 出 report/figures/) + +三类图 + 模型/QPM 图: + 1) bw_vs_size.png L1 原语: 有效带宽 vs value 大小 (TCP/RDMA × put/get) + 2) bw_vs_concurrency.png L1 原语: put 带宽 vs 并发 (TCP/RDMA, 展示 RDMA scale) + 3) l2_zerocopy_bw.png L2 真实 KV: zerocopy vs bytes 的 get 带宽 (TCP/RDMA) + 4) ttft_vs_load.png L4: P95 TTFT vs 到达率 (三传输 + SLO 线, N=16k 两 regime) + 5) qpm_compare.png L4: QPM 柱状 (两 regime × 三传输 × 三 N) + 6) protocol_summary.png 一图总览: L1/L2/L4 关键指标的 RDMA/TCP 倍数 + +用法: + python3 scripts/plot_results.py --raw results/raw --figdir report/figures +""" +import argparse, csv, os, statistics +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +GB = 1 << 30 + + +def read_csv(path): + if not os.path.exists(path): + return [] + with open(path) as f: + return list(csv.DictReader(f)) + + +def med(rows, pred, col): + vals = [] + for r in rows: + if pred(r): + try: + vals.append(float(r[col])) + except (ValueError, KeyError): + pass + return statistics.median(vals) if vals else None + + +# ---- 图1: L1 带宽 vs value 大小 (put/get × tcp/rdma) ---- +def plot_bw_vs_size(raw, figdir): + tcp = read_csv(f"{raw}/transfer_tcp.csv") + rdma = read_csv(f"{raw}/transfer_rdma.csv") + if not tcp or not rdma: + return None + sizes = ["64MB", "256MB", "1GB"] + fig, axes = plt.subplots(1, 2, figsize=(10, 4)) + for ax, op in zip(axes, ("put", "get")): + for data, name, c in ((tcp, "tcp", "C0"), (rdma, "rdma", "C1")): + ys = [med(data, lambda r, s=s: r["size"] == s and r["op"] == op + and r["success"] == "1", "bandwidth_gbps") for s in sizes] + ax.plot(sizes, ys, "o-", color=c, label=name) + ax.set_title(f"L1 {op} bandwidth") + ax.set_xlabel("value size"); ax.set_ylabel("GB/s") + ax.grid(alpha=.3); ax.legend() + fig.suptitle("L1 primitive: effective bandwidth vs value size") + fig.tight_layout(); fig.savefig(f"{figdir}/bw_vs_size.png", dpi=120); plt.close(fig) + return "bw_vs_size.png" + + +# ---- 图2: L1 put 带宽 vs 并发 ---- +def plot_bw_vs_conc(raw, figdir): + tcp = read_csv(f"{raw}/transfer_tcp.csv") + rdma = read_csv(f"{raw}/transfer_rdma.csv") + if not tcp or not rdma: + return None + concs = ["1", "2", "4", "8"] + fig, ax = plt.subplots(figsize=(6, 4)) + for data, name, c in ((tcp, "tcp", "C0"), (rdma, "rdma", "C1")): + ys = [med(data, lambda r, cc=cc: r["concurrency"] == cc and r["op"] == "put" + and r["size"] == "64MB" and r["success"] == "1", "bandwidth_gbps") + for cc in concs] + ax.plot(concs, ys, "o-", color=c, label=name) + ax.set_title("L1 put bandwidth vs concurrency (64MB)") + ax.set_xlabel("concurrency"); ax.set_ylabel("GB/s") + ax.grid(alpha=.3); ax.legend() + fig.tight_layout(); fig.savefig(f"{figdir}/bw_vs_concurrency.png", dpi=120); plt.close(fig) + return "bw_vs_concurrency.png" + + +# ---- 图3: L2 真实 KV, zerocopy vs bytes 的 get 带宽 (tcp/rdma) ---- +def plot_l2_zerocopy(raw, figdir): + tcp = read_csv(f"{raw}/kv_tensor_tcp.csv") + rdma = read_csv(f"{raw}/kv_tensor_rdma.csv") + if not tcp or not rdma: + return None + fig, ax = plt.subplots(figsize=(6, 4)) + labels, vals, colors = [], [], [] + for data, name in ((tcp, "tcp"), (rdma, "rdma")): + for path in ("bytes", "zerocopy"): + v = med(data, lambda r, p=path: r.get("path") == p and r.get("op") == "get" + and r.get("success") == "1", "effective_bandwidth_gbps") + labels.append(f"{name}\n{path}"); vals.append(v or 0) + colors.append("C0" if name == "tcp" else "C1") + bars = ax.bar(range(len(vals)), vals, color=colors) + ax.set_xticks(range(len(labels))); ax.set_xticklabels(labels) + ax.set_ylabel("get bandwidth (GB/s)") + ax.set_title("L2 real-KV get: zerocopy vs bytes (RDMA-zc ~18.7x TCP-zc)") + for b, v in zip(bars, vals): + ax.text(b.get_x() + b.get_width() / 2, v, f"{v:.1f}", ha="center", va="bottom", fontsize=8) + ax.grid(alpha=.3, axis="y") + fig.tight_layout(); fig.savefig(f"{figdir}/l2_zerocopy_bw.png", dpi=120); plt.close(fig) + return "l2_zerocopy_bw.png" + + +# ---- 图4: L4 P95 TTFT vs 到达率 (三传输 + SLO 线), N=16k 两 regime ---- +def plot_ttft_vs_load(raw, figdir): + mod = read_csv(f"{raw}/conc_sweep.csv") + hi = read_csv(f"{raw}/conc_hihit_sweep.csv") + if not mod or not hi: + return None + fig, axes = plt.subplots(1, 2, figsize=(11, 4)) + for ax, rows, title in ((axes[0], mod, "moderate hit h=.5"), + (axes[1], hi, "high hit h=.95")): + N = 16384 + slo = None + for tp, c in (("nocache", "C2"), ("tcp", "C0"), ("rdma", "C1")): + pts = sorted((float(r["lambda_rps"]), float(r["p95_ttft_ms"])) + for r in rows if r["transport"] == tp + and int(r["context_length"]) == N) + if not pts: + continue + ax.plot([p[0] for p in pts], [p[1] / 1e3 for p in pts], "o-", + color=c, label=tp, ms=4) + for r in rows: + if r["transport"] == tp and int(r["context_length"]) == N: + slo = float(r["slo_ms"]) / 1e3 + if slo: + ax.axhline(slo, ls="--", color="k", alpha=.6, label=f"SLO={slo:.1f}s") + ax.set_title(f"P95 TTFT vs load (N=16k, {title})") + ax.set_xlabel("arrival rate λ (req/s)"); ax.set_ylabel("P95 TTFT (s)") + ax.grid(alpha=.3); ax.legend(fontsize=8) + fig.tight_layout(); fig.savefig(f"{figdir}/ttft_vs_load.png", dpi=120); plt.close(fig) + return "ttft_vs_load.png" + + +# ---- 图5: QPM 柱状 (两 regime × 三传输 × 三 N) ---- +def plot_qpm_compare(raw, figdir): + mod = read_csv(f"{raw}/conc_summary.csv") + hi = read_csv(f"{raw}/conc_hihit_summary.csv") + if not mod or not hi: + return None + Ns = [8192, 16384, 24576] + fig, axes = plt.subplots(1, 2, figsize=(11, 4), sharey=False) + for ax, rows, title in ((axes[0], mod, "moderate hit h=.5"), + (axes[1], hi, "high hit h=.95")): + qpm = {} + for r in rows: + qpm[(int(r["context_length"]), r["transport"])] = float(r["qpm"]) + x = np.arange(len(Ns)); w = 0.25 + for i, (tp, c) in enumerate((("nocache", "C2"), ("tcp", "C0"), ("rdma", "C1"))): + ys = [qpm.get((N, tp), 0) for N in Ns] + ax.bar(x + (i - 1) * w, ys, w, label=tp, color=c) + ax.set_xticks(x); ax.set_xticklabels([f"{N // 1024}k" for N in Ns]) + ax.set_title(f"QPM ({title})"); ax.set_xlabel("context N"); ax.set_ylabel("QPM") + ax.grid(alpha=.3, axis="y"); ax.legend() + fig.tight_layout(); fig.savefig(f"{figdir}/qpm_compare.png", dpi=120); plt.close(fig) + return "qpm_compare.png" + + +# ---- 图6: 一图总览 RDMA/TCP 倍数 (L1 put / L2 zc get / L4 QPM 两 regime) ---- +def plot_protocol_summary(raw, figdir): + t1 = read_csv(f"{raw}/transfer_tcp.csv"); r1 = read_csv(f"{raw}/transfer_rdma.csv") + t2 = read_csv(f"{raw}/kv_tensor_tcp.csv"); r2 = read_csv(f"{raw}/kv_tensor_rdma.csv") + mod = read_csv(f"{raw}/conc_summary.csv"); hi = read_csv(f"{raw}/conc_hihit_summary.csv") + ratios, labels = [], [] + + def peak(rows, op): + vals = [float(x["bandwidth_gbps"]) for x in rows + if x["op"] == op and x["success"] == "1"] + return max(vals) if vals else None + if t1 and r1: + ratios.append(peak(r1, "put") / peak(t1, "put")); labels.append("L1 put\n(peak)") + if t2 and r2: + zt = med(t2, lambda r: r.get("path") == "zerocopy" and r.get("op") == "get" + and r.get("success") == "1", "effective_bandwidth_gbps") + zr = med(r2, lambda r: r.get("path") == "zerocopy" and r.get("op") == "get" + and r.get("success") == "1", "effective_bandwidth_gbps") + if zt and zr: + ratios.append(zr / zt); labels.append("L2 zc get\n(median)") + + def qpm_ratio(rows, N): + d = {r["transport"]: float(r["qpm"]) for r in rows + if int(r["context_length"]) == N} + return d["rdma"] / d["tcp"] if d.get("tcp") else None + if mod: + ratios.append(qpm_ratio(mod, 16384)); labels.append("L4 QPM\nmod-hit 16k") + if hi: + ratios.append(qpm_ratio(hi, 16384)); labels.append("L4 QPM\nhi-hit 16k") + if not ratios: + return None + fig, ax = plt.subplots(figsize=(7, 4)) + bars = ax.bar(range(len(ratios)), ratios, color=["C1", "C1", "C3", "C3"][:len(ratios)]) + ax.axhline(1.0, ls="--", color="k", alpha=.5) + ax.set_xticks(range(len(labels))); ax.set_xticklabels(labels, fontsize=9) + ax.set_ylabel("RDMA / TCP (×)"); ax.set_yscale("log") + ax.set_title("RDMA vs TCP: transport wins big, e2e QPM depends on retrieve share") + for b, v in zip(bars, ratios): + ax.text(b.get_x() + b.get_width() / 2, v, f"{v:.1f}×", ha="center", va="bottom", fontsize=9) + ax.grid(alpha=.3, axis="y", which="both") + fig.tight_layout(); fig.savefig(f"{figdir}/protocol_summary.png", dpi=120); plt.close(fig) + return "protocol_summary.png" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--raw", default="results/raw") + ap.add_argument("--figdir", default="report/figures") + args = ap.parse_args() + os.makedirs(args.figdir, exist_ok=True) + made = [] + for fn in (plot_bw_vs_size, plot_bw_vs_conc, plot_l2_zerocopy, + plot_ttft_vs_load, plot_qpm_compare, plot_protocol_summary): + try: + r = fn(args.raw, args.figdir) + if r: + made.append(r) + except Exception as e: + print(f"[skip] {fn.__name__}: {e}") + print(f"[out] wrote {made} -> {args.figdir}") + + +if __name__ == "__main__": + main() diff --git a/src/code/issue5/scripts/run_benchmark.sh b/src/code/issue5/scripts/run_benchmark.sh new file mode 100755 index 0000000..b83736a --- /dev/null +++ b/src/code/issue5/scripts/run_benchmark.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# ============================================================================= +# Issue5 TCP/RDMA 基准 (收敛协议切换, 验收2 实现部分) +# ----------------------------------------------------------------------------- +# 把 L1(transfer 原语) / L2(真实 KV 张量) / L4(TTFT-QPM replay) 的协议切换收敛到 +# 一个入口: `TRANSPORT=tcp|rdma bash run_benchmark.sh `。 +# +# 关键实现事实(见 configs/{tcp,rdma}.yaml、rdma/README.md): +# - TCP↔RDMA 的**唯一传输改动** = Mooncake protocol + client device_name。 +# - RDMA client 必须显式 MC_DEVICE=mlx5_3 (prefill 的 eth1); 空会 No available RNIC。 +# - store 端需先用同一 MC_PROTOCOL 起 (MC_PROTOCOL=rdma bash start_kvstore.sh)。 +# - L4 纯本地重放(吃 L1-L3 CSV), 与协议无关, 一次跑出三传输对比, 无需切协议。 +# +# 用法 (在 prefill pod 内, 或本地对已产出的 CSV 只跑 L4/汇总): +# TRANSPORT=tcp bash scripts/run_benchmark.sh l1 # L1 原语微基准 +# TRANSPORT=rdma MC_DEVICE=mlx5_3 bash scripts/run_benchmark.sh l1 +# TRANSPORT=rdma MC_DEVICE=mlx5_3 bash scripts/run_benchmark.sh l2 # L2 真实 KV +# bash scripts/run_benchmark.sh l4 # L4 replay (含中/高命中) +# bash scripts/run_benchmark.sh compare # 只读 CSV 出 TCP/RDMA 对比表 +# TRANSPORT=rdma MC_DEVICE=mlx5_3 bash scripts/run_benchmark.sh all +# +# 环境变量: +# TRANSPORT tcp(默认) | rdma —— 决定 L1/L2 走哪个协议 +# MC_DEVICE RDMA NIC 名, rdma 必填(prefill=mlx5_3); tcp 留空 +# KV_IP store 宿主机 IP (默认 192.0.2.10) +# HOST_IP 本 pod IP (默认 hostname -i) +# OUTDIR CSV 输出目录 (默认 results/raw) +# ============================================================================= +set -euo pipefail + +LAYER="${1:-all}" +export TRANSPORT="${TRANSPORT:-tcp}" +export MC_DEVICE="${MC_DEVICE:-}" +export KV_IP="${KV_IP:-192.0.2.10}" +export HOST_IP="${HOST_IP:-$(hostname -i 2>/dev/null | awk '{print $1}')}" + +# 定位仓库根 (scripts/ 的上一级) +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SELF/.." && pwd)" +OUTDIR="${OUTDIR:-$ROOT/results/raw}" +BM="$ROOT/benchmark" +mkdir -p "$OUTDIR" + +# rdma 协议下强制要求 device (否则 client 自动发现失败 ret=-600) +if [[ "$TRANSPORT" == "rdma" && -z "$MC_DEVICE" ]]; then + echo "[run_benchmark] TRANSPORT=rdma 但 MC_DEVICE 为空 —— prefill 端必须显式 MC_DEVICE=mlx5_3" >&2 + echo " (client device_name='' 自动发现会 No available RNIC ret=-600, 见 rdma/README.md)" >&2 + exit 2 +fi + +TAG="$TRANSPORT" +echo "[run_benchmark] layer=$LAYER transport=$TRANSPORT device='${MC_DEVICE}' KV_IP=$KV_IP HOST_IP=$HOST_IP outdir=$OUTDIR" + +run_l1() { + echo "== L1 transfer 原语微基准 ($TRANSPORT) ==" + python3 "$BM/benchmark_transfer.py" \ + --protocol "$TRANSPORT" --device "$MC_DEVICE" \ + --out "$OUTDIR/transfer_${TAG}.csv" +} + +run_l2() { + echo "== L2 真实 KV 张量微基准 ($TRANSPORT) ==" + python3 "$BM/benchmark_kv_tensor.py" \ + --protocol "$TRANSPORT" --device "$MC_DEVICE" \ + --paths bytes,zerocopy \ + --out "$OUTDIR/kv_tensor_${TAG}.csv" +} + +run_l4() { + echo "== L4 TTFT/QPM replay (协议无关, 吃 L1-L3 CSV) ==" + # 中等命中 (算力主导): h=0.5 phit=0.5 + python3 "$BM/benchmark_concurrency.py" --context 8k,16k,24k \ + --hit-prefix 0.5 --phit 0.5 --slo-factor 2.5 --out-prefix "$OUTDIR/conc" + # 高命中 (retrieve 主导): h=0.95 phit=0.8 + python3 "$BM/benchmark_concurrency.py" --context 8k,16k,24k \ + --hit-prefix 0.95 --phit 0.8 --slo-factor 2.5 --out-prefix "$OUTDIR/conc_hihit" +} + +# 只读已产出的 CSV 出 TCP vs RDMA 对比表 (不重跑, 便于本地汇总) +run_compare() { + echo "== TCP vs RDMA 对比 (读 $OUTDIR/*.csv) ==" + python3 - "$OUTDIR" <<'PY' +import csv, os, statistics, sys +outdir = sys.argv[1] +def med(path, pred, col): + if not os.path.exists(path): return None + vals=[] + with open(path) as f: + for r in csv.DictReader(f): + if pred(r): + try: vals.append(float(r[col])) + except (ValueError, KeyError): pass + return statistics.median(vals) if vals else None + +print("\n[L1 原语] put/get 有效带宽峰值 (GB/s):") +for tp in ("tcp","rdma"): + p=f"{outdir}/transfer_{tp}.csv" + if not os.path.exists(p): print(f" {tp}: (无 {p})"); continue + puts=med(p, lambda r:r.get("op")=="put" and r.get("success")=="1", "bandwidth_gbps") + gets=med(p, lambda r:r.get("op")=="get" and r.get("success")=="1", "bandwidth_gbps") + print(f" {tp:4}: put~{puts} get~{gets}") + +print("\n[L2 真实 KV] zerocopy get 中位带宽 (GB/s):") +z={} +for tp in ("tcp","rdma"): + p=f"{outdir}/kv_tensor_{tp}.csv" + z[tp]=med(p, lambda r:r.get("path")=="zerocopy" and r.get("op")=="get" and r.get("success")=="1", + "effective_bandwidth_gbps") + print(f" {tp:4}: zerocopy_get~{z[tp]}") +if z.get("tcp") and z.get("rdma"): + print(f" => RDMA-zc / TCP-zc = {z['rdma']/z['tcp']:.1f}x") + +print("\n[L4 QPM] 各 regime × 传输 (读 conc_summary / conc_hihit_summary):") +for tag,label in (("conc","中等命中 h=.5"),("conc_hihit","高命中 h=.95")): + p=f"{outdir}/{tag}_summary.csv" + if not os.path.exists(p): print(f" {label}: (无 {p})"); continue + byN={} + with open(p) as f: + for r in csv.DictReader(f): + byN.setdefault(int(r["context_length"]),{})[r["transport"]]=float(r["qpm"]) + print(f" {label}:") + for N,d in sorted(byN.items()): + rt = d["rdma"]/d["tcp"] if d.get("tcp") else float("nan") + print(f" N={N//1024}k: nocache={d.get('nocache')} tcp={d.get('tcp')} " + f"rdma={d.get('rdma')} rdma/tcp={rt:.2f}x") +PY +} + +case "$LAYER" in + l1) run_l1 ;; + l2) run_l2 ;; + l4) run_l4 ;; + compare) run_compare ;; + all) run_l1; run_l2; run_l4; run_compare ;; + *) echo "unknown layer '$LAYER' (l1|l2|l4|compare|all)"; exit 2 ;; +esac +echo "[run_benchmark] done: layer=$LAYER transport=$TRANSPORT" diff --git a/src/code/issue5/scripts/start_kvstore.sh b/src/code/issue5/scripts/start_kvstore.sh new file mode 100755 index 0000000..cb839b8 --- /dev/null +++ b/src/code/issue5/scripts/start_kvstore.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# ============================================================================= +# Issue 5 - 远端 KV 存储节点(Mooncake L3 Store)启动脚本 +# ----------------------------------------------------------------------------- +# 只起 mooncake_master + mooncake_store_service, 不跑模型 +# 用法: +# MC_PROTOCOL=rdma MC_SEGMENT_SIZE=200gb bash start_kvstore.sh +# +# 可选: +# MC_PROTOCOL rdma(默认) | tcp —— 需与 prefill 端一致 +# MC_SEGMENT_SIZE 贡献给全局 KV 池的本机内存, 默认 200gb, 按可用内存调整 +# +# 启动: kubectl get pod -o wide 拿到本 pod 宿主机 IP, +# 填到 prefill 端 start_prefill.sh 的 MOONCAKE_STORAGE_NODE_IP。 +# ============================================================================= +set -euo pipefail + +set +u +. ~/.bashrc || true +set -u + +export MC_CONDA_ENV="${MC_CONDA_ENV:-python310_torch29_cuda}" +export PATH="/root/miniconda/envs/${MC_CONDA_ENV}/bin:$PATH" + +export HOST_IP=$(hostname -i) +export MC_PROTOCOL="${MC_PROTOCOL:-rdma}" +export MC_SEGMENT_SIZE="${MC_SEGMENT_SIZE:-200gb}" +# metadata / store 端口(默认 18080/18081; 8080/8081 常被同宿主机其它 hostNetwork pod 占用) +# 注意: 改这里后 prefill 端 start_prefill.sh 的 MC_META_PORT 需保持一致 +export MC_META_PORT="${MC_META_PORT:-18080}" +export MC_STORE_PORT="${MC_STORE_PORT:-18081}" + +# mooncake rdma 网卡自动发现; 如需指定, 设 MC_MS_AUTO_DISC=0 并给 MOONCAKE_DEVICE 传网卡名 +export MC_MS_AUTO_DISC="${MC_MS_AUTO_DISC:-1}" + +echo "[issue5-kvstore] HOST_IP=${HOST_IP} protocol=${MC_PROTOCOL} segment=${MC_SEGMENT_SIZE} meta_port=${MC_META_PORT} store_port=${MC_STORE_PORT}" + +python3 -c "import mooncake; print('[issue5-kvstore] mooncake OK:', mooncake.__file__)" + +# ---- 1) 启动 master(内嵌 http metadata server: ${MC_META_PORT}; master RPC: 50051)---- +mooncake_master \ + --enable_http_metadata_server=true \ + --http_metadata_server_port=${MC_META_PORT} \ + --eviction_high_watermark_ratio=0.95 \ + > /tmp/mooncake_master.log 2>&1 & +MASTER_PID=$! +echo "[issue5-kvstore] mooncake_master started, pid=${MASTER_PID}, log=/tmp/mooncake_master.log" +sleep 8 + +# ---- 2) 启动 store service, 贡献本机内存到全局 KV 池 ---- +# 本镜像里的 mooncake 0.3.5 store_service 不再读 MOONCAKE_* 环境变量, +# 而是从 --config (或 MOONCAKE_CONFIG_PATH) 读配置, 可用 -D key=value 覆盖。 +# 纯存储节点不发起 get/put, 故 local_buffer_size=0。 +# global_segment_size 需为字节整数, 这里把 MC_SEGMENT_SIZE(如 200gb) 换算成字节。 +seg_bytes() { + local s="${1,,}"; local n="${s//[a-z]/}" + case "$s" in + *tb) echo $(( n * 1024 * 1024 * 1024 * 1024 ));; + *gb) echo $(( n * 1024 * 1024 * 1024 ));; + *mb) echo $(( n * 1024 * 1024 ));; + *) echo "$n";; + esac +} +MC_SEGMENT_BYTES=$(seg_bytes "$MC_SEGMENT_SIZE") + +MC_CFG=/tmp/mooncake_kvstore.json +cat > "$MC_CFG" < $MC_CFG (segment=${MC_SEGMENT_BYTES} bytes):" +cat "$MC_CFG" + +python3 -m mooncake.mooncake_store_service --config "$MC_CFG" --port=${MC_STORE_PORT} diff --git a/src/code/issue5/scripts/start_prefill_v32.sh b/src/code/issue5/scripts/start_prefill_v32.sh new file mode 100644 index 0000000..368daec --- /dev/null +++ b/src/code/issue5/scripts/start_prefill_v32.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# ============================================================================= +# Issue 5 - 计算节点(Prefill)启动脚本 —— DeepSeek-V3.2 (MLA + NSA/DSA) 版 +# ----------------------------------------------------------------------------- +# DeepSeek-V3.2(DeepseekV32ForCausalLM, MLA + DSA +# 稀疏索引), 其 KV 池是 NSATokenToKVPool —— HiRadixCache 明确支持 +# (mem_cache/hiradix_cache.py: isinstance(..., NSATokenToKVPool) -> NSATokenToKVPoolHost)。 +# +# 用法: +# MOONCAKE_STORAGE_NODE_IP=<存储节点宿主机IP> MC_PROTOCOL=tcp bash start_prefill_v32.sh +# ============================================================================= +set -euo pipefail + +# ---- 参数校验 ---- +if [[ -z "${MOONCAKE_STORAGE_NODE_IP:-}" ]]; then + echo "[ERROR] 请先设置 MOONCAKE_STORAGE_NODE_IP=<存储节点宿主机IP>" >&2 + exit 1 +fi +export MC_PROTOCOL="${MC_PROTOCOL:-tcp}" +export MC_META_PORT="${MC_META_PORT:-18080}" +echo "[issue5-prefill-v32] storage_ip=${MOONCAKE_STORAGE_NODE_IP} protocol=${MC_PROTOCOL} meta_port=${MC_META_PORT}" + +# ---- 基础环境 ---- +set +u +. ~/.bashrc || true +set -u + +# conda env: py3.10 + torch2.9, 预装 mooncake 0.3.5 + sglang(与镜像 glibc2.31 匹配) +export MC_CONDA_ENV="${MC_CONDA_ENV:-python310_torch29_cuda}" +export PATH="/root/miniconda/envs/${MC_CONDA_ENV}/bin:$PATH" + +unset XPU_DUMMY_EVENT || true +export PYTHONPATH=/workspace/sglang/python:/workspace/sglang/sgl-kernel/python:${PYTHONPATH:-} +# DeepSeek-V3.2 int8(w8a8) 权重; 与 --quantization w8a8_int8 匹配。 + +export MODEL_PATH=/root/v32_int8_fix/ + +export SGLANG_APPLY_CONFIG_BACKUP=none +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export MODEL_ID="${MODEL_ID:-p800_deepseek_v32_issue5}" + +export SGLANG_OPT_USE_TILELANG_MHC_PRE=0 +export SGLANG_OPT_DEEPGEMM_HC_PRENORM=0 +export SGLANG_OPT_USE_TILELANG_MHC_POST=0 +export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=0 +export SGLANG_IS_FLASHINFER_AVAILABLE=False +export USE_FAST_ALLOC_EXTEND_KUNLUN=False +export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=False +export SGLANG_OPT_USE_JIT_NORM=True +export SGLANG_OPT_USE_FUSED_STORE_CACHE=0 +export SGLANG_OPT_FP8_WO_A_GEMM=False +export SGLANG_OPT_BF16_FP32_GEMM_ALGO="torch" +export SGLANG_TOPK_TRANSFORM_512_TORCH=False +export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 +export SGLANG_PREP_IN_CUDA_GRAPH=False +export SGLANG_OPT_CP_REARRANGE_TRITON=False +export BKCL_RDMA_VERBS=1 +export BKCL_ENABLE_XDR=1 +export BKCL_RDMA_NICS="eth1,eth1,eth2,eth2,eth3,eth3,eth4,eth4" +export NCCL_IB_GID_INDEX=3 +export XSHMEM_QP_NUM_PER_RANK=32 +export XSHMEM_MODE=1 +export XSHMEM_SYMMETRIC_SIZE=4294967296 +export CUDA_DEVICE_ORDER=OAM_ID +export CUDA_ENABLE_P2P_NO_UVA=1 +export OPS_DEBUG_CHECK=1 +export SGLANG_TOOL_STRICT_LEVEL=2 + +export HOST_IP=$(hostname -i) +LOG_DIR=logs_issue5_prefill_v32 +mkdir -p "$LOG_DIR" + +# ---- 依赖(镜像若已装可注释掉)---- +pip3 install "cocopod-1.3.0.torch29-cp310-cp310-linux_x86_64.whl" --force-reinstall + +python3 -m sglang.launch_server \ + --model-path "$MODEL_PATH" \ + --tp 8 \ + --ep 8 \ + --port 8000 \ + --quantization w8a8_int8 \ + --trust-remote-code \ + --kv-cache-dtype bfloat16 \ + --dtype bfloat16 \ + --mem-fraction-static 0.83 \ + --chunked-prefill-size 2048 \ + --max-running-requests 4 \ + --max-prefill-tokens 16384 \ + --disable-cuda-graph \ + --skip-server-warmup \ + --model-loader-extra-config '{"enable_multithread_load": "true","num_threads": 64}' \ + --enable-metrics --collect-tokens-histogram \ + --host "$HOST_IP" \ + --allow-auto-truncate \ + --moe-a2a-backend deepep \ + --deepep-mode normal \ + --served-model-name "$MODEL_ID" \ + --enable-cache-report \ + --page-size 64 \ + --enable-hierarchical-cache \ + --disable-hicache-numa-detect \ + --hicache-storage-backend mooncake \ + --hicache-mem-layout page_first \ + --hicache-write-policy write_through \ + --hicache-storage-prefetch-policy timeout \ + --hicache-ratio 2 \ + --hicache-storage-backend-extra-config "$HICACHE_MOONCAKE_CFG"