[issue5] KV Cache 通信临界带宽 / TTFT / QPM 上界模型与扫描报告

src/code/issue5/README.md

@@ -0,0 +1,20 @@
+# Issue 5 单机可完成部分：KV Cache 临界带宽模型
+
+模型使用：
+
+```text
+T_hybrid = lookup_latency + hit_bytes / bandwidth + (1-hit_rate) * T_full
+```
+
+由此直接求出使“以存代算”严格优于全量重算的临界带宽，并同时估算 TCP/RDMA
+下的 TTFT 与 QPM 上界。
+
+```bash
+python -m pytest src/code/issue5/test_kv_cache_break_even.py
+python src/code/issue5/kv_cache_break_even.py
+```
+
+默认扫描 8K/32K/128K 上下文和 10%/30%/50%/80% 命中率，报告写入 `results/`。
+默认 KV 大小、重算时间、TCP/RDMA 带宽只是可替换示例参数；多机阶段需要用实测值
+重跑，并接入真实 RDMA transport 才能完成该 issue 的第二项验收。
+

src/code/issue5/kv_cache_break_even.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Bandwidth break-even model for remote KV-cache reuse during prefill."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import math
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Scenario:
+    context_tokens: int
+    hit_rate: float
+    kv_bytes_per_token: float
+    full_recompute_ms: float
+
+    def __post_init__(self) -> None:
+        if self.context_tokens <= 0 or self.kv_bytes_per_token <= 0 or self.full_recompute_ms <= 0:
+            raise ValueError("token count, KV bytes/token, and recompute time must be positive")
+        if not 0.0 <= self.hit_rate <= 1.0:
+            raise ValueError("hit rate must be in [0, 1]")
+
+    @property
+    def hit_bytes(self) -> float:
+        return self.context_tokens * self.hit_rate * self.kv_bytes_per_token
+
+    @property
+    def remaining_compute_ms(self) -> float:
+        return (1.0 - self.hit_rate) * self.full_recompute_ms
+
+    @property
+    def saved_compute_ms(self) -> float:
+        return self.hit_rate * self.full_recompute_ms
+
+
+@dataclass(frozen=True)
+class Transport:
+    name: str
+    bandwidth_gbytes_s: float
+    lookup_latency_ms: float
+
+    def __post_init__(self) -> None:
+        if self.bandwidth_gbytes_s <= 0 or self.lookup_latency_ms < 0:
+            raise ValueError("transport bandwidth must be positive and latency non-negative")
+
+
+def critical_bandwidth_gbytes_s(scenario: Scenario, lookup_latency_ms: float) -> float:
+    """Return the strict break-even bandwidth for hybrid reuse.
+
+    Hybrid TTFT = lookup + transfer + unhit recompute.  It beats full
+    recompute exactly when transfer < hit_rate * full_recompute - lookup.
+    """
+
+    time_budget_ms = scenario.saved_compute_ms - lookup_latency_ms
+    if scenario.hit_bytes == 0:
+        return math.inf
+    if time_budget_ms <= 0:
+        return math.inf
+    return scenario.hit_bytes / (time_budget_ms / 1_000.0) / 1e9
+
+
+def evaluate_transport(scenario: Scenario, transport: Transport) -> dict[str, Any]:
+    transfer_ms = scenario.hit_bytes / (transport.bandwidth_gbytes_s * 1e9) * 1_000.0
+    hybrid_ttft_ms = transport.lookup_latency_ms + transfer_ms + scenario.remaining_compute_ms
+    full_ttft_ms = scenario.full_recompute_ms
+    critical = critical_bandwidth_gbytes_s(scenario, transport.lookup_latency_ms)
+
+    # Capacity ceilings assume the network and prefill compute engine can be
+    # independently pipelined.  They are bounds, not a queueing simulation.
+    network_qpm = (
+        math.inf if scenario.hit_bytes == 0 else transport.bandwidth_gbytes_s * 1e9 * 60 / scenario.hit_bytes
+    )
+    compute_qpm = (
+        math.inf
+        if scenario.remaining_compute_ms == 0
+        else 60_000.0 / scenario.remaining_compute_ms
+    )
+    hybrid_qpm_bound = min(network_qpm, compute_qpm)
+    full_recompute_qpm_bound = 60_000.0 / full_ttft_ms
+    return {
+        "transport": asdict(transport),
+        "hit_bytes": scenario.hit_bytes,
+        "transfer_ms": transfer_ms,
+        "remaining_compute_ms": scenario.remaining_compute_ms,
+        "hybrid_ttft_ms": hybrid_ttft_ms,
+        "full_recompute_ttft_ms": full_ttft_ms,
+        "ttft_saved_ms": full_ttft_ms - hybrid_ttft_ms,
+        "ttft_saved_ratio": 1.0 - hybrid_ttft_ms / full_ttft_ms,
+        "critical_bandwidth_gbytes_s": critical,
+        "is_beneficial": hybrid_ttft_ms < full_ttft_ms,
+        "network_qpm_upper_bound": network_qpm,
+        "compute_qpm_upper_bound": compute_qpm,
+        "hybrid_qpm_upper_bound": hybrid_qpm_bound,
+        "full_recompute_qpm_upper_bound": full_recompute_qpm_bound,
+        "qpm_bound_ratio": hybrid_qpm_bound / full_recompute_qpm_bound,
+    }
+
+
+def parse_int_list(value: str) -> list[int]:
+    result = [int(part) for part in value.split(",")]
+    if not result or any(item <= 0 for item in result):
+        raise argparse.ArgumentTypeError("values must be positive comma-separated integers")
+    return result
+
+
+def parse_float_list(value: str) -> list[float]:
+    result = [float(part) for part in value.split(",")]
+    if not result or any(not 0 <= item <= 1 for item in result):
+        raise argparse.ArgumentTypeError("hit rates must be comma-separated values in [0, 1]")
+    return result
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Model remote KV-cache bandwidth break-even")
+    parser.add_argument("--contexts", type=parse_int_list, default=[8192, 32768, 131072])
+    parser.add_argument("--hit-rates", type=parse_float_list, default=[0.1, 0.3, 0.5, 0.8])
+    parser.add_argument(
+        "--kv-bytes-per-token",
+        type=float,
+        default=327680.0,
+        help="all-layer KV footprint per token (default: 320 KiB)",
+    )
+    parser.add_argument(
+        "--recompute-ms-per-1k-tokens",
+        type=float,
+        default=12.0,
+        help="linearized full-prefill baseline used for the example sweep",
+    )
+    parser.add_argument("--tcp-gbytes-s", type=float, default=2.5)
+    parser.add_argument("--tcp-latency-ms", type=float, default=1.0)
+    parser.add_argument("--rdma-gbytes-s", type=float, default=25.0)
+    parser.add_argument("--rdma-latency-ms", type=float, default=0.1)
+    parser.add_argument("--output-dir", type=Path, default=Path("src/code/issue5/results"))
+    return parser
+
+
+def build_report(args: argparse.Namespace) -> dict[str, Any]:
+    transports = [
+        Transport("TCP", args.tcp_gbytes_s, args.tcp_latency_ms),
+        Transport("RDMA", args.rdma_gbytes_s, args.rdma_latency_ms),
+    ]
+    scenarios = []
+    for context in args.contexts:
+        full_recompute_ms = context / 1_000.0 * args.recompute_ms_per_1k_tokens
+        for hit_rate in args.hit_rates:
+            scenario = Scenario(context, hit_rate, args.kv_bytes_per_token, full_recompute_ms)
+            scenarios.append(
+                {
+                    "scenario": asdict(scenario),
+                    "transports": [evaluate_transport(scenario, transport) for transport in transports],
+                }
+            )
+    return {
+        "schema_version": 1,
+        "model": "T_hybrid = lookup_latency + hit_bytes / bandwidth + (1-hit_rate) * T_full",
+        "assumptions": [
+            "Prefill recompute time scales linearly with the unhit token fraction.",
+            "KV lookup/transfer and remaining recompute are serialized for TTFT.",
+            "QPM values are independent-resource upper bounds, not queueing predictions.",
+        ],
+        "inputs": {
+            "kv_bytes_per_token": args.kv_bytes_per_token,
+            "recompute_ms_per_1k_tokens": args.recompute_ms_per_1k_tokens,
+        },
+        "scenarios": scenarios,
+    }
+
+
+def write_csv(report: dict[str, Any], path: Path) -> None:
+    fields = [
+        "context_tokens",
+        "hit_rate",
+        "full_recompute_ms",
+        "transport",
+        "bandwidth_gbytes_s",
+        "critical_bandwidth_gbytes_s",
+        "transfer_ms",
+        "hybrid_ttft_ms",
+        "ttft_saved_ratio",
+        "is_beneficial",
+        "hybrid_qpm_upper_bound",
+        "qpm_bound_ratio",
+    ]
+    with path.open("w", encoding="utf-8-sig", newline="") as handle:
+        writer = csv.DictWriter(handle, fields)
+        writer.writeheader()
+        for item in report["scenarios"]:
+            scenario = item["scenario"]
+            for result in item["transports"]:
+                writer.writerow(
+                    {
+                        "context_tokens": scenario["context_tokens"],
+                        "hit_rate": scenario["hit_rate"],
+                        "full_recompute_ms": scenario["full_recompute_ms"],
+                        "transport": result["transport"]["name"],
+                        "bandwidth_gbytes_s": result["transport"]["bandwidth_gbytes_s"],
+                        "critical_bandwidth_gbytes_s": result["critical_bandwidth_gbytes_s"],
+                        "transfer_ms": result["transfer_ms"],
+                        "hybrid_ttft_ms": result["hybrid_ttft_ms"],
+                        "ttft_saved_ratio": result["ttft_saved_ratio"],
+                        "is_beneficial": result["is_beneficial"],
+                        "hybrid_qpm_upper_bound": result["hybrid_qpm_upper_bound"],
+                        "qpm_bound_ratio": result["qpm_bound_ratio"],
+                    }
+                )
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+    report = build_report(args)
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    json_path = args.output_dir / "kv_cache_break_even.json"
+    csv_path = args.output_dir / "kv_cache_break_even.csv"
+    json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    write_csv(report, csv_path)
+    print(f"Wrote {json_path}")
+    print(f"Wrote {csv_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/code/issue5/results/kv_cache_break_even.csv

@@ -0,0 +1,25 @@
+context_tokens,hit_rate,full_recompute_ms,transport,bandwidth_gbytes_s,critical_bandwidth_gbytes_s,transfer_ms,hybrid_ttft_ms,ttft_saved_ratio,is_beneficial,hybrid_qpm_upper_bound,qpm_bound_ratio
+8192,0.1,98.304,TCP,2.5,30.39901431418735,107.37418240000001,196.84778240000003,-1.0024391927083336,False,558.7935447692871,0.91552734375
+8192,0.1,98.304,RDMA,25.0,27.587299186056065,10.73741824,99.31101824000001,-0.010243919270833457,False,678.1684027777777,1.111111111111111
+8192,0.3,98.304,TCP,2.5,28.265091256247548,322.1225472,391.93534719999997,-2.9869725260416664,False,186.2645149230957,0.30517578125
+8192,0.3,98.304,RDMA,25.0,27.399574294346607,32.21225472,101.12505472,-0.028697252604166668,False,871.9308035714287,1.4285714285714288
+8192,0.5,98.304,TCP,2.5,27.8737597607576,536.8709120000001,587.0229120000001,-4.971505859375001,False,111.75870895385742,0.18310546875
+8192,0.5,98.304,RDMA,25.0,27.362335480714343,53.687091200000005,102.93909120000001,-0.0471505859375001,False,1117.5870895385742,1.8310546875
+8192,0.8,98.304,TCP,2.5,27.658360912481708,858.9934592000001,879.6542592000001,-7.948305859375001,False,69.84919309616089,0.11444091796875
+8192,0.8,98.304,RDMA,25.0,27.341433096690736,85.89934592,105.66014591999999,-0.0748305859374998,False,698.4919309616089,1.1444091796875
+32768,0.1,393.216,TCP,2.5,28.019232599891442,429.49672960000004,784.3911296000001,-0.9948097981770836,False,139.69838619232178,0.91552734375
+32768,0.1,393.216,RDMA,25.0,27.3762881677443,42.94967296,396.94407296,-0.009480979817708457,False,169.54210069444443,1.111111111111111
+32768,0.3,393.216,TCP,2.5,27.540127217761242,1288.4901888,1564.7413887999999,-2.9793431315104164,False,46.566128730773926,0.30517578125
+32768,0.3,393.216,RDMA,25.0,27.32983445439181,128.84901888,404.20021887999997,-0.027934313151041668,False,217.98270089285717,1.4285714285714288
+32768,0.5,393.216,TCP,2.5,27.446265592409308,2147.4836480000004,2345.0916480000005,-4.963876464843751,False,27.939677238464355,0.18310546875
+32768,0.5,393.216,RDMA,25.0,27.320562623404644,214.74836480000002,411.4563648,-0.0463876464843751,False,279.39677238464355,1.8310546875
+32768,0.8,393.216,TCP,2.5,27.393749049662468,3435.9738368000003,3515.6170368000003,-7.940676464843751,False,17.462298274040222,0.11444091796875
+32768,0.8,393.216,RDMA,25.0,27.31534998257401,343.59738368,422.34058368,-0.07406764648437503,False,174.62298274040222,1.1444091796875
+131072,0.1,1572.864,TCP,2.5,27.48138863010473,1717.9869184000001,3134.5645184000005,-0.9929024495442711,False,34.924596548080444,0.91552734375
+131072,0.1,1572.864,RDMA,25.0,27.324038822697126,171.79869184,1587.4762918400002,-0.009290244954427207,False,42.38552517361111,1.111111111111111
+131072,0.3,1572.864,TCP,2.5,27.364659940806085,5153.9607552,6255.9655551999995,-2.977435782877604,False,11.641532182693481,0.30517578125
+131072,0.3,1572.864,RDMA,25.0,27.312454930396694,515.39607552,1616.50087552,-0.027743578287760418,False,54.49567522321429,1.4285714285714288
+131072,0.5,1572.864,TCP,2.5,27.34143309669074,8589.934592000001,9377.366592000002,-4.9619691162109385,False,6.984919309616089,0.18310546875
+131072,0.5,1572.864,RDMA,25.0,27.310139330460924,858.9934592000001,1645.5254592000001,-0.04619691162109385,False,69.84919309616089,1.8310546875
+131072,0.8,1572.864,TCP,2.5,27.32838531598726,13743.895347200001,14059.468147200001,-7.938769116210938,False,4.3655745685100555,0.11444091796875
+131072,0.8,1572.864,RDMA,25.0,27.308836978036403,1374.38953472,1689.0623347199999,-0.07387691162109356,False,43.655745685100555,1.1444091796875