From 5cded5b12da86a103fc00ac97dd0802e24da7b92 Mon Sep 17 00:00:00 2001 From: shsaihdsaiudh <196440533+shsaihdsaiudh@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:14:40 +0800 Subject: [PATCH] feat(issue5): KV cache store-vs-recompute critical bandwidth model --- src/code/issue5/README.md | 20 + src/code/issue5/kv_cache_break_even.py | 227 +++++++ .../issue5/results/kv_cache_break_even.csv | 25 + .../issue5/results/kv_cache_break_even.json | 639 ++++++++++++++++++ src/code/issue5/test_kv_cache_break_even.py | 31 + 5 files changed, 942 insertions(+) create mode 100644 src/code/issue5/README.md create mode 100644 src/code/issue5/kv_cache_break_even.py create mode 100644 src/code/issue5/results/kv_cache_break_even.csv create mode 100644 src/code/issue5/results/kv_cache_break_even.json create mode 100644 src/code/issue5/test_kv_cache_break_even.py diff --git a/src/code/issue5/README.md b/src/code/issue5/README.md new file mode 100644 index 0000000..4088c3a --- /dev/null +++ b/src/code/issue5/README.md @@ -0,0 +1,20 @@ +# Issue 5 单机可完成部分：KV Cache 临界带宽模型 + +模型使用： + +```text +T_hybrid = lookup_latency + hit_bytes / bandwidth + (1-hit_rate) * T_full +``` + +由此直接求出使“以存代算”严格优于全量重算的临界带宽，并同时估算 TCP/RDMA +下的 TTFT 与 QPM 上界。 + +```bash +python -m pytest src/code/issue5/test_kv_cache_break_even.py +python src/code/issue5/kv_cache_break_even.py +``` + +默认扫描 8K/32K/128K 上下文和 10%/30%/50%/80% 命中率，报告写入 `results/`。 +默认 KV 大小、重算时间、TCP/RDMA 带宽只是可替换示例参数；多机阶段需要用实测值 +重跑，并接入真实 RDMA transport 才能完成该 issue 的第二项验收。 + diff --git a/src/code/issue5/kv_cache_break_even.py b/src/code/issue5/kv_cache_break_even.py new file mode 100644 index 0000000..9588dd7 --- /dev/null +++ b/src/code/issue5/kv_cache_break_even.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Bandwidth break-even model for remote KV-cache reuse during prefill.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class Scenario: + context_tokens: int + hit_rate: float + kv_bytes_per_token: float + full_recompute_ms: float + + def __post_init__(self) -> None: + if self.context_tokens <= 0 or self.kv_bytes_per_token <= 0 or self.full_recompute_ms <= 0: + raise ValueError("token count, KV bytes/token, and recompute time must be positive") + if not 0.0 <= self.hit_rate <= 1.0: + raise ValueError("hit rate must be in [0, 1]") + + @property + def hit_bytes(self) -> float: + return self.context_tokens * self.hit_rate * self.kv_bytes_per_token + + @property + def remaining_compute_ms(self) -> float: + return (1.0 - self.hit_rate) * self.full_recompute_ms + + @property + def saved_compute_ms(self) -> float: + return self.hit_rate * self.full_recompute_ms + + +@dataclass(frozen=True) +class Transport: + name: str + bandwidth_gbytes_s: float + lookup_latency_ms: float + + def __post_init__(self) -> None: + if self.bandwidth_gbytes_s <= 0 or self.lookup_latency_ms < 0: + raise ValueError("transport bandwidth must be positive and latency non-negative") + + +def critical_bandwidth_gbytes_s(scenario: Scenario, lookup_latency_ms: float) -> float: + """Return the strict break-even bandwidth for hybrid reuse. + + Hybrid TTFT = lookup + transfer + unhit recompute. It beats full + recompute exactly when transfer < hit_rate * full_recompute - lookup. + """ + + time_budget_ms = scenario.saved_compute_ms - lookup_latency_ms + if scenario.hit_bytes == 0: + return math.inf + if time_budget_ms <= 0: + return math.inf + return scenario.hit_bytes / (time_budget_ms / 1_000.0) / 1e9 + + +def evaluate_transport(scenario: Scenario, transport: Transport) -> dict[str, Any]: + transfer_ms = scenario.hit_bytes / (transport.bandwidth_gbytes_s * 1e9) * 1_000.0 + hybrid_ttft_ms = transport.lookup_latency_ms + transfer_ms + scenario.remaining_compute_ms + full_ttft_ms = scenario.full_recompute_ms + critical = critical_bandwidth_gbytes_s(scenario, transport.lookup_latency_ms) + + # Capacity ceilings assume the network and prefill compute engine can be + # independently pipelined. They are bounds, not a queueing simulation. + network_qpm = ( + math.inf if scenario.hit_bytes == 0 else transport.bandwidth_gbytes_s * 1e9 * 60 / scenario.hit_bytes + ) + compute_qpm = ( + math.inf + if scenario.remaining_compute_ms == 0 + else 60_000.0 / scenario.remaining_compute_ms + ) + hybrid_qpm_bound = min(network_qpm, compute_qpm) + full_recompute_qpm_bound = 60_000.0 / full_ttft_ms + return { + "transport": asdict(transport), + "hit_bytes": scenario.hit_bytes, + "transfer_ms": transfer_ms, + "remaining_compute_ms": scenario.remaining_compute_ms, + "hybrid_ttft_ms": hybrid_ttft_ms, + "full_recompute_ttft_ms": full_ttft_ms, + "ttft_saved_ms": full_ttft_ms - hybrid_ttft_ms, + "ttft_saved_ratio": 1.0 - hybrid_ttft_ms / full_ttft_ms, + "critical_bandwidth_gbytes_s": critical, + "is_beneficial": hybrid_ttft_ms < full_ttft_ms, + "network_qpm_upper_bound": network_qpm, + "compute_qpm_upper_bound": compute_qpm, + "hybrid_qpm_upper_bound": hybrid_qpm_bound, + "full_recompute_qpm_upper_bound": full_recompute_qpm_bound, + "qpm_bound_ratio": hybrid_qpm_bound / full_recompute_qpm_bound, + } + + +def parse_int_list(value: str) -> list[int]: + result = [int(part) for part in value.split(",")] + if not result or any(item <= 0 for item in result): + raise argparse.ArgumentTypeError("values must be positive comma-separated integers") + return result + + +def parse_float_list(value: str) -> list[float]: + result = [float(part) for part in value.split(",")] + if not result or any(not 0 <= item <= 1 for item in result): + raise argparse.ArgumentTypeError("hit rates must be comma-separated values in [0, 1]") + return result + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Model remote KV-cache bandwidth break-even") + parser.add_argument("--contexts", type=parse_int_list, default=[8192, 32768, 131072]) + parser.add_argument("--hit-rates", type=parse_float_list, default=[0.1, 0.3, 0.5, 0.8]) + parser.add_argument( + "--kv-bytes-per-token", + type=float, + default=327680.0, + help="all-layer KV footprint per token (default: 320 KiB)", + ) + parser.add_argument( + "--recompute-ms-per-1k-tokens", + type=float, + default=12.0, + help="linearized full-prefill baseline used for the example sweep", + ) + parser.add_argument("--tcp-gbytes-s", type=float, default=2.5) + parser.add_argument("--tcp-latency-ms", type=float, default=1.0) + parser.add_argument("--rdma-gbytes-s", type=float, default=25.0) + parser.add_argument("--rdma-latency-ms", type=float, default=0.1) + parser.add_argument("--output-dir", type=Path, default=Path("src/code/issue5/results")) + return parser + + +def build_report(args: argparse.Namespace) -> dict[str, Any]: + transports = [ + Transport("TCP", args.tcp_gbytes_s, args.tcp_latency_ms), + Transport("RDMA", args.rdma_gbytes_s, args.rdma_latency_ms), + ] + scenarios = [] + for context in args.contexts: + full_recompute_ms = context / 1_000.0 * args.recompute_ms_per_1k_tokens + for hit_rate in args.hit_rates: + scenario = Scenario(context, hit_rate, args.kv_bytes_per_token, full_recompute_ms) + scenarios.append( + { + "scenario": asdict(scenario), + "transports": [evaluate_transport(scenario, transport) for transport in transports], + } + ) + return { + "schema_version": 1, + "model": "T_hybrid = lookup_latency + hit_bytes / bandwidth + (1-hit_rate) * T_full", + "assumptions": [ + "Prefill recompute time scales linearly with the unhit token fraction.", + "KV lookup/transfer and remaining recompute are serialized for TTFT.", + "QPM values are independent-resource upper bounds, not queueing predictions.", + ], + "inputs": { + "kv_bytes_per_token": args.kv_bytes_per_token, + "recompute_ms_per_1k_tokens": args.recompute_ms_per_1k_tokens, + }, + "scenarios": scenarios, + } + + +def write_csv(report: dict[str, Any], path: Path) -> None: + fields = [ + "context_tokens", + "hit_rate", + "full_recompute_ms", + "transport", + "bandwidth_gbytes_s", + "critical_bandwidth_gbytes_s", + "transfer_ms", + "hybrid_ttft_ms", + "ttft_saved_ratio", + "is_beneficial", + "hybrid_qpm_upper_bound", + "qpm_bound_ratio", + ] + with path.open("w", encoding="utf-8-sig", newline="") as handle: + writer = csv.DictWriter(handle, fields) + writer.writeheader() + for item in report["scenarios"]: + scenario = item["scenario"] + for result in item["transports"]: + writer.writerow( + { + "context_tokens": scenario["context_tokens"], + "hit_rate": scenario["hit_rate"], + "full_recompute_ms": scenario["full_recompute_ms"], + "transport": result["transport"]["name"], + "bandwidth_gbytes_s": result["transport"]["bandwidth_gbytes_s"], + "critical_bandwidth_gbytes_s": result["critical_bandwidth_gbytes_s"], + "transfer_ms": result["transfer_ms"], + "hybrid_ttft_ms": result["hybrid_ttft_ms"], + "ttft_saved_ratio": result["ttft_saved_ratio"], + "is_beneficial": result["is_beneficial"], + "hybrid_qpm_upper_bound": result["hybrid_qpm_upper_bound"], + "qpm_bound_ratio": result["qpm_bound_ratio"], + } + ) + + +def main() -> int: + args = build_parser().parse_args() + report = build_report(args) + args.output_dir.mkdir(parents=True, exist_ok=True) + json_path = args.output_dir / "kv_cache_break_even.json" + csv_path = args.output_dir / "kv_cache_break_even.csv" + json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + write_csv(report, csv_path) + print(f"Wrote {json_path}") + print(f"Wrote {csv_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/code/issue5/results/kv_cache_break_even.csv b/src/code/issue5/results/kv_cache_break_even.csv new file mode 100644 index 0000000..9977cdf --- /dev/null +++ b/src/code/issue5/results/kv_cache_break_even.csv @@ -0,0 +1,25 @@ +context_tokens,hit_rate,full_recompute_ms,transport,bandwidth_gbytes_s,critical_bandwidth_gbytes_s,transfer_ms,hybrid_ttft_ms,ttft_saved_ratio,is_beneficial,hybrid_qpm_upper_bound,qpm_bound_ratio +8192,0.1,98.304,TCP,2.5,30.39901431418735,107.37418240000001,196.84778240000003,-1.0024391927083336,False,558.7935447692871,0.91552734375 +8192,0.1,98.304,RDMA,25.0,27.587299186056065,10.73741824,99.31101824000001,-0.010243919270833457,False,678.1684027777777,1.111111111111111 +8192,0.3,98.304,TCP,2.5,28.265091256247548,322.1225472,391.93534719999997,-2.9869725260416664,False,186.2645149230957,0.30517578125 +8192,0.3,98.304,RDMA,25.0,27.399574294346607,32.21225472,101.12505472,-0.028697252604166668,False,871.9308035714287,1.4285714285714288 +8192,0.5,98.304,TCP,2.5,27.8737597607576,536.8709120000001,587.0229120000001,-4.971505859375001,False,111.75870895385742,0.18310546875 +8192,0.5,98.304,RDMA,25.0,27.362335480714343,53.687091200000005,102.93909120000001,-0.0471505859375001,False,1117.5870895385742,1.8310546875 +8192,0.8,98.304,TCP,2.5,27.658360912481708,858.9934592000001,879.6542592000001,-7.948305859375001,False,69.84919309616089,0.11444091796875 +8192,0.8,98.304,RDMA,25.0,27.341433096690736,85.89934592,105.66014591999999,-0.0748305859374998,False,698.4919309616089,1.1444091796875 +32768,0.1,393.216,TCP,2.5,28.019232599891442,429.49672960000004,784.3911296000001,-0.9948097981770836,False,139.69838619232178,0.91552734375 +32768,0.1,393.216,RDMA,25.0,27.3762881677443,42.94967296,396.94407296,-0.009480979817708457,False,169.54210069444443,1.111111111111111 +32768,0.3,393.216,TCP,2.5,27.540127217761242,1288.4901888,1564.7413887999999,-2.9793431315104164,False,46.566128730773926,0.30517578125 +32768,0.3,393.216,RDMA,25.0,27.32983445439181,128.84901888,404.20021887999997,-0.027934313151041668,False,217.98270089285717,1.4285714285714288 +32768,0.5,393.216,TCP,2.5,27.446265592409308,2147.4836480000004,2345.0916480000005,-4.963876464843751,False,27.939677238464355,0.18310546875 +32768,0.5,393.216,RDMA,25.0,27.320562623404644,214.74836480000002,411.4563648,-0.0463876464843751,False,279.39677238464355,1.8310546875 +32768,0.8,393.216,TCP,2.5,27.393749049662468,3435.9738368000003,3515.6170368000003,-7.940676464843751,False,17.462298274040222,0.11444091796875 +32768,0.8,393.216,RDMA,25.0,27.31534998257401,343.59738368,422.34058368,-0.07406764648437503,False,174.62298274040222,1.1444091796875 +131072,0.1,1572.864,TCP,2.5,27.48138863010473,1717.9869184000001,3134.5645184000005,-0.9929024495442711,False,34.924596548080444,0.91552734375 +131072,0.1,1572.864,RDMA,25.0,27.324038822697126,171.79869184,1587.4762918400002,-0.009290244954427207,False,42.38552517361111,1.111111111111111 +131072,0.3,1572.864,TCP,2.5,27.364659940806085,5153.9607552,6255.9655551999995,-2.977435782877604,False,11.641532182693481,0.30517578125 +131072,0.3,1572.864,RDMA,25.0,27.312454930396694,515.39607552,1616.50087552,-0.027743578287760418,False,54.49567522321429,1.4285714285714288 +131072,0.5,1572.864,TCP,2.5,27.34143309669074,8589.934592000001,9377.366592000002,-4.9619691162109385,False,6.984919309616089,0.18310546875 +131072,0.5,1572.864,RDMA,25.0,27.310139330460924,858.9934592000001,1645.5254592000001,-0.04619691162109385,False,69.84919309616089,1.8310546875 +131072,0.8,1572.864,TCP,2.5,27.32838531598726,13743.895347200001,14059.468147200001,-7.938769116210938,False,4.3655745685100555,0.11444091796875 +131072,0.8,1572.864,RDMA,25.0,27.308836978036403,1374.38953472,1689.0623347199999,-0.07387691162109356,False,43.655745685100555,1.1444091796875 diff --git a/src/code/issue5/results/kv_cache_break_even.json b/src/code/issue5/results/kv_cache_break_even.json new file mode 100644 index 0000000..34edf16 --- /dev/null +++ b/src/code/issue5/results/kv_cache_break_even.json @@ -0,0 +1,639 @@ +{ + "schema_version": 1, + "model": "T_hybrid = lookup_latency + hit_bytes / bandwidth + (1-hit_rate) * T_full", + "assumptions": [ + "Prefill recompute time scales linearly with the unhit token fraction.", + "KV lookup/transfer and remaining recompute are serialized for TTFT.", + "QPM values are independent-resource upper bounds, not queueing predictions." + ], + "inputs": { + "kv_bytes_per_token": 327680.0, + "recompute_ms_per_1k_tokens": 12.0 + }, + "scenarios": [ + { + "scenario": { + "context_tokens": 8192, + "hit_rate": 0.1, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 98.304 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 268435456.0, + "transfer_ms": 107.37418240000001, + "remaining_compute_ms": 88.4736, + "hybrid_ttft_ms": 196.84778240000003, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -98.54378240000003, + "ttft_saved_ratio": -1.0024391927083336, + "critical_bandwidth_gbytes_s": 30.39901431418735, + "is_beneficial": false, + "network_qpm_upper_bound": 558.7935447692871, + "compute_qpm_upper_bound": 678.1684027777777, + "hybrid_qpm_upper_bound": 558.7935447692871, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 0.91552734375 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 268435456.0, + "transfer_ms": 10.73741824, + "remaining_compute_ms": 88.4736, + "hybrid_ttft_ms": 99.31101824000001, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -1.0070182400000078, + "ttft_saved_ratio": -0.010243919270833457, + "critical_bandwidth_gbytes_s": 27.587299186056065, + "is_beneficial": false, + "network_qpm_upper_bound": 5587.935447692871, + "compute_qpm_upper_bound": 678.1684027777777, + "hybrid_qpm_upper_bound": 678.1684027777777, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 1.111111111111111 + } + ] + }, + { + "scenario": { + "context_tokens": 8192, + "hit_rate": 0.3, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 98.304 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 805306368.0, + "transfer_ms": 322.1225472, + "remaining_compute_ms": 68.8128, + "hybrid_ttft_ms": 391.93534719999997, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -293.63134719999994, + "ttft_saved_ratio": -2.9869725260416664, + "critical_bandwidth_gbytes_s": 28.265091256247548, + "is_beneficial": false, + "network_qpm_upper_bound": 186.2645149230957, + "compute_qpm_upper_bound": 871.9308035714287, + "hybrid_qpm_upper_bound": 186.2645149230957, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 0.30517578125 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 805306368.0, + "transfer_ms": 32.21225472, + "remaining_compute_ms": 68.8128, + "hybrid_ttft_ms": 101.12505472, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -2.8210547199999922, + "ttft_saved_ratio": -0.028697252604166668, + "critical_bandwidth_gbytes_s": 27.399574294346607, + "is_beneficial": false, + "network_qpm_upper_bound": 1862.645149230957, + "compute_qpm_upper_bound": 871.9308035714287, + "hybrid_qpm_upper_bound": 871.9308035714287, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 1.4285714285714288 + } + ] + }, + { + "scenario": { + "context_tokens": 8192, + "hit_rate": 0.5, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 98.304 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 1342177280.0, + "transfer_ms": 536.8709120000001, + "remaining_compute_ms": 49.152, + "hybrid_ttft_ms": 587.0229120000001, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -488.71891200000016, + "ttft_saved_ratio": -4.971505859375001, + "critical_bandwidth_gbytes_s": 27.8737597607576, + "is_beneficial": false, + "network_qpm_upper_bound": 111.75870895385742, + "compute_qpm_upper_bound": 1220.703125, + "hybrid_qpm_upper_bound": 111.75870895385742, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 0.18310546875 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 1342177280.0, + "transfer_ms": 53.687091200000005, + "remaining_compute_ms": 49.152, + "hybrid_ttft_ms": 102.93909120000001, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -4.635091200000005, + "ttft_saved_ratio": -0.0471505859375001, + "critical_bandwidth_gbytes_s": 27.362335480714343, + "is_beneficial": false, + "network_qpm_upper_bound": 1117.5870895385742, + "compute_qpm_upper_bound": 1220.703125, + "hybrid_qpm_upper_bound": 1117.5870895385742, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 1.8310546875 + } + ] + }, + { + "scenario": { + "context_tokens": 8192, + "hit_rate": 0.8, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 98.304 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 2147483648.0, + "transfer_ms": 858.9934592000001, + "remaining_compute_ms": 19.660799999999995, + "hybrid_ttft_ms": 879.6542592000001, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -781.3502592000001, + "ttft_saved_ratio": -7.948305859375001, + "critical_bandwidth_gbytes_s": 27.658360912481708, + "is_beneficial": false, + "network_qpm_upper_bound": 69.84919309616089, + "compute_qpm_upper_bound": 3051.757812500001, + "hybrid_qpm_upper_bound": 69.84919309616089, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 0.11444091796875 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 2147483648.0, + "transfer_ms": 85.89934592, + "remaining_compute_ms": 19.660799999999995, + "hybrid_ttft_ms": 105.66014591999999, + "full_recompute_ttft_ms": 98.304, + "ttft_saved_ms": -7.356145919999989, + "ttft_saved_ratio": -0.0748305859374998, + "critical_bandwidth_gbytes_s": 27.341433096690736, + "is_beneficial": false, + "network_qpm_upper_bound": 698.4919309616089, + "compute_qpm_upper_bound": 3051.757812500001, + "hybrid_qpm_upper_bound": 698.4919309616089, + "full_recompute_qpm_upper_bound": 610.3515625, + "qpm_bound_ratio": 1.1444091796875 + } + ] + }, + { + "scenario": { + "context_tokens": 32768, + "hit_rate": 0.1, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 393.216 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 1073741824.0, + "transfer_ms": 429.49672960000004, + "remaining_compute_ms": 353.8944, + "hybrid_ttft_ms": 784.3911296000001, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -391.1751296000001, + "ttft_saved_ratio": -0.9948097981770836, + "critical_bandwidth_gbytes_s": 28.019232599891442, + "is_beneficial": false, + "network_qpm_upper_bound": 139.69838619232178, + "compute_qpm_upper_bound": 169.54210069444443, + "hybrid_qpm_upper_bound": 139.69838619232178, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 0.91552734375 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 1073741824.0, + "transfer_ms": 42.94967296, + "remaining_compute_ms": 353.8944, + "hybrid_ttft_ms": 396.94407296, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -3.72807296000002, + "ttft_saved_ratio": -0.009480979817708457, + "critical_bandwidth_gbytes_s": 27.3762881677443, + "is_beneficial": false, + "network_qpm_upper_bound": 1396.9838619232178, + "compute_qpm_upper_bound": 169.54210069444443, + "hybrid_qpm_upper_bound": 169.54210069444443, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 1.111111111111111 + } + ] + }, + { + "scenario": { + "context_tokens": 32768, + "hit_rate": 0.3, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 393.216 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 3221225472.0, + "transfer_ms": 1288.4901888, + "remaining_compute_ms": 275.2512, + "hybrid_ttft_ms": 1564.7413887999999, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -1171.5253887999997, + "ttft_saved_ratio": -2.9793431315104164, + "critical_bandwidth_gbytes_s": 27.540127217761242, + "is_beneficial": false, + "network_qpm_upper_bound": 46.566128730773926, + "compute_qpm_upper_bound": 217.98270089285717, + "hybrid_qpm_upper_bound": 46.566128730773926, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 0.30517578125 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 3221225472.0, + "transfer_ms": 128.84901888, + "remaining_compute_ms": 275.2512, + "hybrid_ttft_ms": 404.20021887999997, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -10.984218879999958, + "ttft_saved_ratio": -0.027934313151041668, + "critical_bandwidth_gbytes_s": 27.32983445439181, + "is_beneficial": false, + "network_qpm_upper_bound": 465.66128730773926, + "compute_qpm_upper_bound": 217.98270089285717, + "hybrid_qpm_upper_bound": 217.98270089285717, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 1.4285714285714288 + } + ] + }, + { + "scenario": { + "context_tokens": 32768, + "hit_rate": 0.5, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 393.216 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 5368709120.0, + "transfer_ms": 2147.4836480000004, + "remaining_compute_ms": 196.608, + "hybrid_ttft_ms": 2345.0916480000005, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -1951.8756480000006, + "ttft_saved_ratio": -4.963876464843751, + "critical_bandwidth_gbytes_s": 27.446265592409308, + "is_beneficial": false, + "network_qpm_upper_bound": 27.939677238464355, + "compute_qpm_upper_bound": 305.17578125, + "hybrid_qpm_upper_bound": 27.939677238464355, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 0.18310546875 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 5368709120.0, + "transfer_ms": 214.74836480000002, + "remaining_compute_ms": 196.608, + "hybrid_ttft_ms": 411.4563648, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -18.24036480000001, + "ttft_saved_ratio": -0.0463876464843751, + "critical_bandwidth_gbytes_s": 27.320562623404644, + "is_beneficial": false, + "network_qpm_upper_bound": 279.39677238464355, + "compute_qpm_upper_bound": 305.17578125, + "hybrid_qpm_upper_bound": 279.39677238464355, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 1.8310546875 + } + ] + }, + { + "scenario": { + "context_tokens": 32768, + "hit_rate": 0.8, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 393.216 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 8589934592.0, + "transfer_ms": 3435.9738368000003, + "remaining_compute_ms": 78.64319999999998, + "hybrid_ttft_ms": 3515.6170368000003, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -3122.4010368000004, + "ttft_saved_ratio": -7.940676464843751, + "critical_bandwidth_gbytes_s": 27.393749049662468, + "is_beneficial": false, + "network_qpm_upper_bound": 17.462298274040222, + "compute_qpm_upper_bound": 762.9394531250002, + "hybrid_qpm_upper_bound": 17.462298274040222, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 0.11444091796875 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 8589934592.0, + "transfer_ms": 343.59738368, + "remaining_compute_ms": 78.64319999999998, + "hybrid_ttft_ms": 422.34058368, + "full_recompute_ttft_ms": 393.216, + "ttft_saved_ms": -29.12458368, + "ttft_saved_ratio": -0.07406764648437503, + "critical_bandwidth_gbytes_s": 27.31534998257401, + "is_beneficial": false, + "network_qpm_upper_bound": 174.62298274040222, + "compute_qpm_upper_bound": 762.9394531250002, + "hybrid_qpm_upper_bound": 174.62298274040222, + "full_recompute_qpm_upper_bound": 152.587890625, + "qpm_bound_ratio": 1.1444091796875 + } + ] + }, + { + "scenario": { + "context_tokens": 131072, + "hit_rate": 0.1, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 1572.864 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 4294967296.0, + "transfer_ms": 1717.9869184000001, + "remaining_compute_ms": 1415.5776, + "hybrid_ttft_ms": 3134.5645184000005, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -1561.7005184000004, + "ttft_saved_ratio": -0.9929024495442711, + "critical_bandwidth_gbytes_s": 27.48138863010473, + "is_beneficial": false, + "network_qpm_upper_bound": 34.924596548080444, + "compute_qpm_upper_bound": 42.38552517361111, + "hybrid_qpm_upper_bound": 34.924596548080444, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 0.91552734375 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 4294967296.0, + "transfer_ms": 171.79869184, + "remaining_compute_ms": 1415.5776, + "hybrid_ttft_ms": 1587.4762918400002, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -14.612291840000125, + "ttft_saved_ratio": -0.009290244954427207, + "critical_bandwidth_gbytes_s": 27.324038822697126, + "is_beneficial": false, + "network_qpm_upper_bound": 349.24596548080444, + "compute_qpm_upper_bound": 42.38552517361111, + "hybrid_qpm_upper_bound": 42.38552517361111, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 1.111111111111111 + } + ] + }, + { + "scenario": { + "context_tokens": 131072, + "hit_rate": 0.3, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 1572.864 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 12884901888.0, + "transfer_ms": 5153.9607552, + "remaining_compute_ms": 1101.0048, + "hybrid_ttft_ms": 6255.9655551999995, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -4683.101555199999, + "ttft_saved_ratio": -2.977435782877604, + "critical_bandwidth_gbytes_s": 27.364659940806085, + "is_beneficial": false, + "network_qpm_upper_bound": 11.641532182693481, + "compute_qpm_upper_bound": 54.49567522321429, + "hybrid_qpm_upper_bound": 11.641532182693481, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 0.30517578125 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 12884901888.0, + "transfer_ms": 515.39607552, + "remaining_compute_ms": 1101.0048, + "hybrid_ttft_ms": 1616.50087552, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -43.636875519999876, + "ttft_saved_ratio": -0.027743578287760418, + "critical_bandwidth_gbytes_s": 27.312454930396694, + "is_beneficial": false, + "network_qpm_upper_bound": 116.41532182693481, + "compute_qpm_upper_bound": 54.49567522321429, + "hybrid_qpm_upper_bound": 54.49567522321429, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 1.4285714285714288 + } + ] + }, + { + "scenario": { + "context_tokens": 131072, + "hit_rate": 0.5, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 1572.864 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 21474836480.0, + "transfer_ms": 8589.934592000001, + "remaining_compute_ms": 786.432, + "hybrid_ttft_ms": 9377.366592000002, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -7804.502592000003, + "ttft_saved_ratio": -4.9619691162109385, + "critical_bandwidth_gbytes_s": 27.34143309669074, + "is_beneficial": false, + "network_qpm_upper_bound": 6.984919309616089, + "compute_qpm_upper_bound": 76.2939453125, + "hybrid_qpm_upper_bound": 6.984919309616089, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 0.18310546875 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 21474836480.0, + "transfer_ms": 858.9934592000001, + "remaining_compute_ms": 786.432, + "hybrid_ttft_ms": 1645.5254592000001, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -72.66145920000008, + "ttft_saved_ratio": -0.04619691162109385, + "critical_bandwidth_gbytes_s": 27.310139330460924, + "is_beneficial": false, + "network_qpm_upper_bound": 69.84919309616089, + "compute_qpm_upper_bound": 76.2939453125, + "hybrid_qpm_upper_bound": 69.84919309616089, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 1.8310546875 + } + ] + }, + { + "scenario": { + "context_tokens": 131072, + "hit_rate": 0.8, + "kv_bytes_per_token": 327680.0, + "full_recompute_ms": 1572.864 + }, + "transports": [ + { + "transport": { + "name": "TCP", + "bandwidth_gbytes_s": 2.5, + "lookup_latency_ms": 1.0 + }, + "hit_bytes": 34359738368.0, + "transfer_ms": 13743.895347200001, + "remaining_compute_ms": 314.5727999999999, + "hybrid_ttft_ms": 14059.468147200001, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -12486.604147200002, + "ttft_saved_ratio": -7.938769116210938, + "critical_bandwidth_gbytes_s": 27.32838531598726, + "is_beneficial": false, + "network_qpm_upper_bound": 4.3655745685100555, + "compute_qpm_upper_bound": 190.73486328125006, + "hybrid_qpm_upper_bound": 4.3655745685100555, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 0.11444091796875 + }, + { + "transport": { + "name": "RDMA", + "bandwidth_gbytes_s": 25.0, + "lookup_latency_ms": 0.1 + }, + "hit_bytes": 34359738368.0, + "transfer_ms": 1374.38953472, + "remaining_compute_ms": 314.5727999999999, + "hybrid_ttft_ms": 1689.0623347199999, + "full_recompute_ttft_ms": 1572.864, + "ttft_saved_ms": -116.19833471999982, + "ttft_saved_ratio": -0.07387691162109356, + "critical_bandwidth_gbytes_s": 27.308836978036403, + "is_beneficial": false, + "network_qpm_upper_bound": 43.655745685100555, + "compute_qpm_upper_bound": 190.73486328125006, + "hybrid_qpm_upper_bound": 43.655745685100555, + "full_recompute_qpm_upper_bound": 38.14697265625, + "qpm_bound_ratio": 1.1444091796875 + } + ] + } + ] +} diff --git a/src/code/issue5/test_kv_cache_break_even.py b/src/code/issue5/test_kv_cache_break_even.py new file mode 100644 index 0000000..0faaf59 --- /dev/null +++ b/src/code/issue5/test_kv_cache_break_even.py @@ -0,0 +1,31 @@ +import math + +from kv_cache_break_even import Scenario, Transport, critical_bandwidth_gbytes_s, evaluate_transport + + +def test_critical_bandwidth_is_exact_break_even_without_lookup_latency(): + scenario = Scenario( + context_tokens=1_000, + hit_rate=0.5, + kv_bytes_per_token=1_000_000, + full_recompute_ms=100, + ) + critical = critical_bandwidth_gbytes_s(scenario, lookup_latency_ms=0) + assert critical == 10.0 + + result = evaluate_transport(scenario, Transport("at-boundary", critical, 0)) + assert math.isclose(result["hybrid_ttft_ms"], scenario.full_recompute_ms) + assert result["is_beneficial"] is False # strict improvement is required + + +def test_lookup_can_make_reuse_impossible(): + scenario = Scenario(1_000, 0.1, 10_000, 20) + assert math.isinf(critical_bandwidth_gbytes_s(scenario, lookup_latency_ms=2.0)) + + +def test_faster_transport_reduces_ttft(): + scenario = Scenario(32_000, 0.3, 320_000, 400) + tcp = evaluate_transport(scenario, Transport("TCP", 2.5, 1.0)) + rdma = evaluate_transport(scenario, Transport("RDMA", 25.0, 0.1)) + assert rdma["hybrid_ttft_ms"] < tcp["hybrid_ttft_ms"] +