diff --git a/.claude/rules/modify-component-must-read.md b/.claude/rules/modify-component-must-read.md new file mode 100644 index 000000000000..948cc22279c4 --- /dev/null +++ b/.claude/rules/modify-component-must-read.md @@ -0,0 +1,6 @@ +# Must-Read Skills Before Modifying Components + +Before modifying the following components, read the listed skill first. + +- **Speculative decoding code** (anything under `python/sglang/srt/speculative/`, related attention backends, scheduler accumulators, IPC fields, observability metrics, or CLI flags) → [`speculative-naming`](../skills/speculative-naming/SKILL.md) +- **`Scheduler` / `TokenizerManager` / `ModelRunner` `__init__`** (`python/sglang/srt/managers/scheduler.py`, `python/sglang/srt/managers/tokenizer_manager.py`, `python/sglang/srt/model_executor/model_runner.py`) → [`large-class-init-style`](../skills/large-class-init-style/SKILL.md) diff --git a/.claude/skills/clean-startup-log/SKILL.md b/.claude/skills/clean-startup-log/SKILL.md index 8f7c254115ca..c1b9e886f952 100644 --- a/.claude/skills/clean-startup-log/SKILL.md +++ b/.claude/skills/clean-startup-log/SKILL.md @@ -23,6 +23,11 @@ For TP>1 testing: uv run sglang serve --model-path Qwen/Qwen3-8B --tp 2 2>&1 | tee /tmp/startup_log.txt ``` +For MoE / hybrid-SWA models (e.g. gpt-oss), test separately — they exercise different code paths: +```bash +uv run sglang serve --model-path openai/gpt-oss-20b 2>&1 | tee /tmp/startup_log.txt +``` + ### 2. Compare against the clean reference log Read `/tmp/startup_log.txt` and compare it against the reference log at the bottom of this file. Identify lines that: @@ -31,6 +36,7 @@ Read `/tmp/startup_log.txt` and compare it against the reference log at the bott - Contain `WARNING`, `deprecated`, `is deprecated`, or similar noise - Are printed by third-party libraries (transformers, torchao, NCCL, Gloo, tqdm, etc.) - Are duplicate/redundant with information already logged by SGLang +- Appear multiple times due to `ModelConfig` being constructed in multiple processes ### 3. Classify each noisy line @@ -40,6 +46,7 @@ For each noisy line, determine: |----------|--------| | **SGLang code using wrong API** | Fix the SGLang code (e.g., replace deprecated API with new one) | | **SGLang code logging at wrong level** | Change log level (e.g., warning -> debug for non-actionable messages) | +| **Duplicated across processes** | Downgrade to debug — info logged in one process becomes noise in 3-4 | | **Third-party lib prints at import time** | Suppress the logger or redirect stdout during that import | | **C-level print from .so library** | Redirect fd 1 during the specific C call, or accept it if too invasive | | **Real warning the user should see** | Keep it | @@ -52,6 +59,23 @@ List all noisy lines with their source and proposed fix. Ask the user to review After approval, apply fixes one at a time, re-launch the server, and verify each fix works. +## Key Architecture: Why Logs Repeat + +`ModelConfig` is constructed **3-4 times** during startup across different processes: +1. Main process: `ServerArgs.__post_init__()` → `get_model_config()` → `ModelConfig()` +2. Scheduler subprocess: `Scheduler.init_model_config()` → `ModelConfig.from_server_args()` +3. Scheduler subprocess: `TpModelWorker._init_model_config()` → `ModelConfig.from_server_args()` +4. Main process: `TokenizerManager.init_model_config()` → `ModelConfig.from_server_args()` + +Similarly, `get_tokenizer()` is called **5 times** across processes: +1. `resolve_auto_parsers` (main) — `template_detection.py` +2. `Scheduler.init_tokenizer()` (scheduler subprocess) — `scheduler.py` +3. `DetokenizerManager` (detokenizer subprocess) — `detokenizer_manager.py` +4. `TpModelWorker.__init__()` (scheduler subprocess) — `tp_worker.py` +5. `TokenizerManager` (main) — `tokenizer_manager.py` + +Any `logger.info()` or `logger.warning()` in `ModelConfig.__init__()` or `get_tokenizer()` will appear 3-5 times. **Keep these at `logger.debug()`.** + ## Known Noise Sources and Fixes (from past sessions) ### 1. torchao "Skipping import of cpp extensions due to incompatible torch version" @@ -69,11 +93,21 @@ After approval, apply fixes one at a time, re-launch the server, and verify each _torchao_logger.setLevel(_prev_level) ``` -### 2. "`torch_dtype` is deprecated! Use `dtype` instead!" +### 2. "`torch_dtype` is deprecated! Use `dtype` instead!" (PARTIALLY FIXED) - **Source:** `transformers/configuration_utils.py` — the `torch_dtype` property warns via `logger.warning_once()` -- **Trigger:** `get_hf_text_config()` in `sglang/srt/utils/hf_transformers/common.py` accesses `config.torch_dtype` -- **Fix:** Replace all `getattr(config, "torch_dtype", ...)` with `getattr(config, "dtype", ...)` and `config.torch_dtype = X` with `config.dtype = X` in `common.py` +- **Trigger:** Model files accessing `config.torch_dtype` instead of `config.dtype` +- **Fix applied so far:** Only `models/gpt_oss.py` (lines 222, 471) — tested with `openai/gpt-oss-20b`. +- **Remaining files that still use `config.torch_dtype`** (fix each only after testing with the corresponding model): + - `models/bailing_moe.py` (line 302) + - `models/llada2.py` (line 313) + - `models/qwen3_next.py` (lines 192, 209) + - `models/qwen3_5.py` (line 245) + - `models/nano_nemotron_vl.py` (lines 79, 102, 284) + - `models/llava.py` (lines 732, 734-737) + - `model_loader/loader.py` (line 649) +- **Note:** `common.py` was already fixed in a prior session. If new model files are added with `config.torch_dtype`, the warning will reappear — grep for `\.torch_dtype` to find them. +- **Important:** Only change `config.torch_dtype` → `config.dtype` for models you have actually tested. The `dtype` property should return the same value, but verify per-model to avoid regressions. ### 3. "`BaseImageProcessorFast` is deprecated" @@ -105,6 +139,67 @@ After approval, apply fixes one at a time, re-launch the server, and verify each - **Status:** These are expected and useful. They show progress during weight loading and CUDA graph capture. Keep them. +### 9. CUTE_DSL "Unexpected error during package walk" — double-logged (FIXED) + +- **Source:** `nvidia-cutlass-dsl` package at `.venv/.../cutlass/cutlass_dsl/cutlass.py`, line 391. Logger named `CUTE_DSL` with its own `StreamHandler`. +- **Trigger:** During CUDA graph capture, cutlass DSL walks packages and hits an unexpected error for `cutlass.cute.experimental`. +- **Root cause of double-logging:** The CUTE_DSL logger has `propagate=True` (default), so the warning is emitted by both the CUTE_DSL handler (with its format) and the root logger (SGLang's format). +- **Fix applied:** In `entrypoints/engine.py`, changed `CUTE_DSL_LOG_LEVEL` from `"30"` (WARNING) to `"40"` (ERROR). This suppresses the WARNING at both the CUTE_DSL logger and root propagation levels. The env var controls both `logger.setLevel()` and `console_handler.setLevel()` in cutlass's `setup_log()`. + +### 10. ModelConfig init logs repeated 3x (FIXED) + +- **Lines:** `"Downcasting torch.float32 to ..."`, `"Hybrid swa model: ..."`, `"DeepGemm is enabled but ..."` +- **Source:** `configs/model_config.py` — `_get_and_verify_dtype()` (line 1457), `_derive_hybrid_model()` (line 497), `_verify_quantization()` (line 1236) +- **Root cause:** `ModelConfig.__init__()` is called 3-4 times in different processes (see "Key Architecture" above). Each construction fires the same log lines. +- **Fix applied:** Downgraded all three from `logger.info()`/`logger.warning()` to `logger.debug()`. The dtype is already visible in `server_args` and `Load weight end`. Hybrid SWA info appears in `Tree cache initialized`. DeepGemm is not actionable. + +### 11. Tokenizer retry/fallback messages repeated 3-4x (FIXED) + +- **Lines:** `"Tokenizer loaded as generic TokenizersBackend ... retrying"`, `"Loading tokenizer ... directly as PreTrainedTokenizerFast"`, `"Tokenizer for ... loaded as generic TokenizersBackend. Set --trust-remote-code"` +- **Source:** `utils/hf_transformers/tokenizer.py` — `_resolve_tokenizers_backend()` (line 215), `_load_tokenizer_by_declared_class()` (line 110), final warning (line 244) +- **Root cause:** 5 separate `get_tokenizer()` calls across processes (see "Key Architecture" above). Each produces 3 log lines. Concurrent subprocess launches cause interleaved/doubled output. +- **Fix applied:** Downgraded all three from `logger.warning()`/`logger.info()` to `logger.debug()`. + +### 12. Template detection logs — 5 lines consolidated to 1 (FIXED) + +- **Lines:** `"Detected reasoning config '...' from template rule '...'"`, `"Detected reasoning parser '...' from template rule '...'"`, `"Detected tool-call parser '...' from template rule '...'"`, `"Auto-detected reasoning parser: ..."`, `"Auto-detected tool-call parser: ..."` +- **Source:** `managers/template_detection.py` (lines 337, 370) logged each detection rule match. `managers/template_manager.py` (lines 177-182) logged summary lines that duplicated the detection logs. +- **Fix applied:** Removed per-rule logs from `template_detection.py`. Consolidated the 5 lines in `template_manager.py` into a single summary: `"Auto-detected template features: reasoning_config=..., reasoning_parser=..., tool_call_parser=..."` + +### 13. KV cache dtype logged separately from allocation (FIXED) + +- **Lines:** `"Using KV cache dtype: torch.bfloat16"` then `"KV Cache is allocated. #tokens: ..., K size: ..., V size: ..."` +- **Source:** `model_executor/model_runner.py` (line 2217) and `mem_cache/memory_pool.py` (line 740) +- **Fix applied:** Removed the standalone dtype log from `model_runner.py`. Added `dtype` field to the allocation log in `memory_pool.py`: `"KV Cache is allocated. dtype: torch.bfloat16, #tokens: ..., K size: ..., V size: ..."` + +### 14. CUTLASS backend warning — B200 → SM100, warning → info (FIXED) + +- **Line:** `"CUTLASS backend is disabled when piecewise cuda graph is enabled due to TMA descriptor initialization issues on B200."` +- **Source:** `layers/attention/flashinfer_backend.py` (line 249) +- **Fix applied:** Changed "B200" to "SM100 GPUs" (the condition checks `is_sm100_supported()` which matches SM10x, not just B200). Downgraded from `logger.warning()` to `logger.info()` since it's an expected automatic fallback. + +### 15. `max_total_num_tokens` and `Tree cache initialized` log ordering + +- **Issue:** `max_total_num_tokens=...` appears before `Tree cache initialized:...` even though tree cache is conceptually part of memory setup. +- **Root cause:** `max_total_num_tokens` is logged inside `init_model_worker()` (scheduler.py:972), which runs before `build_kv_cache()` (scheduler.py:425) where tree cache is created. +- **Status:** Not fixed — reordering was reverted. Acceptable as-is. + +### 16. `Ignore import error when loading sglang.srt.models.midashenglm` + +- **Source:** `models/registry.py` (line 109) — `logger.warning()` during `import_model_classes()` which iterates all model modules via `pkgutil.iter_modules` +- **Trigger:** The `midashenglm` model depends on `torchaudio`, which fails to load +- **Status:** Should be downgraded to `logger.debug()` — not actionable when loading an unrelated model. Same pattern exists in `managers/multimodal_processor.py`, `dllm/algorithm/__init__.py`, `multimodal_gen/runtime/models/registry.py`. + +### 17. `Multiple NUMA nodes found for GPU X` + +- **Source:** `utils/numa_utils.py` (line 112) — `logger.warning()` +- **Status:** Could be downgraded to `logger.info()`. The situation is handled gracefully ("Using the first one") and not actionable. + +### 18. Warmup `/model_info` access log + +- **Source:** Uvicorn access log, triggered by SGLang's own warmup at `entrypoints/http_server.py` (line 1877) +- **Status:** SGLang talking to itself. Could suppress uvicorn access logger during warmup, or exclude `/model_info` from warmup access logging. + ## Investigation Techniques ### Trace what triggers an import @@ -137,43 +232,51 @@ logging.getLogger('TARGET_LOGGER_NAME').addHandler(h) strings /path/to/library.so | grep "SEARCH_STRING" ``` +### Find all config.torch_dtype accesses (for deprecation warning) +```bash +grep -rn '\.torch_dtype' python/sglang/srt/models/ python/sglang/srt/model_loader/ python/sglang/srt/utils/hf_transformers/ +``` + ## Reference: Clean Startup Log (TP=1, Qwen3-8B) ``` -[2026-04-27 02:35:53] Attention backend not specified. Use trtllm_mha backend by default. -[2026-04-27 02:35:53] TensorRT-LLM MHA only supports page_size of 16, 32 or 64, changing page_size from None to 64. -[2026-04-27 02:35:54] server_args=ServerArgs(model_path='Qwen/Qwen3-8B', ...) -[2026-04-27 02:35:56] Using default HuggingFace chat template with detected content format: string -[2026-04-27 02:36:03] Init torch distributed begin. +[2026-05-24 00:52:39] Attention backend not specified. Use trtllm_mha backend by default. +[2026-05-24 00:52:39] TensorRT-LLM MHA only supports page_size of 16, 32 or 64, changing page_size from None to 64. +[2026-05-24 00:52:40] server_args=ServerArgs(model_path='Qwen/Qwen3-8B', ...) +[2026-05-24 00:52:40] Multiple NUMA nodes found for GPU 0: [...]. Using the first one. +[2026-05-24 00:52:42] Using default HuggingFace chat template with detected content format: string +[2026-05-24 00:52:42] Auto-detected template features: reasoning_config=..., reasoning_parser=qwen3, tool_call_parser=qwen +[2026-05-24 00:52:50] Init torch distributed begin. [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[2026-04-27 02:36:03] Init torch distributed ends. elapsed=0.27 s, mem usage=0.09 GB -[2026-04-27 02:36:04] Load weight begin. avail mem=177.57 GB -[2026-04-27 02:36:04] Found local HF snapshot for Qwen/Qwen3-8B at ...; skipping download. -Multi-thread loading shards: 100% Completed | 5/5 [00:01<00:00, 3.08it/s] -[2026-04-27 02:36:06] Load weight end. elapsed=1.97 s, type=Qwen3ForCausalLM, avail mem=162.30 GB, mem usage=15.28 GB. -[2026-04-27 02:36:06] Using KV cache dtype: torch.bfloat16 -[2026-04-27 02:36:06] KV Cache is allocated. #tokens: 992896, K size: 68.18 GB, V size: 68.18 GB -[2026-04-27 02:36:06] Memory pool end. avail mem=25.26 GB -[2026-04-27 02:36:06] Capture cuda graph begin. This can take up to several minutes. avail mem=24.14 GB -[2026-04-27 02:36:06] Capture cuda graph bs [1, 2, 4, ...] -Capturing batches (bs=1 avail_mem=23.54 GB): 100% | 52/52 [00:03<00:00, 16.76it/s] -[2026-04-27 02:36:09] Capture cuda graph end. Time elapsed: 3.74 s. mem usage=0.60 GB. avail mem=23.54 GB. -[2026-04-27 02:36:09] Capture piecewise CUDA graph begin. avail mem=23.54 GB -[2026-04-27 02:36:09] Capture cuda graph num tokens [4, 8, 12, ...] -Compiling num tokens (num_tokens=4): 100% | 74/74 [00:09<00:00, 8.16it/s] -Capturing num tokens (num_tokens=4 avail_mem=21.23 GB): 100% | 74/74 [00:08<00:00, 9.11it/s] -[2026-04-27 02:36:27] Capture piecewise CUDA graph end. Time elapsed: 17.62 s. mem usage=2.32 GB. avail mem=21.22 GB. -[2026-04-27 02:36:28] max_total_num_tokens=992896, chunked_prefill_size=16384, ... -[2026-04-27 02:36:29] INFO: Started server process [399368] -[2026-04-27 02:36:29] INFO: Waiting for application startup. -[2026-04-27 02:36:29] Using default chat sampling params from model generation config: ... -[2026-04-27 02:36:29] INFO: Application startup complete. -[2026-04-27 02:36:29] INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) -[2026-04-27 02:36:30] Prefill batch, #new-seq: 1, #new-token: 64, ... -[2026-04-27 02:36:30] INFO: 127.0.0.1:34916 - "POST /generate HTTP/1.1" 200 OK -[2026-04-27 02:36:30] The server is fired up and ready to roll! +[2026-05-24 00:52:50] Init torch distributed ends. elapsed=0.21 s, mem usage=0.10 GB +[2026-05-24 00:52:51] Load weight begin. avail mem=275.75 GB +[2026-05-24 00:52:51] Found local HF snapshot for Qwen/Qwen3-8B at ...; skipping download. +Multi-thread loading shards: 100% Completed | 5/5 [00:01<00:00, 2.62it/s] +[2026-05-24 00:52:54] Load weight end. elapsed=2.62 s, type=Qwen3ForCausalLM, avail mem=260.48 GB, mem usage=15.28 GB. +[2026-05-24 00:52:54] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1707904, K size: 117.28 GB, V size: 117.28 GB +[2026-05-24 00:52:54] Memory pool end. avail mem=25.28 GB +[2026-05-24 00:52:54] CUTLASS backend is disabled when piecewise cuda graph is enabled due to TMA descriptor initialization issues on SM100 GPUs. Using auto backend instead for stability. +[2026-05-24 00:52:54] Capture cuda graph begin. This can take up to several minutes. avail mem=24.16 GB +[2026-05-24 00:52:54] Capture cuda graph bs [1, 2, 4, ...] +Capturing batches (bs=1 avail_mem=23.56 GB): 100% | 52/52 [00:05<00:00, 10.36it/s] +[2026-05-24 00:53:00] Capture cuda graph end. Time elapsed: 5.38 s. mem usage=0.60 GB. avail mem=23.56 GB. +[2026-05-24 00:53:00] Capture piecewise CUDA graph begin. avail mem=23.56 GB +[2026-05-24 00:53:00] Capture cuda graph num tokens [4, 8, 12, ...] +Compiling num tokens (num_tokens=4): 100% | 74/74 [00:09<00:00, 7.44it/s] +Capturing num tokens (num_tokens=4 avail_mem=21.24 GB): 100% | 74/74 [00:07<00:00, 10.44it/s] +[2026-05-24 00:53:18] Capture piecewise CUDA graph end. Time elapsed: 18.18 s. mem usage=2.32 GB. avail mem=21.24 GB. +[2026-05-24 00:53:20] Tree cache initialized: source=default impl=RadixCache hybrid_swa=False hybrid_ssm=False hierarchical=False streaming_wrapped=False +[2026-05-24 00:53:20] max_total_num_tokens=1707904, chunked_prefill_size=16384, max_prefill_tokens=16384, max_running_requests=4096, context_len=40960, available_gpu_mem=21.24 GB +[2026-05-24 00:53:20] INFO: Started server process [1964249] +[2026-05-24 00:53:20] INFO: Waiting for application startup. +[2026-05-24 00:53:20] Using default chat sampling params from model generation config: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95} +[2026-05-24 00:53:20] INFO: Application startup complete. +[2026-05-24 00:53:20] INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) +[2026-05-24 00:53:21] Prefill batch, #new-seq: 1, #new-token: 64, ... +[2026-05-24 00:53:21] INFO: 127.0.0.1:... - "POST /generate HTTP/1.1" 200 OK +[2026-05-24 00:53:21] The server is fired up and ready to roll! ``` -Note: `[Gloo]` messages and tqdm progress bars are acceptable. The key is no warnings or deprecation messages from transformers, torchao, or other third-party libraries. +Note: `[Gloo]` messages and tqdm progress bars are acceptable. The key is no warnings or deprecation messages from transformers, torchao, or other third-party libraries. The `CUTLASS backend is disabled` message is now `info` level, not a warning. diff --git a/.claude/skills/large-class-init-style/SKILL.md b/.claude/skills/large-class-init-style/SKILL.md new file mode 100644 index 000000000000..752c23bdc0ad --- /dev/null +++ b/.claude/skills/large-class-init-style/SKILL.md @@ -0,0 +1,32 @@ +--- +name: large-class-init-style +description: '`__init__` style for SGLang `Scheduler`, `TokenizerManager`, and `ModelRunner`. Use when modifying the `__init__` of any of these three classes, or reviewing changes that add new construction logic to them.' +--- + +# `__init__` Style for Scheduler / TokenizerManager / ModelRunner + +Apply when modifying the `__init__` of: + +- `Scheduler` — `python/sglang/srt/managers/scheduler.py` +- `TokenizerManager` — `python/sglang/srt/managers/tokenizer_manager.py` +- `ModelRunner` — `python/sglang/srt/model_executor/model_runner.py` + +## Why + +- Downstream forks override one piece (tokenizer, KV cache, IPC, …). +- Inline logic forces them to copy the whole `__init__`, which rots against upstream. +- Splitting into `init_*` helpers lets them override exactly what they need. +- Reference shape: `TokenizerManager.__init__` in `python/sglang/srt/managers/tokenizer_manager.py`. + +## Rules + +- **`__init__` is an orchestrator.** Sequence of `self.init_*(...)` calls + minimal glue. No non-trivial construction inlined. +- **One helper per overridable unit.** Each `init_*` = one concern a subclass might swap. Don't lump. +- **Naming:** `init_` (snake_case, names the component). Conditional construction → `maybe_init_`, gate inside the helper. +- **No silent state coupling.** A helper only reads `self.*` set by earlier helpers. Ordering lives in `__init__`. Shared intermediates → pass as args, not via `self.*`. +- **New logic = new helper.** Default to adding `init_`, not another inline block. One-line `self.foo = server_args.foo` is fine; structured logic is not. +- **Preserve override points.** Prefer additive changes to existing `init_*` signatures. Breaking changes → call out in PR. + +## Scope + +Only the three classes listed above. Not other manager-style classes, not small dataclass/utility constructors. diff --git a/.claude/skills/sglang-cherrypick/SKILL.md b/.claude/skills/sglang-cherrypick/SKILL.md new file mode 100644 index 000000000000..452ddce07b2d --- /dev/null +++ b/.claude/skills/sglang-cherrypick/SKILL.md @@ -0,0 +1,331 @@ +--- +name: sglang-cherrypick +description: Trigger the bot-cherry-pick workflow for a batch of merged PRs onto a release branch and monitor each run to completion. Use when an SGLang release manager asks to cherry-pick a list of PRs to a release branch. +--- + +# SGLang Cherry-Pick + +Trigger `.github/workflows/bot-cherry-pick.yml` for each PR in a list, then monitor the resulting workflow runs and report per-PR success/failure with links to the created cherry-pick PRs (or the failure reason). + +## Slash Command + +`/sglang-cherrypick [pr2 pr3 ...]` + +Examples: +- `/sglang-cherrypick release/v0.5.7 25956 25958 25987` +- `/sglang-cherrypick release/v0.5.7 25956,25958,25987` (comma-separated also accepted) + +## Arguments + +- **`target_branch`** (required): Release branch in the form `release/vX.Y` or `release/vX.Y.Z`. Must already exist on `origin` (i.e., `sgl-project/sglang`). +- **`pr_numbers`** (required, one or more): Merged PR numbers to cherry-pick. Each must be a positive integer. + +## Repository + +Always targets the upstream repo `sgl-project/sglang`. The workflow's job guard (`if: github.repository == 'sgl-project/sglang'`) means triggering it on a fork is a no-op. + +## Workflow + +### Step 1 — Validate arguments + +Fail fast before triggering anything. + +```bash +# target branch shape (matches the workflow's own validator) +[[ "$TARGET_BRANCH" =~ ^release/v[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || die "Invalid target_branch" + +# branch exists on upstream +gh api "repos/sgl-project/sglang/branches/$TARGET_BRANCH" --jq '.name' >/dev/null || die "Branch not found" + +# each PR is numeric, exists, MERGED, and has a recorded merge commit +declare -A PR_TO_SHA=() +declare -A PR_TO_TITLE=() +for PR in "${PRS[@]}"; do + [[ "$PR" =~ ^[0-9]+$ ]] || die "PR '$PR' is not a positive integer" + PR_JSON=$(gh pr view "$PR" --repo sgl-project/sglang --json state,mergeCommit,title) \ + || die "PR #$PR not found" + STATE=$(jq -r .state <<<"$PR_JSON") + [[ "$STATE" == "MERGED" ]] || die "PR #$PR is not MERGED (state=$STATE)" + SHA=$(jq -r '.mergeCommit.oid // empty' <<<"$PR_JSON") + [[ -n "$SHA" ]] || die "PR #$PR has no merge commit recorded" + PR_TO_SHA[$PR]="$SHA" + PR_TO_TITLE[$PR]=$(jq -r .title <<<"$PR_JSON") +done +``` + +Report any failures and **stop** — do not trigger partial batches. + +### Step 2 — Pre-flight: list changed files and detect conflicts locally + +Before dispatching any workflow, simulate each cherry-pick locally with `git merge-tree` to (a) show the user which files would change and (b) catch conflicts before paying for a CI run. `git merge-tree --write-tree` is a side-effect-free 3-way merge — it touches neither the working tree nor any ref. + +**2a. Locate the upstream remote (`sgl-project/sglang`).** Both the dual-remote (`upstream` + `origin` fork) and single-remote setups need to work. + +```bash +UPSTREAM_REMOTE=$(git remote -v \ + | awk '$2 ~ /[:\/]sgl-project\/sglang(\.git)?$/ && $3 == "(fetch)" {print $1; exit}') +[[ -n "$UPSTREAM_REMOTE" ]] || die "No remote points to sgl-project/sglang" +``` + +**2b. Fetch the target branch and each PR's merge commit.** Fetch the commits by SHA (in case they're not on a ref the user has locally) and the target branch in one call. + +```bash +git fetch "$UPSTREAM_REMOTE" "$TARGET_BRANCH" "${PR_TO_SHA[@]}" --quiet \ + || die "Failed to fetch from $UPSTREAM_REMOTE" + +TARGET_REF="refs/remotes/$UPSTREAM_REMOTE/$TARGET_BRANCH" +``` + +**2c. Index existing cherry-pick PRs on the target branch.** One `gh pr list` call gets every cherry-pick PR ever filed against this branch (any state). For each input PR, we cross-reference by the title suffix `(#)` that the bot workflow always uses. + +```bash +# Fetch all cherry-pick PRs against this branch (any state), then bucket by +# source-PR number using the title pattern "(#)". +EXISTING_CP_JSON=$(gh pr list --repo sgl-project/sglang \ + --base "$TARGET_BRANCH" \ + --label cherry-pick \ + --state all \ + --limit 200 \ + --json number,title,url,state) + +declare -A PR_TO_EXISTING_CP=() # source_pr -> JSON array of existing cherry-pick PRs +for PR in "${PRS[@]}"; do + PR_TO_EXISTING_CP[$PR]=$(jq -c \ + "[.[] | select(.title | contains(\"(#${PR})\"))]" <<<"$EXISTING_CP_JSON") +done +``` + +For each input PR, classify the existing cherry-picks: + +- **`MERGED`** present → the cherry-pick already landed. **Skip** this PR in Step 3. +- **`OPEN`** present (and no `MERGED`) → a previous dispatch is still in flight. **Warn**, ask the user whether to skip or re-dispatch, but default to **skip** (re-dispatching creates a duplicate). +- Only `CLOSED` (no merged, no open) → previous attempts were abandoned; safe to re-dispatch. +- Empty → no prior attempt; proceed normally. + +**2d. For each PR, run `git merge-tree` and diff the result.** The semantics of `cherry-pick` are: 3-way-merge with base = parent of source commit, ours = target tip, theirs = source commit. For merge commits the workflow uses `-m 1`, which means base = **first** parent — `${SHA}^` resolves to `${SHA}^1` for both regular and merge commits, so one form covers both. + +```bash +declare -A PR_TO_CONFLICTS=() +declare -A PR_TO_FILES=() + +for PR in "${PRS[@]}"; do + SHA="${PR_TO_SHA[$PR]}" + + # --write-tree: print the resulting tree SHA on success + # Exit 0 = clean merge; exit 1 = conflicts + if MERGE_OUT=$(git merge-tree --write-tree \ + --merge-base="${SHA}^" \ + "$TARGET_REF" "$SHA" 2>&1); then + RESULT_TREE=$(head -1 <<<"$MERGE_OUT") + PR_TO_CONFLICTS[$PR]="" + # Show files that actually differ between target tip and the merged tree. + # This is more accurate than `git show --name-status $SHA` because it + # accounts for changes already present on the release branch. As a + # side-effect, an already-cherry-picked commit shows up here as "0 files". + PR_TO_FILES[$PR]=$(git diff --name-status "$TARGET_REF" "$RESULT_TREE") + else + # Conflict output format (git ≥2.40): first line is the (partial) tree, + # remaining lines list conflicted paths and informational messages. + # We just capture and surface it; user decides what to do. + PR_TO_CONFLICTS[$PR]="$MERGE_OUT" + PR_TO_FILES[$PR]=$(git show --name-status --format= "$SHA" 2>/dev/null) + fi +done +``` + +**2e. Print a pre-flight report.** One table summarizing each PR, followed by per-PR file lists. The "Prior cherry-pick" column uses the classification from 2c. + +```markdown +## Cherry-Pick Pre-Flight — `release/vX.Y.Z` + +| PR | Title | Merge SHA | Prior cherry-pick | Conflicts | # files | +|--------|--------------------------|-----------|----------------------|--------------|---------| +| #25733 | [Bug] Fix V4-Pro NaN ... | 79ea30d1 | ✅ merged as #26063 | clean | 0 | +| #25562 | [bugfix] Fix wrong ... | b19052c9 | none | **CONFLICT** | — | +| #25585 | [Bugfix] Fix missing ... | 86c6c77f | none | clean | 2 | + +### Files (PR #25585 — clean) +M python/sglang/srt/layers/communicator.py +M python/sglang/srt/models/deepseek_v4.py + +### Conflict detail (PR #25562) + +``` + +**2f. Gate before dispatching.** Stop and report if any PR is in either of these states: + +- `git merge-tree` reports a **conflict** — the workflow would just fail; let the user fix or remove that PR. +- Already has a **MERGED** cherry-pick PR on the target branch — re-dispatching would create a redundant PR. Skip it (or, if the user really wants a re-run, they can pass an explicit override list). +- Has an **OPEN** cherry-pick PR with no merged one — default to skipping with a warning; surface the open PR's URL so the user can review/merge/close it before re-dispatching. + +Only PRs that are **clean** AND have **no merged-or-open** prior cherry-pick should proceed to Step 3. + +As a sanity check, a clean pre-flight that shows **0 files changed** is the structural signature of "this commit is already on the branch" — if you see it without an existing merged cherry-pick PR being detected (rare, e.g. the original PR was force-merged onto the release branch directly), surface that too and skip the dispatch. + +### Step 3 — Dispatch each PR's workflow run + +`gh workflow run` (gh ≥2.45) prints the dispatched run's URL on stdout — parse it directly. Fall back to the snapshot/diff polling only if the URL isn't returned (older gh). + +```bash +# Snapshot once up front in case we need the fallback path. +mapfile -t SEEN < <(gh run list \ + --workflow=bot-cherry-pick.yml \ + --repo sgl-project/sglang \ + --limit 50 \ + --json databaseId --jq '.[].databaseId') + +declare -A PR_TO_RUN=() # pr_number -> run_id + +for PR in "${PRS[@]}"; do + DISPATCH_OUT=$(gh workflow run bot-cherry-pick.yml \ + --repo sgl-project/sglang \ + -f pr_number="$PR" \ + -f target_branch="$TARGET_BRANCH" 2>&1) || { echo "$DISPATCH_OUT"; die "dispatch failed for PR #$PR"; } + + # Preferred path: gh prints the run URL like + # https://github.com/sgl-project/sglang/actions/runs/26275460359 + RUN_URL=$(grep -oE 'https://github.com/[^[:space:]]+/actions/runs/[0-9]+' \ + <<<"$DISPATCH_OUT" | head -1) + RUN_ID="${RUN_URL##*/}" + + # Fallback for older gh that doesn't print the URL: poll the runs list, + # filter to workflow_dispatch events we haven't seen yet. + if [[ -z "$RUN_ID" ]]; then + for _ in $(seq 1 30); do + sleep 2 + CANDIDATE=$(gh run list \ + --workflow=bot-cherry-pick.yml \ + --repo sgl-project/sglang \ + --limit 10 \ + --json databaseId,event \ + --jq '[.[] | select(.event=="workflow_dispatch") | .databaseId] | .[0]') + if [[ -n "$CANDIDATE" ]] \ + && ! printf '%s\n' "${SEEN[@]}" | grep -qx "$CANDIDATE" \ + && ! printf '%s\n' "${PR_TO_RUN[@]}" | grep -qx "$CANDIDATE"; then + RUN_ID="$CANDIDATE" + break + fi + done + fi + + if [[ -z "$RUN_ID" ]]; then + echo "::warning::No new workflow run detected for PR #$PR within 60s" + PR_TO_RUN[$PR]="UNKNOWN" + else + PR_TO_RUN[$PR]="$RUN_ID" + fi +done +``` + +**Notes:** +- The workflow has `concurrency: cherry-pick-${{ target_branch }}` with `cancel-in-progress: false`. So multiple dispatches against the same target branch **queue serially**, not in parallel. That's fine — we batch the triggers and the GitHub side serializes execution. +- `gh workflow run` is fire-and-forget; the dispatched run shows up in `gh run list` within a few seconds. + +### Step 4 — Monitor each run to completion + +Use `gh run watch` per run id, sequentially (since they execute serially anyway). + +```bash +for PR in "${PRS[@]}"; do + RUN_ID="${PR_TO_RUN[$PR]}" + [[ "$RUN_ID" == "UNKNOWN" ]] && continue + + gh run watch "$RUN_ID" \ + --repo sgl-project/sglang \ + --exit-status \ + --interval 15 \ + >/dev/null 2>&1 || true # we read conclusion below; don't abort the loop on fail +done +``` + +`gh run watch` blocks until the run completes. Use `--interval 15` to be polite on rate limits. + +### Step 5 — Collect outcomes per PR + +For each PR, fetch the run conclusion and (if successful) the URL of the created cherry-pick PR. + +```bash +for PR in "${PRS[@]}"; do + RUN_ID="${PR_TO_RUN[$PR]}" + + if [[ "$RUN_ID" == "UNKNOWN" ]]; then + echo "PR #$PR: UNKNOWN (no run found)" + continue + fi + + CONCLUSION=$(gh run view "$RUN_ID" --repo sgl-project/sglang \ + --json conclusion,status,url \ + --jq '"\(.status) \(.conclusion) \(.url)"') + + STATUS=$(awk '{print $1}' <<<"$CONCLUSION") + RESULT=$(awk '{print $2}' <<<"$CONCLUSION") + RUN_URL=$(awk '{print $3}' <<<"$CONCLUSION") + + if [[ "$RESULT" == "success" ]]; then + # Find the cherry-pick PR created by this run. Title format from the workflow: + # "[Cherry-pick to ] (#)" + CP_PR=$(gh pr list --repo sgl-project/sglang \ + --base "$TARGET_BRANCH" \ + --label cherry-pick \ + --state all \ + --limit 30 \ + --json number,title,url,createdAt \ + --jq "[.[] | select(.title | contains(\"(#${PR})\"))][0]") + + CP_URL=$(jq -r '.url // "N/A"' <<<"$CP_PR") + CP_NUM=$(jq -r '.number // "?"' <<<"$CP_PR") + echo "PR #$PR -> SUCCESS cherry-pick PR #$CP_NUM ($CP_URL) [run: $RUN_URL]" + else + # Failure: pull the cherry-pick step's last error line so the user sees why. + REASON=$(gh run view "$RUN_ID" --repo sgl-project/sglang --log-failed 2>/dev/null \ + | grep -m1 -E "::error::" \ + | sed -E 's/^[^:]*::error::?//' \ + || echo "(see run logs)") + echo "PR #$PR -> $RESULT reason: $REASON [run: $RUN_URL]" + fi +done +``` + +### Step 6 — Final summary + +Print one table sorted by input order: + +```markdown +## Cherry-Pick Batch Summary — `release/v0.5.7` + +| PR | Status | Cherry-pick PR | Run | Notes | +|-----|----------|----------------|-----|-------| +| #25956 | success | #26031 | run/12345 | — | +| #25958 | failure | — | run/12346 | Cherry-pick of onto release/v0.5.7 failed due to conflicts | +| #25987 | success | #26032 | run/12347 | — | + +**Totals:** N succeeded, M failed, K unknown. +``` + +For any **failure**, suggest the manual fallback from the workflow's own error message: + +> Resolve locally: `git checkout release/v0.5.7 && git cherry-pick `, fix conflicts, push a branch, and open the PR by hand. + +## Common Failure Modes + +| Symptom | Cause | Action | +|---------|-------|--------| +| `PR #X is not merged (state=OPEN)` | PR not yet merged | Wait for merge or pass `--commit-sha` (not supported by this slash command) | +| `Target branch '...' does not exist` | Typo or branch not cut yet | Confirm branch name; release manager may not have cut it | +| `Cherry-pick of onto failed due to conflicts` | Code drift on release branch (should already have been caught in Step 2 pre-flight) | Do it manually as instructed above | +| Pre-flight `git merge-tree` reports a conflict | Same as above, caught locally before any CI run | Remove that PR from the batch and resolve manually | +| `No remote points to sgl-project/sglang` | Skill invoked from a checkout that only has a fork remote | Add the upstream remote: `git remote add upstream https://github.com/sgl-project/sglang.git` | +| Pre-flight `git merge-tree` errors with `unknown option` | git < 2.38 | Upgrade git, or run the skill on a machine with a modern git | +| Pre-flight reports `Prior cherry-pick: merged as #N` | The PR has already been cherry-picked and merged onto this release branch | Skip this PR — re-dispatching would create a duplicate PR. Verify #N is the right one before removing from the list. | +| Pre-flight reports `Prior cherry-pick: OPEN as #N` | A previous dispatch is still in flight (PR not yet merged or closed) | Default: skip and ask the user to land or close #N first. Re-dispatching creates a parallel duplicate that needs to be cleaned up afterwards. | +| Pre-flight is clean but `# files = 0` and no prior cherry-pick PR was found | Commit landed on the release branch by direct merge (not via the bot), or via a rebase that rewrote the SHA | Skip the dispatch — the change is already there. Surface this anomaly so the user knows the bot wasn't the source. | +| Multiple runs but only one detected | Two dispatches landed in the same `gh run list` poll cycle | Re-run for the missing PR, or look up its run by hand: `gh run list --workflow=bot-cherry-pick.yml --event workflow_dispatch -L 20` | +| `403` on `gh workflow run` | Missing `actions:write` on the token | Use a token that has workflow dispatch rights on `sgl-project/sglang` | + +## Notes + +- The skill **never modifies** the workflow file. It only dispatches it. +- The skill operates on the upstream repo only (`sgl-project/sglang`); the user's fork is irrelevant here. +- Per-branch concurrency means picking 20 PRs to the same release branch will take ~20× the runtime of one. There is no parallelism to be gained client-side. If the user batches across **different** target branches, those run concurrently. +- Do not skip the merged-state precheck — the workflow will reject unmerged PRs, but we want a single batched validation report up front rather than N individual workflow failures. +- The skill should be invoked with `gh auth status` already passing; if not, surface the auth error and stop. diff --git a/.claude/rules/speculative-naming.md b/.claude/skills/speculative-naming/SKILL.md similarity index 90% rename from .claude/rules/speculative-naming.md rename to .claude/skills/speculative-naming/SKILL.md index 76e6e09e7167..4109ba1b5742 100644 --- a/.claude/rules/speculative-naming.md +++ b/.claude/skills/speculative-naming/SKILL.md @@ -1,6 +1,11 @@ +--- +name: speculative-naming +description: Naming conventions for SGLang speculative decoding identifiers. Use when adding, renaming, or reviewing identifiers in speculative decoding code — anything under `python/sglang/srt/speculative/`, related attention backends, scheduler accumulators, IPC fields, observability metrics, or CLI flags. +--- + # Speculative Decoding — Naming Conventions -Apply this rule when adding, renaming, or reviewing identifiers in speculative decoding code (anything under `python/sglang/srt/speculative/`, related attention backends, scheduler accumulators, IPC fields, observability metrics, or CLI flags). +Apply this skill when adding, renaming, or reviewing identifiers in speculative decoding code (anything under `python/sglang/srt/speculative/`, related attention backends, scheduler accumulators, IPC fields, observability metrics, or CLI flags). ## Rule 1 — Verb form, drop `-ed` diff --git a/.codespellrc b/.codespellrc index 409cc4ce698a..3b258e417c44 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] -ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd, fo, visibles, nothink, renderD, ond, tbe, CopyIn +ignore-words-list = ans, als, hel, boostrap, childs, te, vas, hsa, ment, cann, thi, makro, wil, rouge, PRIS, ather, MIS, medias, allready, inout, nd, fo, visibles, nothink, renderD, ond, tbe, CopyIn, notin skip = *.json, *.jsonl, *.patch, *.txt, *.lock diff --git a/.github/CI_PERMISSIONS.json b/.github/CI_PERMISSIONS.json index 0811aabf2bd6..8cc8655ab9de 100644 --- a/.github/CI_PERMISSIONS.json +++ b/.github/CI_PERMISSIONS.json @@ -1399,6 +1399,13 @@ "cooldown_interval_minutes": 0, "reason": "top contributor" }, + "zianglih": { + "can_tag_run_ci_label": true, + "can_rerun_failed_ci": true, + "can_rerun_stage": true, + "cooldown_interval_minutes": 0, + "reason": "top contributor" + }, "zminglei": { "can_tag_run_ci_label": true, "can_rerun_failed_ci": true, diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 54df0e5479e5..c2f0ffa39946 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,6 @@ .github @merrymercy @Fridge003 @ispobock @Kangyan-Zhou @bingxche /docker @Fridge003 @ispobock @HaiShaw @ishandhanani @yctseng0211 -/docker/npu.Dockerfile @ping1jing2 @iforgetmyname +/docker/npu.Dockerfile @ping1jing2 @iforgetmyname @whybeyoung /docs @wisclmy0611 @zijiexia /docs_new @wisclmy0611 @zijiexia @Richardczl98 @JustinTong0323 /python/pyproject.toml @merrymercy @Fridge003 @ispobock @@ -29,7 +29,7 @@ /python/sglang/srt/grpc @CatherineSue @slin1237 /python/sglang/srt/hardware_backend/mlx @yeahdongcn /python/sglang/srt/hardware_backend/musa @yeahdongcn -/python/sglang/srt/hardware_backend/npu @ping1jing2 @iforgetmyname +/python/sglang/srt/hardware_backend/npu @ping1jing2 @iforgetmyname @whybeyoung /python/sglang/srt/hardware_backend/npu/quantization @OrangeRedeng @TamirBaydasov @iforgetmyname /python/sglang/srt/layers @merrymercy @Ying1123 @Fridge003 @ispobock @HaiShaw @ch-wan @BBuf @Edwardf0t1 /python/sglang/srt/layers/attention @merrymercy @Fridge003 @ispobock @Qiaolin-Yu @hebiao064 @HaiShaw @@ -45,6 +45,8 @@ /python/sglang/srt/managers/scheduler_pp_mixin.py @ShangmingCai @XucSh /python/sglang/srt/managers/tokenizer_manager_score_mixin.py @sundar24295s @chanh @fortunecookiee /python/sglang/srt/mem_cache @merrymercy @Ying1123 @hnyls2002 @xiezhq-hermann @hanming-lu @yizhang2077 @hzh0425 @ispobock +/python/sglang/srt/mem_cache/storage/mooncake_store/embedding_cache_controller.py @liusy58 +/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_embedding_store.py @liusy58 /python/sglang/srt/model_executor @merrymercy @Ying1123 @hnyls2002 @Fridge003 @ispobock /python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py @hebiao064 /python/sglang/srt/models/deepseek_common @Fridge003 @ispobock @fzyzcjy @ch-wan @@ -52,6 +54,7 @@ /python/sglang/srt/models/transformers.py @adarshxs /python/sglang/srt/multimodal @mickqian @JustinTong0323 @yhyang201 @yuan-luo /python/sglang/srt/observability @merrymercy @fzyzcjy @sufeng-buaa +/python/sglang/srt/platforms @merrymercy @whybeyoung /python/sglang/srt/ray @Qiaolin-Yu @xyuzh /python/sglang/srt/speculative @Ying1123 @merrymercy @hnyls2002 @Qiaolin-Yu /sgl-kernel @ispobock @BBuf @yizhang2077 @merrymercy @FlamingoPg @HaiShaw diff --git a/.github/actions/upload-cuda-coredumps/action.yml b/.github/actions/upload-cuda-coredumps/action.yml index 0e9fdde2799d..c741df174ac5 100644 --- a/.github/actions/upload-cuda-coredumps/action.yml +++ b/.github/actions/upload-cuda-coredumps/action.yml @@ -1,5 +1,5 @@ name: Upload CUDA Coredumps -description: Upload CUDA coredump files as artifacts and clean up the directory. +description: Upload CUDA coredump files as artifacts, optionally signal to a tracker issue, and clean up. inputs: artifact-suffix: @@ -10,17 +10,75 @@ inputs: description: Number of days to retain the artifact required: false default: "7" + tracker-issue: + description: | + If set, post a one-line comment to sgl-project/sglang issue + # when at least one coredump is detected. Requires + `bot-token` with issues:write on sgl-project/sglang. + required: false + default: "" + bot-token: + description: PAT with issues:write on sgl-project/sglang. Required when tracker-issue is set. + required: false + default: "" runs: using: composite steps: + - name: Check for coredumps + id: check + shell: bash + run: | + dir="${SGLANG_CUDA_COREDUMP_DIR:-/tmp/sglang_cuda_coredumps}" + if [ -d "$dir" ] && [ -n "$(ls -A "$dir" 2>/dev/null)" ]; then + echo "has_dumps=true" >> "$GITHUB_OUTPUT" + else + echo "has_dumps=false" >> "$GITHUB_OUTPUT" + fi + - name: Upload CUDA coredumps + if: steps.check.outputs.has_dumps == 'true' uses: actions/upload-artifact@v4 with: name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }} path: ${{ env.SGLANG_CUDA_COREDUMP_DIR || '/tmp/sglang_cuda_coredumps' }}/ retention-days: ${{ inputs.retention-days }} - if-no-files-found: ignore + + - name: Signal coredump to tracker issue + if: steps.check.outputs.has_dumps == 'true' && inputs.tracker-issue != '' && inputs.bot-token != '' + shell: bash + env: + BOT_TOKEN: ${{ inputs.bot-token }} + PR_NUM: ${{ github.event.pull_request.number }} + EVENT_NAME: ${{ github.event_name }} + TRACKER_ISSUE: ${{ inputs.tracker-issue }} + run: | + if [ -n "$PR_NUM" ]; then + ref_label="PR #${PR_NUM}" + else + ref_label="$EVENT_NAME" + fi + # Resolve own job_id via REST API: match by runner_name + status + # in_progress (the current job is the only one in_progress on this + # runner). Robust across matrix expansions and artifact-suffix shapes. + job_id=$(curl -sS \ + -H "Authorization: Bearer ${BOT_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs?per_page=100" \ + | python3 -c 'import json,sys,os; print(next((j["id"] for j in json.load(sys.stdin)["jobs"] if j.get("runner_name")==os.environ["RUNNER_NAME"] and j.get("status")=="in_progress"), ""))') + if [ -n "$job_id" ]; then + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/job/${job_id}" + else + # Fallback to run-attempt URL if job_id lookup failed + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" + fi + body_json=$(printf '{"body":"@hnyls2002 [Coredump Tracker] %s - %s"}' "${ref_label}" "${run_url}") + curl -sS -X POST \ + -H "Authorization: Bearer ${BOT_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/sgl-project/sglang/issues/${TRACKER_ISSUE}/comments" \ + -d "${body_json}" - name: Cleanup CUDA coredumps shell: bash diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index e544224d6b97..2c7dcd61870b 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -171,6 +171,8 @@ jobs: if: failure() with: artifact-suffix: ${{ matrix.partition }} + tracker-issue: "26340" + bot-token: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }} - name: Cleanup venv if: always() diff --git a/.github/workflows/bot-cherry-pick.yml b/.github/workflows/bot-cherry-pick.yml index 4f173820a7b5..1c5e26671226 100644 --- a/.github/workflows/bot-cherry-pick.yml +++ b/.github/workflows/bot-cherry-pick.yml @@ -20,15 +20,10 @@ permissions: contents: write pull-requests: write -concurrency: - group: cherry-pick-${{ github.event.inputs.target_branch }} - cancel-in-progress: false - jobs: cherry-pick: if: github.repository == 'sgl-project/sglang' runs-on: ubuntu-latest - environment: 'prod' steps: - name: Validate inputs env: diff --git a/.github/workflows/cancel-pr-workflow-on-merge.yml b/.github/workflows/cancel-pr-workflow-on-merge.yml index 535884ba6002..fcc457dcf2e3 100644 --- a/.github/workflows/cancel-pr-workflow-on-merge.yml +++ b/.github/workflows/cancel-pr-workflow-on-merge.yml @@ -1,4 +1,4 @@ -name: Cancel PR Workflows on Merge +name: Cancel PR Workflows on Close on: pull_request_target: @@ -10,7 +10,6 @@ permissions: jobs: cancel: - if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - name: Cancel Previous Runs diff --git a/.github/workflows/cancel-unfinished-pr-tests.yml b/.github/workflows/cancel-unfinished-pr-tests.yml index 9df969d582da..6f213d6363c6 100644 --- a/.github/workflows/cancel-unfinished-pr-tests.yml +++ b/.github/workflows/cancel-unfinished-pr-tests.yml @@ -8,12 +8,17 @@ on: description: 'Space-separated list of workflow filenames to cancel' required: true type: string - default: 'pr-test.yml' + default: 'pr-test.yml pr-test-extra.yml' include_high_priority: description: 'Also cancel runs from high-priority PRs' required: false type: boolean default: false + include_rerun_test: + description: 'Also cancel /rerun-test dispatched runs (rerun-test.yml)' + required: false + type: boolean + default: false permissions: actions: write # Needed to cancel runs @@ -30,34 +35,113 @@ jobs: - name: Install GitHub CLI run: sudo apt-get install -y gh jq - - name: Cancel unfinished PR-associated runs (skip high-priority PRs) + - name: Cancel unfinished PR-associated runs env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} - WORKFLOWS: ${{ github.event.inputs.workflows || 'pr-test.yml' }} - INCLUDE_HIGH_PRIORITY: ${{ github.event.inputs.include_high_priority || 'false' }} + WORKFLOWS: ${{ github.event.inputs.workflows }} + INCLUDE_HIGH_PRIORITY: ${{ github.event.inputs.include_high_priority }} + INCLUDE_RERUN_TEST: ${{ github.event.inputs.include_rerun_test }} shell: bash run: | set -euo pipefail # Read the space-separated string from the input into a bash array read -r -a WORKFLOW_FILES <<< "${WORKFLOWS}" + if [ "$INCLUDE_RERUN_TEST" = "true" ]; then + WORKFLOW_FILES+=("rerun-test.yml") + fi - echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${WORKFLOWS}" + echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${WORKFLOW_FILES[*]}" + echo "include_high_priority=$INCLUDE_HIGH_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST" echo "" + # Decide whether to cancel run_id given a PR-lookup endpoint. + # $1 = run_id + # $2 = gh api path returning a list of PR objects (head=... or commits//pulls) + # $3 = short label for log messages + maybe_cancel_for_pr() { + local run_id="$1" + local pr_query="$2" + local pr_label="$3" + + local pr_info + pr_info=$(gh api -H "Accept: application/vnd.github+json" "$pr_query" \ + --jq '.[0] | {number, state, merged_at}' 2>/dev/null || true) + + if [ -z "$pr_info" ] || [ "$pr_info" = "null" ]; then + echo " ⚠️ No PR found ($pr_label), skipping" + return + fi + + local pr_number pr_state + pr_number=$(echo "$pr_info" | jq -r '.number // empty') + pr_state=$(echo "$pr_info" | jq -r '.state // empty') + + if [ -z "$pr_number" ]; then + echo " ⚠️ PR lookup returned empty number, skipping" + return + fi + + local pr_url="https://github.com/$REPO/pull/$pr_number" + echo " PR: $pr_url ($pr_state)" + + # Closed PR (merged or not): always cancel, skip label checks. + if [ "$pr_state" = "closed" ]; then + echo " 🚫 Cancelling (PR closed)..." + gh run cancel "$run_id" --repo "$REPO" || echo " ⚠️ Cancellation failed" + return + fi + + # Open PR: apply label-based skip rules. + local labels + labels=$(gh pr view "$pr_number" --repo "$REPO" --json labels \ + | jq -r '.labels[].name' 2>/dev/null || true) + + if echo "$labels" | grep -Fxq "bypass-maintenance"; then + echo " 🛑 Skipping (bypass-maintenance label, never cancelled)" + return + fi + + if echo "$labels" | grep -Fxq "high priority"; then + if [ "$INCLUDE_HIGH_PRIORITY" != "true" ]; then + echo " 🛑 Skipping (high priority label)" + return + fi + echo " ⚠️ High priority PR, but include_high_priority is enabled" + fi + + echo " 🚫 Cancelling..." + gh run cancel "$run_id" --repo "$REPO" || echo " ⚠️ Cancellation failed" + } + export -f maybe_cancel_for_pr + for workflow_file in "${WORKFLOW_FILES[@]}"; do echo "=========================================" echo "Workflow: $workflow_file" echo "=========================================" - # Get all unfinished runs - all_runs=$(gh run list \ - --repo "$REPO" \ - --workflow "$workflow_file" \ - --json databaseId,status,event,url,createdAt \ - --limit 1000 \ - | jq -c '.[] | select(.status=="queued" or .status=="waiting" or .status=="in_progress")') + # Get all unfinished runs. + # Use server-side --status filter: without it, `gh run list --limit 1000` + # only sees the most recent 1000 runs by createdAt, which on busy workflows + # like pr-test.yml is < 3 days. Old stuck runs would be missed. + all_runs="" + for status in queued in_progress waiting; do + batch=$(gh run list \ + --repo "$REPO" \ + --workflow "$workflow_file" \ + --status "$status" \ + --json databaseId,status,event,url,createdAt,displayTitle \ + --limit 1000 \ + | jq -c '.[]') + if [ -n "$batch" ]; then + if [ -n "$all_runs" ]; then + all_runs="$all_runs"$'\n'"$batch" + else + all_runs="$batch" + fi + fi + done if [ -z "$all_runs" ]; then echo "✅ No unfinished runs found" @@ -68,15 +152,16 @@ jobs: # Count runs by event type total_runs=$(echo "$all_runs" | wc -l) pr_runs=$(echo "$all_runs" | jq -s '[.[] | select(.event=="pull_request")] | length') - other_runs=$(echo "$all_runs" | jq -s '[.[] | select(.event!="pull_request")] | length') + dispatch_runs=$(echo "$all_runs" | jq -s '[.[] | select(.event=="workflow_dispatch")] | length') + other_runs=$(echo "$all_runs" | jq -s '[.[] | select(.event!="pull_request" and .event!="workflow_dispatch")] | length') - echo "📊 Summary: $total_runs unfinished runs ($pr_runs PR-related, $other_runs other)" + echo "📊 Summary: $total_runs unfinished ($pr_runs pull_request, $dispatch_runs workflow_dispatch, $other_runs other)" echo "" - # Process non-PR runs first + # Other runs: list only, do not cancel. if [ "$other_runs" -gt 0 ]; then - echo "--- Non-PR Runs ---" - echo "$all_runs" | jq -c 'select(.event!="pull_request")' | while read -r run; do + echo "--- Other Runs (listed only, not cancelled) ---" + echo "$all_runs" | jq -c 'select(.event!="pull_request" and .event!="workflow_dispatch")' | while read -r run; do run_url=$(echo "$run" | jq -r '.url') run_event=$(echo "$run" | jq -r '.event') run_status=$(echo "$run" | jq -r '.status') @@ -85,9 +170,9 @@ jobs: echo "" fi - # Process PR runs + # PR runs: resolve PR via head=owner:branch. if [ "$pr_runs" -gt 0 ]; then - echo "--- PR Runs (checking for cancellation) ---" + echo "--- PR Runs (resolving via head=owner:branch) ---" echo "$all_runs" | jq -c 'select(.event=="pull_request")' | while read -r run; do run_id=$(echo "$run" | jq -r '.databaseId') run_url=$(echo "$run" | jq -r '.url') @@ -96,62 +181,56 @@ jobs: echo "" echo "Run ($run_status): $run_url" - # Fetch full run details to get head repository and branch info run_details=$(gh api -H "Accept: application/vnd.github+json" \ "repos/$REPO/actions/runs/$run_id" 2>/dev/null || true) - if [ -z "$run_details" ]; then echo " ⚠️ Could not fetch run details, skipping" continue fi - # Get head owner and branch (works for both fork and non-fork PRs) head_owner=$(echo "$run_details" | jq -r '.head_repository.owner.login // empty') head_branch=$(echo "$run_details" | jq -r '.head_branch // empty') - if [ -z "$head_owner" ] || [ -z "$head_branch" ]; then echo " ⚠️ Missing head info, skipping" continue fi echo " Branch: ${head_owner}:${head_branch}" + maybe_cancel_for_pr "$run_id" \ + "repos/$REPO/pulls?state=all&head=${head_owner}:${head_branch}" \ + "head=${head_owner}:${head_branch}" + done + echo "" + fi - # Find PR by searching with head=owner:branch - pr_number=$(gh api -H "Accept: application/vnd.github+json" \ - "repos/$REPO/pulls?state=open&head=${head_owner}:${head_branch}" \ - --jq '.[0].number // empty' 2>/dev/null || true) - - if [ -z "$pr_number" ]; then - echo " ⚠️ No open PR found, skipping" - continue - fi - - pr_url="https://github.com/$REPO/pull/$pr_number" - echo " PR: $pr_url" + # workflow_dispatch runs (e.g. /rerun-test): resolve PR via pr_head_sha + # parsed from run-name (`[rerun-test] `). + if [ "$dispatch_runs" -gt 0 ]; then + echo "--- Dispatch Runs (resolving via pr_head_sha in display_title) ---" + echo "$all_runs" | jq -c 'select(.event=="workflow_dispatch")' | while read -r run; do + run_id=$(echo "$run" | jq -r '.databaseId') + run_url=$(echo "$run" | jq -r '.url') + run_status=$(echo "$run" | jq -r '.status') + display_title=$(echo "$run" | jq -r '.displayTitle // empty') - # Check for high priority label - labels=$(gh pr view "$pr_number" --repo "$REPO" --json labels \ - | jq -r '.labels[].name' 2>/dev/null || true) + echo "" + echo "Run ($run_status): $run_url" + echo " Title: $display_title" - if echo "$labels" | grep -Fxq "bypass-maintenance"; then - echo " 🛑 Skipping (bypass-maintenance label, never cancelled)" + # Last whitespace-delimited token if it is a 40-hex SHA. + last_token=$(echo "$display_title" | awk '{print $NF}') + if ! [[ "$last_token" =~ ^[0-9a-f]{40}$ ]]; then + echo " ⚠️ No pr_head_sha in title, skipping" continue fi - if echo "$labels" | grep -Fxq "high priority"; then - if [ "$INCLUDE_HIGH_PRIORITY" != "true" ]; then - echo " 🛑 Skipping (high priority label)" - continue - fi - echo " ⚠️ High priority PR, but include_high_priority is enabled" - fi - - echo " 🚫 Cancelling..." - gh run cancel "$run_id" --repo "$REPO" || echo " ⚠️ Cancellation failed" + echo " pr_head_sha: $last_token" + maybe_cancel_for_pr "$run_id" \ + "repos/$REPO/commits/$last_token/pulls" \ + "sha=$last_token" done + echo "" fi - - echo "" done echo "=========================================" diff --git a/.github/workflows/diffusion-ci-gt-gen.yml b/.github/workflows/diffusion-ci-gt-gen.yml index 909cfff54053..2c935f0a1b67 100644 --- a/.github/workflows/diffusion-ci-gt-gen.yml +++ b/.github/workflows/diffusion-ci-gt-gen.yml @@ -85,6 +85,7 @@ jobs: "diffusers": [ "flux_2_image_t2i", "flux_2_klein_image_t2i", + "flux_2_klein_base_image_t2i", "flux_2_ti2i", "flux_image_t2i", "qwen_image_edit_2509_ti2i", @@ -120,6 +121,7 @@ jobs: h200_cases = { "flux_2_image_t2i", "flux_2_klein_image_t2i", + "flux_2_klein_base_image_t2i", "flux_2_ti2i", } include = [] @@ -292,6 +294,15 @@ jobs: python/${{ env.OUTPUT_NAME }}/official_ltx23_manifest.json retention-days: 7 + - name: Validate generated GT images + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + run: | + python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ + --source-dir python/${{ env.OUTPUT_NAME }} \ + --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \ + --check-only + - name: Publish official GT images to sgl-project/ci-data env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} @@ -392,6 +403,15 @@ jobs: path: python/${{ env.OUTPUT_NAME }} retention-days: 7 + - name: Validate generated GT images + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + run: | + python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ + --source-dir python/${{ env.OUTPUT_NAME }} \ + --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \ + --check-only + - name: Publish GT images to sgl-project/ci-data env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} @@ -458,6 +478,15 @@ jobs: path: python/${{ env.OUTPUT_NAME }} retention-days: 7 + - name: Validate generated GT images + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + run: | + python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ + --source-dir python/${{ env.OUTPUT_NAME }} \ + --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \ + --check-only + - name: Publish GT images to sgl-project/ci-data env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} @@ -524,6 +553,15 @@ jobs: path: python/${{ env.OUTPUT_NAME }} retention-days: 7 + - name: Validate generated GT images + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + run: | + python scripts/ci/utils/diffusion/publish_diffusion_gt.py \ + --source-dir python/${{ env.OUTPUT_NAME }} \ + --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \ + --check-only + - name: Publish GT images to sgl-project/ci-data env: GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 72a67d25b0e4..f9ae7f05a46e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,6 +11,31 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Reject changes under legacy docs/ + if: github.event_name == 'pull_request' + run: | + set -euo pipefail + BASE_REF="${{ github.base_ref }}" + # Refresh origin/ with full history; --depth=1 would + # shallow the ref and break the merge-base used by `...`. + git fetch --no-tags origin "$BASE_REF" + # First, verify the diff itself succeeds and check whether any + # files changed. A silent failure here would let docs/ changes + # through, so the explicit `if !` guard is important. + if ! CHANGED=$(git diff --name-only --diff-filter=ACMRDTUXB "origin/${BASE_REF}...HEAD"); then + echo "git diff origin/${BASE_REF}...HEAD failed; aborting." >&2 + exit 2 + fi + if [ -z "$CHANGED" ]; then + echo "No changed files vs origin/${BASE_REF}; skipping." + exit 0 + fi + # Re-emit with -z so xargs can safely handle whitespace in paths. + git diff -z --name-only --diff-filter=ACMRDTUXB "origin/${BASE_REF}...HEAD" \ + | xargs -0 python3 scripts/ci/check_no_docs_changes.py - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/nightly-experimental-sgl-router-docker.yml b/.github/workflows/nightly-experimental-sgl-router-docker.yml new file mode 100644 index 000000000000..e070382afcd0 --- /dev/null +++ b/.github/workflows/nightly-experimental-sgl-router-docker.yml @@ -0,0 +1,34 @@ +name: Nightly Experimental sgl-router Docker Image + +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + docker buildx build . -f docker/sgl-router.Dockerfile \ + --platform linux/amd64 \ + -t lmsysorg/sglang-staging:experimental-router-dev \ + --push diff --git a/.github/workflows/nightly-test-musa.yml b/.github/workflows/nightly-test-musa.yml new file mode 100644 index 000000000000..9216800ddff3 --- /dev/null +++ b/.github/workflows/nightly-test-musa.yml @@ -0,0 +1,252 @@ +name: Nightly Test (MUSA) + +on: + schedule: + - cron: '0 16 * * *' + workflow_dispatch: + inputs: + job_filter: + description: "Select which job to run (empty/all to run all jobs)" + required: false + type: choice + default: 'all' + options: + - 'all' + - 'nightly-test-musa-general-kernel' + - 'nightly-test-musa-general-multimodal-layer' + - 'nightly-test-multimodal-server-1-gpu-musa' + - 'nightly-test-multimodal-server-2-gpu-musa' + workflow_call: + inputs: + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + required: false + type: string + default: '' + job_filter: + description: 'Select which job to run (empty or "all" to run all jobs)' + required: false + type: string + default: 'all' + +concurrency: + group: nightly-test-musa-${{ inputs.ref || github.ref }} + cancel-in-progress: ${{ github.event_name != 'workflow_call' }} + +env: + SGLANG_IS_IN_CI: true + +jobs: + # ==================== General: kernel ==================== + nightly-test-musa-general-kernel: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-1-gpu-runner + timeout-minutes: 240 + env: + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HUB_CACHE: /hf-cache/hub + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-kernel" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + timeout-minutes: 10 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 10 + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run sgl-kernel unit tests (MUSA) + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 30 + run: | + pytest sgl-kernel/tests/test_dsv3_router_gemm.py + pytest sgl-kernel/tests/test_per_token_quant_fp8.py + pytest sgl-kernel/tests/speculative/test_eagle_utils.py + pytest sgl-kernel/tests/speculative/test_ngram_utils.py + pytest sgl-kernel/tests/speculative/test_speculative_sampling.py + pytest sgl-kernel/tests/test_torch_defaults_reset.py + + # ==================== General: multimodal layer ==================== + nightly-test-musa-general-multimodal-layer: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-1-gpu-runner + timeout-minutes: 240 + env: + SGLANG_USE_MODELSCOPE: false + SGLANG_IS_IN_CI: true + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-multimodal-layer" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run multimodal MUSA layer unit tests + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 30 + run: | + pytest python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py + pytest python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py + + # ==================== Multimodal: 1-GPU (split) ==================== + nightly-test-multimodal-server-1-gpu-musa: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-1-gpu-runner + strategy: + fail-fast: false + max-parallel: 2 + matrix: + part: [0, 1] + timeout-minutes: 240 + env: + SGLANG_USE_MODELSCOPE: false + SGLANG_IS_IN_CI: true + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-1-gpu-musa" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run diffusion server tests (1-GPU) + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 60 + env: + RUNAI_STREAMER_MEMORY_LIMIT: 0 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite_musa.py \ + --suite 1-gpu-musa-nightly \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 \ + --continue-on-error + + # ==================== Multimodal: 2-GPU ==================== + nightly-test-multimodal-server-2-gpu-musa: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-2-gpu-runner + timeout-minutes: 240 + env: + SGLANG_USE_MODELSCOPE: false + SGLANG_IS_IN_CI: true + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-2-gpu-musa" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run diffusion server tests (2-GPU) + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 60 + env: + RUNAI_STREAMER_MEMORY_LIMIT: 0 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite_musa.py \ + --suite 2-gpu-musa \ + --continue-on-error + + # ==================== finish ==================== + nightly-test-musa-finish: + needs: + [ + nightly-test-musa-general-kernel, + nightly-test-musa-general-multimodal-layer, + nightly-test-multimodal-server-1-gpu-musa, + nightly-test-multimodal-server-2-gpu-musa, + ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + json_needs='${{ toJson(needs) }}' + job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]') + + for job in $job_names; do + result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') + echo "$job: $result" + if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then + echo "Nightly failed." + exit 1 + fi + done + echo "All jobs completed successfully." diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index ef79e9764631..316441b40d57 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -326,7 +326,7 @@ jobs: run: | bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 15 + timeout-minutes: 30 run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} @@ -409,7 +409,7 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 60 run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} @@ -450,7 +450,7 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 45 run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} @@ -533,7 +533,7 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 45 run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} @@ -575,7 +575,7 @@ jobs: - name: Install dependencies run: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 45 run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index ce64fac11f6f..3e4057712246 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -336,7 +336,7 @@ jobs: bash scripts/ci/amd/amd_ci_install_dependency.sh - name: Run test - timeout-minutes: 15 + timeout-minutes: 30 run: | bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml index 9abafa7c01e3..6a6d356ac944 100644 --- a/.github/workflows/pr-test-extra.yml +++ b/.github/workflows/pr-test-extra.yml @@ -33,6 +33,11 @@ on: required: false type: boolean default: false + run_all_tests: + description: "Run all tests — bypasses paths-filter (needed when dispatching against long-lived branches whose diff against main can't resolve a merge-base in a shallow clone)" + required: false + type: boolean + default: false workflow_call: inputs: git_ref: diff --git a/.github/workflows/pr-test-musa.yml b/.github/workflows/pr-test-musa.yml index 21381c652db2..5d6ae58da12d 100644 --- a/.github/workflows/pr-test-musa.yml +++ b/.github/workflows/pr-test-musa.yml @@ -68,19 +68,22 @@ jobs: with: filters: | main_package: - - "python/sglang/!(multimodal_gen)/**" + - ".github/workflows/pr-test-musa.yml" - "python/pyproject_other.toml" + - "python/sglang/!(multimodal_gen)/**" + - "python/sglang/srt/hardware_backend/musa/**" - "scripts/ci/musa/*" - "scripts/ci/utils/*" - - "test/**" - - ".github/workflows/pr-test-musa.yml" multimodal_gen: - - "python/sglang/multimodal_gen/**" - - "python/sglang/cli/**" - "python/pyproject_other.toml" + - "python/sglang/multimodal_gen/runtime/platforms/musa.py" + - "python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py" + - "python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py" + - "python/sglang/multimodal_gen/test/run_suite_musa.py" + - "python/sglang/multimodal_gen/test/server/musa/**" sgl_kernel: - - "sgl-kernel/**" - ".github/workflows/pr-test-musa.yml" + - "sgl-kernel/csrc/musa/**" # ==================== PR Gate ==================== # pr-gate: @@ -100,10 +103,12 @@ jobs: runs-on: s5000-1-gpu-runner timeout-minutes: 240 env: - USE_MODELSCOPE: true + SGLANG_USE_MODELSCOPE: false SGLANG_IS_IN_CI: true TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 steps: - name: Checkout code timeout-minutes: 10 @@ -112,6 +117,7 @@ jobs: ref: ${{ inputs.ref || github.ref }} - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh @@ -132,10 +138,12 @@ jobs: runs-on: s5000-2-gpu-runner timeout-minutes: 240 env: - USE_MODELSCOPE: true + SGLANG_USE_MODELSCOPE: false SGLANG_IS_IN_CI: true TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 steps: - name: Checkout code timeout-minutes: 10 @@ -144,6 +152,7 @@ jobs: ref: ${{ inputs.ref || github.ref }} - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh @@ -162,16 +171,19 @@ jobs: runs-on: s5000-1-gpu-runner timeout-minutes: 240 env: - USE_MODELSCOPE: true + SGLANG_USE_MODELSCOPE: false SGLANG_IS_IN_CI: true TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 steps: - name: Checkout code timeout-minutes: 10 uses: actions/checkout@v4 - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh @@ -198,6 +210,7 @@ jobs: ref: ${{ inputs.ref || github.ref }} - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh diff --git a/.github/workflows/pr-test-sgl-router.yml b/.github/workflows/pr-test-sgl-router.yml index 2e1ec6be6924..585ff2749782 100644 --- a/.github/workflows/pr-test-sgl-router.yml +++ b/.github/workflows/pr-test-sgl-router.yml @@ -1,35 +1,101 @@ name: PR Test (sgl-router) +# Trigger contract — modeled on `pr-test.yml`: +# +# * No `paths:` filter at the trigger level. The workflow ALWAYS +# fires on every PR synchronize / push, so the run record always +# appears as a check on the PR (no more "workflow silently didn't +# fire" mystery debugging). Path-based skip decisions are made +# inside the `sgl-router-gate` job below, where we can log the +# reason. Same for the `run-ci` label gate — moved off the +# workflow trigger and into the gate job so its outcome is +# observable. +# +# * `push` on `main` keeps firing so post-merge runs still record +# against the default branch. on: push: branches: [main] - paths: - - "experimental/sgl-router/**" - - ".github/workflows/pr-test-sgl-router.yml" - - "scripts/ci/cuda/ci_install_dependency.sh" pull_request: branches: [main] types: [opened, synchronize, reopened, labeled] - paths: - - "experimental/sgl-router/**" - - ".github/workflows/pr-test-sgl-router.yml" - - "scripts/ci/cuda/ci_install_dependency.sh" workflow_dispatch: concurrency: - group: sgl-router-${{ github.ref }} + group: sgl-router-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }} cancel-in-progress: true env: SGLANG_IS_IN_CI: true jobs: + # Stage 0 — decide whether to run the heavy tiers. Always runs + # (cheap, ubuntu-latest), and emits a single `should_run` output + # consumed by every tier below. Reasons it might evict downstream + # work: + # * `pull_request` event without the `run-ci` label (budget gate) + # * No files matching the sgl-router paths changed in this PR + # Either outcome is logged on the gate job's page, so operators can + # see *why* the tiers were skipped instead of guessing. + sgl-router-gate: + name: gate + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.decide.outputs.should_run }} + paths_changed: ${{ steps.paths.outputs.sgl_router }} + has_run_ci_label: ${{ steps.label.outputs.has_run_ci }} + steps: + - uses: actions/checkout@v4 + # `dorny/paths-filter` computes the diff against the PR base + # (for pull_request) or the push's `before` SHA (for push) and + # exposes `sgl_router=true|false` on whether any tracked path + # changed. Replaces the old workflow-level `paths:` filter so + # the gate is observable. + - name: Detect sgl-router path changes + id: paths + uses: dorny/paths-filter@v3 + with: + filters: | + sgl_router: + - 'experimental/sgl-router/**' + - '.github/workflows/pr-test-sgl-router.yml' + - 'scripts/ci/cuda/ci_install_dependency.sh' + # `run-ci` label gate. Cheap workflow runs (e.g. docs-only PRs + # that happen to touch the sgl-router dir) still need the + # opt-in label before consuming the H100 / kind tiers below. + # `push` events on main and `workflow_dispatch` skip this gate. + - name: Check run-ci label + id: label + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "has_run_ci=true" >> "$GITHUB_OUTPUT" + echo "Non-PR event (${{ github.event_name }}); label gate bypassed." + exit 0 + fi + if [[ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci') }}" == "true" ]]; then + echo "has_run_ci=true" >> "$GITHUB_OUTPUT" + echo "PR has run-ci label; downstream tiers will run." + else + echo "has_run_ci=false" >> "$GITHUB_OUTPUT" + echo "::warning::PR is missing the 'run-ci' label; skipping sgl-router tiers. Apply the label to opt in." + fi + - name: Decide + id: decide + run: | + paths='${{ steps.paths.outputs.sgl_router }}' + label='${{ steps.label.outputs.has_run_ci }}' + if [[ "$paths" == "true" && "$label" == "true" ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + echo "Both path filter and run-ci label match; running tiers." + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + echo "Skipping tiers (paths_changed=$paths, has_run_ci=$label)." + fi + sgl-router-lint: name: tier-1 — lint - if: | - github.event_name != 'pull_request' || - (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || - (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + needs: sgl-router-gate + if: needs.sgl-router-gate.outputs.should_run == 'true' runs-on: ubuntu-latest env: # Scoped per-job (not workflow-wide) to avoid SMG's documented breakage @@ -120,11 +186,8 @@ jobs: sgl-router-build-and-test: name: tier-2 — build + test - needs: sgl-router-lint - if: | - github.event_name != 'pull_request' || - (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || - (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + needs: [sgl-router-gate, sgl-router-lint] + if: needs.sgl-router-gate.outputs.should_run == 'true' runs-on: ubuntu-latest env: RUSTC_WRAPPER: sccache @@ -177,10 +240,13 @@ jobs: - name: cargo test working-directory: experimental/sgl-router - # Skip tokenizer_parity here: ubuntu-latest has no HuggingFace cache, - # so the matrix would hard-fail (see the test docstring). The e2e job - # runs it after pytest populates the Qwen3-0.6B tokenizer.json. - run: cargo test --release -- --skip tokenizer_parity + # Skip the tokenizer parity_matrix test here: ubuntu-latest has + # no HuggingFace cache, so the matrix would hard-fail (see the + # test docstring). The e2e job runs it after pytest populates + # the Qwen3-0.6B tokenizer.json. Filter is matched against the + # full test path `tokenizer::parity::parity_matrix`; substring + # `parity_matrix` is unique to that test. + run: cargo test --release -- --skip parity_matrix # Regenerate the cross-impl block-hash parity fixture and fail if it # differs from the committed file. The Python script replicates @@ -218,11 +284,8 @@ jobs: sgl-router-docker-build-test: name: tier-3 — docker (placeholder) - needs: sgl-router-build-and-test - if: | - github.event_name != 'pull_request' || - (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || - (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + needs: [sgl-router-gate, sgl-router-build-and-test] + if: needs.sgl-router-gate.outputs.should_run == 'true' runs-on: ubuntu-24.04 timeout-minutes: 15 steps: @@ -231,13 +294,16 @@ jobs: echo "docker-build-test not implemented yet." exit 0 + # tier-3 k8s integration is decoupled from tier-3 e2e: each runs on a + # different runner type (ubuntu kind cluster vs H100), each manages its + # own test scope (k8s_integration/ vs everything-else-in-e2e), and + # each is allowed to fail without blocking the other. They both still + # gate on tier-2's build+test passing — a Rust compile failure blocks + # the GPU/cluster runners from spinning up. sgl-router-k8s-integration: name: tier-3 — k8s integration - needs: sgl-router-build-and-test - if: | - github.event_name != 'pull_request' || - (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || - (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + needs: [sgl-router-gate, sgl-router-build-and-test] + if: needs.sgl-router-gate.outputs.should_run == 'true' runs-on: ubuntu-22.04 timeout-minutes: 30 steps: @@ -267,21 +333,30 @@ jobs: /tmp/e2e-venv/bin/pip install -r experimental/sgl-router/tests/e2e/k8s_integration/requirements.txt - name: Run E2E run: /tmp/e2e-venv/bin/pytest experimental/sgl-router/tests/e2e/k8s_integration/ -v --tb=short - - name: Dump router logs on failure + - name: Dump cluster + router diagnostics on failure if: failure() - run: kubectl -n sgl-router-test logs deploy/sgl-router --tail=200 || true + run: | + set +e + kubectl -n sgl-router-test get pods -o wide + kubectl -n sgl-router-test describe pod -l app=sgl-router + kubectl -n sgl-router-test describe pod -l app=sglang + kubectl -n sgl-router-test logs deploy/sgl-router --tail=300 + kubectl -n sgl-router-test logs deploy/sgl-router --tail=300 --previous + kubectl -n sgl-router-test get events --sort-by=.lastTimestamp + kubectl -n sgl-router-test get endpointslices -o wide + kubectl -n sgl-router-test get svc + true sgl-router-e2e: name: tier-3 — e2e - needs: sgl-router-build-and-test - if: | - github.event_name != 'pull_request' || - (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || - (github.event.action == 'labeled' && github.event.label.name == 'run-ci') + needs: [sgl-router-gate, sgl-router-build-and-test] + if: needs.sgl-router-gate.outputs.should_run == 'true' runs-on: 2-gpu-h100 timeout-minutes: 45 steps: - uses: actions/checkout@v4 + - name: Install Rust toolchain + run: bash scripts/ci/utils/install_rust_protoc.sh - name: Rust cache uses: Swatinem/rust-cache@v2 with: @@ -289,7 +364,9 @@ jobs: shared-key: "sgl-router-cache" - name: cargo build (release) working-directory: experimental/sgl-router - run: cargo build --release + run: | + source "$HOME/.cargo/env" + cargo build --release # Install SGLang from the local checkout in editable mode (no PyPI # version pin) — mirrors `pr-test-rust.yml` so the router e2e runs @@ -306,41 +383,51 @@ jobs: run: | python3 -m pip install -r experimental/sgl-router/tests/e2e/requirements.txt + # IMPORTANT: --ignore=tests/e2e/k8s_integration. The k8s integration + # suite is owned by the `sgl-router-k8s-integration` job which has + # kind + kubectl installed and a real cluster running. The e2e GPU + # runner has neither, so pytest's recursive collection of + # tests/e2e/ would ERROR every k8s test at setup. Both tiers must + # be allowed to fail independently — k8s flakes (cluster boot, + # image pull) must not surface as e2e failures on H100, and e2e + # flakes (model load, HF auth) must not surface as k8s failures. - name: Run e2e working-directory: experimental/sgl-router env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: python3 -m pytest tests/e2e/ -v -s --tb=short + run: | + python3 -m pytest tests/e2e/ -v -s --tb=short \ + --ignore=tests/e2e/k8s_integration - name: Tokenizer parity matrix (uses HF cache populated by e2e) working-directory: experimental/sgl-router env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - # Runs after pytest so the Qwen3-0.6B tokenizer.json is cached by the - # SGLang worker. Cells without a cached snapshot are skipped; if no - # cells could be checked, the test hard-fails under SGLANG_IS_IN_CI. - run: cargo test --release --test component tokenizer::parity + run: | + source "$HOME/.cargo/env" + cargo test --release --test component tokenizer::parity sgl-router-finish: name: finish needs: + - sgl-router-gate - sgl-router-lint - sgl-router-build-and-test - sgl-router-docker-build-test - sgl-router-k8s-integration - sgl-router-e2e - # Gate finish on the same label as upstream jobs to avoid false-green on - # unlabeled PRs (skipped needs => success), and fail when any upstream - # job actually failed or was cancelled. + # `always()` lets `finish` run after upstream `skipped` outcomes + # (when the gate decided to skip). The downstream-skipped path is + # an explicit success — the workflow was correctly bypassed for a + # PR that didn't need full CI. The downstream-failure / + # downstream-cancelled path is a real fail. We also accept the + # gate's `skipped` (impossible today, kept for symmetry) and + # require the gate itself to have succeeded. if: | always() && + needs.sgl-router-gate.result == 'success' && !contains(needs.*.result, 'failure') && - !contains(needs.*.result, 'cancelled') && - ( - github.event_name != 'pull_request' || - (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) || - (github.event.action == 'labeled' && github.event.label.name == 'run-ci') - ) + !contains(needs.*.result, 'cancelled') runs-on: ubuntu-latest steps: - name: All required checks completed diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b595a9b15b1f..8d5ba07d3a48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,9 +12,14 @@ repos: - id: end-of-file-fixer - id: check-yaml args: [--allow-multiple-documents] + # Helm chart templates embed Go template syntax ({{- ... -}}) + # that is not valid YAML on its own; the rendered output is + # validated by `helm template` / `helm lint`. + exclude: ^experimental/sgl-router/helm/.*/templates/.*\.(yaml|tpl)$ - id: check-toml - id: check-ast - id: check-added-large-files + args: ['--maxkb=1500'] - id: check-merge-conflict - id: check-shebang-scripts-are-executable - id: detect-private-key @@ -130,6 +135,12 @@ repos: language: system files: ^sgl-model-gateway/.*\.rs$ pass_filenames: false + - id: rustfmt-sgl-router + name: rustfmt experimental/sgl-router + entry: bash -c 'cd experimental/sgl-router && cargo fmt -- --check' + language: system + files: ^experimental/sgl-router/.*\.rs$ + pass_filenames: false - repo: https://github.com/lycheeverse/lychee.git rev: lychee-v0.22.0 hooks: diff --git a/benchmark/bench_linear_attention/bench_gdn_prefill_cutedsl.py b/benchmark/bench_linear_attention/bench_gdn_prefill_cutedsl.py new file mode 100644 index 000000000000..acc9fd57f8c6 --- /dev/null +++ b/benchmark/bench_linear_attention/bench_gdn_prefill_cutedsl.py @@ -0,0 +1,473 @@ +""" +Benchmark & Correctness: Triton GDN vs CuTeDSL GDN (prefill, SM100 Blackwell). + +Compares: + - Triton: sglang's chunk_gated_delta_rule (FLA chunkwise, fp32 state, K-contig pool) + - CuteDSL: ported vLLM #43273 chunk_gated_delta_rule_cutedsl (SM100 only) + +The two kernels share the same math and the same g/beta convention (log-space +g, post-sigmoid beta). The CuteDSL kernel needs pre-allocated chunk metadata +from prepare_metadata_cutedsl, and l2norm is done outside the kernel. + +Reports correctness (output & state matching) and performance (ms, TFLOPS, TB/s). + +Usage: + python bench_gdn_prefill_cutedsl.py # default sweep + python bench_gdn_prefill_cutedsl.py --mode bench # benchmark only + python bench_gdn_prefill_cutedsl.py --mode correctness # correctness only + python bench_gdn_prefill_cutedsl.py --preset qwen3-next # Qwen3-Next config +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "python")) + +import torch + +from sglang.srt.layers.attention.fla.chunk import ( + chunk_gated_delta_rule as triton_chunk_gated_delta_rule, +) +from sglang.srt.layers.attention.fla.l2norm import l2norm_fwd +from sglang.srt.layers.attention.linear.kernels.gdn_blackwell import ( + chunk_gated_delta_rule_cutedsl, + prepare_metadata_cutedsl, +) + +# --------------------------------------------------------------------------- +# Helpers (shared shape: pool layout [N, H, K, V] with K-last stride) +# --------------------------------------------------------------------------- + + +def make_k_contiguous(t: torch.Tensor) -> torch.Tensor: + """K-last view of a logical [..., K, V] tensor (physically [..., V, K]).""" + return t.transpose(-2, -1).contiguous().transpose(-2, -1) + + +def gdn_flops(total_seq_len, num_heads, head_size_k, head_size_v): + """Per-token-per-head: k@v^T outer (2*K*V) + q@state output (2*K*V).""" + return 4 * total_seq_len * num_heads * head_size_k * head_size_v + + +def gdn_bytes( + total_seq_len, num_q_heads, num_v_heads, head_size_k, head_size_v, num_seqs, dtype +): + num_o_heads = max(num_q_heads, num_v_heads) + elem = dtype.itemsize + q_b = total_seq_len * num_q_heads * head_size_k * elem + k_b = total_seq_len * num_v_heads * head_size_k * elem + v_b = total_seq_len * num_v_heads * head_size_v * elem + o_b = total_seq_len * num_o_heads * head_size_v * elem + state_b = 2 * num_seqs * num_o_heads * head_size_k * head_size_v * 4 # fp32 r/w + g_b = total_seq_len * num_o_heads * 4 + beta_b = total_seq_len * num_o_heads * 4 + return q_b + k_b + v_b + o_b + state_b + g_b + beta_b + + +# --------------------------------------------------------------------------- +# Input factory +# --------------------------------------------------------------------------- + + +def make_inputs( + B, T_per_seq, H, K, V, pool_size, device, dtype, sequential_indices=False, seed=42 +): + T = B * T_per_seq + torch.manual_seed(seed) + + if sequential_indices: + cache_indices = torch.arange(B, dtype=torch.int32, device=device) + else: + perm = torch.randperm(pool_size, device=device)[:B] + cache_indices = perm.to(torch.int32) + + pool_init = torch.randn(pool_size, H, K, V, dtype=dtype, device=device) * 0.1 + cu_seqlens = torch.arange( + 0, (B + 1) * T_per_seq, T_per_seq, dtype=torch.long, device=device + ) + + q = torch.randn(1, T, H, K, dtype=dtype, device=device) + k = torch.randn(1, T, H, K, dtype=dtype, device=device) + v = torch.randn(1, T, H, V, dtype=dtype, device=device) + + g_raw = torch.randn(1, T, H, dtype=dtype, device=device) + g_triton = torch.nn.functional.logsigmoid(g_raw) + beta_triton = torch.sigmoid(torch.randn(1, T, H, dtype=dtype, device=device)) + + return dict( + B=B, + T=T, + T_per_seq=T_per_seq, + H=H, + K=K, + V=V, + pool_size=pool_size, + cache_indices=cache_indices, + pool_init=pool_init, + cu_seqlens=cu_seqlens, + q=q, + k=k, + v=v, + g_triton=g_triton, + beta_triton=beta_triton, + ) + + +# --------------------------------------------------------------------------- +# Runner wrappers +# --------------------------------------------------------------------------- + + +def run_triton(inp): + """Triton path: K-contiguous pool, pool-indexed, [1,T,H,D] tensors.""" + pool = make_k_contiguous(inp["pool_init"].clone()) + o, _, h = triton_chunk_gated_delta_rule( + q=inp["q"], + k=inp["k"], + v=inp["v"], + g=inp["g_triton"], + beta=inp["beta_triton"], + initial_state=pool, + initial_state_indices=inp["cache_indices"], + cu_seqlens=inp["cu_seqlens"], + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + return o, pool, h + + +def run_cutedsl(inp): + """CuteDSL path: matches CuteDSLGDNKernel.extend() exactly.""" + pool = make_k_contiguous(inp["pool_init"].clone()) + cache_indices = inp["cache_indices"] + cu_seqlens = inp["cu_seqlens"].to(torch.int32) + + q_in = l2norm_fwd(inp["q"][0].contiguous()).unsqueeze(0) + k_in = l2norm_fwd(inp["k"][0].contiguous()).unsqueeze(0) + v_in = inp["v"][0].contiguous().unsqueeze(0) + g_in = inp["g_triton"][0].to(torch.float32).unsqueeze(0) + beta_in = inp["beta_triton"][0].to(torch.float32).unsqueeze(0) + + initial_state = pool[cache_indices.to(torch.long)].contiguous() + chunk_indices, chunk_offsets = prepare_metadata_cutedsl( + cu_seqlens, inp["T"], chunk_size=64 + ) + + o, final_state = chunk_gated_delta_rule_cutedsl( + q=q_in, + k=k_in, + v=v_in, + g=g_in, + beta=beta_in, + initial_state=initial_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + ) + + pool.index_copy_(0, cache_indices.to(torch.long), final_state.to(pool.dtype)) + return o, pool, final_state + + +# --------------------------------------------------------------------------- +# Correctness check +# --------------------------------------------------------------------------- + + +def check_shape( + B, T_per_seq, H, K, V, pool_size, device, dtype, sequential_indices=False, seed=42 +): + tag = ( + f"B={B:>3} T/seq={T_per_seq:>4} H={H:>2} K={K:>3} V={V:>3} pool={pool_size:>4}" + ) + idx_tag = " (seq)" if sequential_indices else "" + + # The ported CuteDSL kernel hard-codes K == V == 128. + if K != 128 or V != 128: + print(f" [SKIP] {tag}{idx_tag} (CuteDSL requires K=V=128)") + return True + + inp = make_inputs( + B, + T_per_seq, + H, + K, + V, + pool_size, + device, + dtype, + sequential_indices=sequential_indices, + seed=seed, + ) + + o_triton, pool_triton, _ = run_triton(inp) + + try: + o_cutedsl, pool_cutedsl, _ = run_cutedsl(inp) + torch.cuda.synchronize() + except Exception as e: + try: + torch.cuda.synchronize() + except Exception: + pass + print(f" [SKIP] {tag}{idx_tag} (CuteDSL error: {e})") + return True + + # Output comparison. Both kernels are bf16 with L2norm + chunked accumulation, + # tolerances mirror bench_gdn_prefill.py. + try: + torch.testing.assert_close(o_triton, o_cutedsl, atol=5e-2, rtol=1e-2) + out_ok = True + except AssertionError as e: + out_ok = False + out_err = str(e).splitlines()[0] + + status = "PASS" if out_ok else "FAIL" + extra = "" if out_ok else f" ({out_err})" + print(f" [{status}] {tag}{idx_tag}{extra}") + return out_ok + + +# --------------------------------------------------------------------------- +# Benchmark +# --------------------------------------------------------------------------- + + +def bench_shape(B, H, T_per_seq, K, V, pool_size, device, dtype): + import triton.testing + + if K != 128 or V != 128: + print(f" [SKIP] B={B} H={H} T={T_per_seq} K={K} V={V} (CuteDSL K=V=128 only)") + return + + T = B * T_per_seq + inp = make_inputs(B, T_per_seq, H, K, V, pool_size, device, dtype) + + q, k_t, v = inp["q"], inp["k"], inp["v"] + g_triton, beta_triton = inp["g_triton"], inp["beta_triton"] + cu_seqlens = inp["cu_seqlens"] + cache_indices = inp["cache_indices"] + pool_v = inp["pool_init"] + T_total = inp["T"] + + def fn_triton(): + pool = make_k_contiguous(pool_v.clone()) + triton_chunk_gated_delta_rule( + q=q, + k=k_t, + v=v, + g=g_triton, + beta=beta_triton, + initial_state=pool, + initial_state_indices=cache_indices, + cu_seqlens=cu_seqlens, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + cu_int32 = cu_seqlens.to(torch.int32) + + def fn_cutedsl(): + q_in = l2norm_fwd(q[0].contiguous()).unsqueeze(0) + k_in = l2norm_fwd(k_t[0].contiguous()).unsqueeze(0) + v_in = v[0].contiguous().unsqueeze(0) + g_in = g_triton[0].to(torch.float32).unsqueeze(0) + beta_in = beta_triton[0].to(torch.float32).unsqueeze(0) + + pool = make_k_contiguous(pool_v.clone()) + initial_state = pool[cache_indices.to(torch.long)].contiguous() + chunk_indices, chunk_offsets = prepare_metadata_cutedsl( + cu_int32, T_total, chunk_size=64 + ) + chunk_gated_delta_rule_cutedsl( + q=q_in, + k=k_in, + v=v_in, + g=g_in, + beta=beta_in, + initial_state=initial_state, + cu_seqlens=cu_int32, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + ) + + quantiles = [0.5, 0.2, 0.8] + + fn_triton() + fn_cutedsl() + torch.cuda.synchronize() + + ms_triton, _, _ = triton.testing.do_bench_cudagraph(fn_triton, quantiles=quantiles) + ms_cutedsl, _, _ = triton.testing.do_bench_cudagraph( + fn_cutedsl, quantiles=quantiles + ) + + flops = gdn_flops(T, H, K, V) + mem_bytes = gdn_bytes(T, H, H, K, V, B, dtype) + + tflops_triton = flops / ms_triton / 1e9 + tflops_cutedsl = flops / ms_cutedsl / 1e9 + tb_s_triton = mem_bytes / ms_triton / 1e9 + tb_s_cutedsl = mem_bytes / ms_cutedsl / 1e9 + speedup = ms_triton / ms_cutedsl if ms_cutedsl > 0 else float("inf") + + print( + f" {B:>5} {H:>3} {T_per_seq:>6} {T:>7} | " + f"{ms_triton:>8.3f} {tflops_triton:>7.2f} {tb_s_triton:>7.2f} | " + f"{ms_cutedsl:>8.3f} {tflops_cutedsl:>7.2f} {tb_s_cutedsl:>7.2f} | " + f"{speedup:>7.2f}x" + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def run_correctness(device, dtype): + print("=" * 78) + print("Correctness sweep: Triton vs CuTeDSL") + print("=" * 78) + + shapes = [ + # (B, T_per_seq, H, K, V, pool_size) + (4, 64, 16, 128, 128, 32), + (4, 256, 16, 128, 128, 32), + (1, 128, 16, 128, 128, 32), + (8, 128, 16, 128, 128, 64), + (16, 64, 16, 128, 128, 128), + (32, 32, 16, 128, 128, 256), + (4, 128, 4, 128, 128, 32), + (4, 128, 8, 128, 128, 32), + (4, 128, 32, 128, 128, 32), + (4, 128, 64, 128, 128, 32), + (4, 1, 16, 128, 128, 32), + (4, 7, 16, 128, 128, 32), + (4, 16, 16, 128, 128, 32), + (4, 128, 16, 128, 128, 512), + (32, 128, 32, 128, 128, 256), + ] + + shapes_seq = [ + (8, 128, 16, 128, 128, 8), + (4, 128, 32, 128, 128, 4), + (4, 128, 64, 128, 128, 4), + (32, 128, 32, 128, 128, 32), + ] + + all_pass = True + for cfg in shapes: + if not check_shape(*cfg, device, dtype): + all_pass = False + + print("\nSequential-index variants:") + for cfg in shapes_seq: + if not check_shape(*cfg, device, dtype, sequential_indices=True): + all_pass = False + + print() + print("ALL PASSED." if all_pass else "SOME FAILED.") + return all_pass + + +def run_benchmark(device, dtype, args): + print() + print("=" * 105) + print("Benchmark: Triton GDN vs CuTeDSL GDN (do_bench_cudagraph)") + print("=" * 105) + + K = args.head_size_k + V = args.head_size_v + pool_size = args.pool_size + + if args.preset == "qwen3-next": + bench_configs = [ + (4, 16, 256), + (4, 32, 256), + (16, 16, 256), + (16, 32, 256), + (32, 16, 256), + (32, 32, 256), + (64, 16, 256), + (64, 32, 256), + (128, 16, 256), + (128, 32, 256), + (4, 16, 1024), + (4, 32, 1024), + (32, 16, 1024), + (32, 32, 1024), + ] + else: + bench_configs = [ + (B, H, T) + for B in args.batch_sizes + for H in args.num_heads + for T in args.seq_lens + ] + + print(f" Config: K={K}, V={V}, pool_size={pool_size}, dtype={dtype}") + print( + f" {'B':>5} {'H':>3} {'T/seq':>6} {'T_tot':>7} | " + f"{'tri(ms)':>8} {'TFLOPS':>7} {'TB/s':>7} | " + f"{'cute(ms)':>8} {'TFLOPS':>7} {'TB/s':>7} | " + f"{'speedup':>8}" + ) + print(" " + "-" * 98) + + for B, H, T_per_seq in bench_configs: + actual_pool = max(pool_size, B) + bench_shape(B, H, T_per_seq, K, V, actual_pool, device, dtype) + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark & Correctness: Triton GDN vs CuTeDSL GDN (SM100)" + ) + parser.add_argument( + "--mode", choices=["all", "correctness", "bench"], default="all" + ) + parser.add_argument( + "--preset", choices=["qwen3-next", "custom"], default="qwen3-next" + ) + parser.add_argument("--dtype", choices=["float16", "bfloat16"], default="bfloat16") + parser.add_argument("--head-size-k", type=int, default=128) + parser.add_argument("--head-size-v", type=int, default=128) + parser.add_argument("--pool-size", type=int, default=256) + parser.add_argument( + "--batch-sizes", type=int, nargs="+", default=[4, 16, 32, 64, 128] + ) + parser.add_argument("--num-heads", type=int, nargs="+", default=[16, 32]) + parser.add_argument( + "--seq-lens", type=int, nargs="+", default=[128, 256, 512, 1024] + ) + args = parser.parse_args() + + if args.preset == "qwen3-next": + args.head_size_k = 128 + args.head_size_v = 128 + + device = "cuda" + dtype = getattr(torch, args.dtype) + + cap = torch.cuda.get_device_capability() + dev_name = torch.cuda.get_device_name() + print(f"Device: {dev_name} (SM {cap[0]}{cap[1]})") + if cap[0] < 10: + print("ERROR: CuTeDSL GDN prefill requires SM100+ (Blackwell). Exiting.") + return 1 + + if args.mode in ("all", "correctness"): + all_pass = run_correctness(device, dtype) + if not all_pass and args.mode == "all": + print("\nSkipping benchmark due to correctness failures.") + return 1 + + if args.mode in ("all", "bench"): + run_benchmark(device, dtype, args) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmark/scheduler/bench_token_storage.py b/benchmark/scheduler/bench_token_storage.py new file mode 100644 index 000000000000..ea8ef7418c4c --- /dev/null +++ b/benchmark/scheduler/bench_token_storage.py @@ -0,0 +1,334 @@ +"""Benchmark `list[int]` vs `array.array('q')` storage for +`Req.origin_input_ids` / `Req.output_ids` over one request lifecycle. + +Simulated steps (per batch): + 1. ingest -- tokenizer list[int] -> storage container. + 2. prefix_match -- scheduler radix-tree lookup; RadixKey.match() + zip+!= walk. Exposes the per-element PyLong-boxing + cost array.array introduces (list[int] iterates + existing PyLongs and pays nothing). + 3. prefill -- (a) fill_ids = origin + output, + (b) per-req slice fill_ids[prefix_len:], + (c) cross-req flatten + pinned cuda tensor build. + 4. decode -- per-step output.append(next_token) for n_decode steps. + 5. finish -- cache_finished_req: + (a) concat (origin + output)[:kv_committed_len] + for the radix-tree insert. + (b) RadixKey.match() zip+!= walk during insert's + tree traversal — second PyLong-boxing hotspot + on the array.array path. + +Usage: + python benchmark/scheduler/bench_token_storage.py +""" + +from __future__ import annotations + +import time +from array import array +from collections import defaultdict +from contextlib import contextmanager +from itertools import chain +from typing import Any, Callable, Iterator + +import numpy as np +import torch + +# Per-req stages accumulate across reqs in a batch; batch_torch_tensor +# is the single cross-req prepare_for_extend tensor build. +STAGES = ( + "ingest", + "prefix_match", + "prefill_concat", + "prefill_perreq_slice", + "batch_torch_tensor", + "decode_append", + "finish_concat", + "cache_finished_req", +) + + +def _ingest_list(seed: list[int]) -> list[int]: + return seed + + +def _ingest_pyarray(seed: list[int]) -> array: + return array("q", seed) + + +def _empty_list() -> list[int]: + return [] + + +def _empty_pyarray() -> array: + return array("q") + + +def _zip_iterate(t0: Any, t1: Any) -> int: + """Simulate zip iteration which surface PyLong boxing cost in array scenario""" + i = 0 + for a, b in zip(t0, t1): + if a != b: + break + i += 1 + return i + + +def _batch_tensor_from_lists(parts: list[list[int]]) -> torch.Tensor: + flat = list(chain.from_iterable(parts)) + return torch.tensor(flat, dtype=torch.int64, pin_memory=True).to( + "cuda", non_blocking=True + ) + + +def _batch_tensor_from_pyarrays(parts: list[array]) -> torch.Tensor: + # np.frombuffer gives a zero-copy view; np.concatenate is one C-level + # memcpy. This bypasses the per-element PyLong->int64 walk that + # torch.tensor(array('q')) would otherwise do. + views = [np.frombuffer(p, dtype=np.int64) for p in parts] + combined = np.concatenate(views) if len(views) > 1 else views[0] + return torch.from_numpy(combined).pin_memory().to("cuda", non_blocking=True) + + +LIST_KIT = { + "ingest_fn": _ingest_list, + "empty_fn": _empty_list, + "batch_torch_fn": _batch_tensor_from_lists, +} + +PYARRAY_KIT = { + "ingest_fn": _ingest_pyarray, + "empty_fn": _empty_pyarray, + "batch_torch_fn": _batch_tensor_from_pyarrays, +} + + +@contextmanager +def timed(timings: dict[str, float], stage: str) -> Iterator[None]: + t0 = time.monotonic_ns() + try: + yield + finally: + timings[stage] += time.monotonic_ns() - t0 + + +def simulate( + seeds: list[list[int]], + n_decode: int, + *, + ingest_fn: Callable[[list[int]], Any], + empty_fn: Callable[[], Any], + batch_torch_fn: Callable[[list[Any]], torch.Tensor], +) -> dict[str, float]: + """One scheduling-round lifecycle. Returns per-stage cumulative ns.""" + timings: dict[str, float] = defaultdict(float) + n_reqs = len(seeds) + n_origins = [len(s) for s in seeds] + origins: list[Any] = [None] * n_reqs + outputs: list[Any] = [None] * n_reqs + + # 1. ingest + for i, seed in enumerate(seeds): + with timed(timings, "ingest"): + origins[i] = ingest_fn(seed) + outputs[i] = empty_fn() + + # 2. prefix_match: simulating the worse scenario of PyLong-boxing overhead during prefix_match + for i in range(n_reqs): + with timed(timings, "prefix_match"): + _ = _zip_iterate(origins[i], origins[i]) + + # 3. prefill + per_req_slices: list[Any] = [None] * n_reqs + for i in range(n_reqs): + # 3a. fill_ids = origin_input_ids + output_ids + with timed(timings, "prefill_concat"): + fill_ids = origins[i] + outputs[i] + # 3b. input_ids = fill_ids[len(prefix_indices):]; prefix_len=0 here. + with timed(timings, "prefill_perreq_slice"): + per_req_slices[i] = fill_ids[0:] + # 3c. prepare_for_extend tensor build: flatten per-req slices, then + # build the pinned GPU tensor (kit-specific path). + with timed(timings, "batch_torch_tensor"): + _ = batch_torch_fn(per_req_slices) + + # 4. decode + for i in range(n_reqs): + with timed(timings, "decode_append"): + for j in range(n_decode): + outputs[i].append(j) + + # 5. finish: cache_finished_req -> insert -> _insert_helper tree walk. + for i in range(n_reqs): + # 5a. (origin + output)[:kv_committed_len] for the radix-tree insert. + with timed(timings, "finish_concat"): + committed = (origins[i] + outputs[i])[: n_origins[i] + n_decode] + # 5b. simulating the worse scenario of PyLong-boxing overhead during cache_finished_req + with timed(timings, "cache_finished_req"): + _ = _zip_iterate(committed, committed) + + return timings + + +def bench_lifecycle( + seeds: list[list[int]], + n_decode: int, + iterations: int, + *, + ingest_fn: Callable[[list[int]], Any], + empty_fn: Callable[[], Any], + batch_torch_fn: Callable[[list[Any]], torch.Tensor], + warmup: int = 5, +) -> dict[str, float]: + """Run simulate() N times, return mean per-stage us per batch. + + GPU sync is excluded from per-iteration timing: production issues + `to(device, non_blocking=True)` and continues, so we measure issue + cost rather than H2D completion. + """ + kit = { + "ingest_fn": ingest_fn, + "empty_fn": empty_fn, + "batch_torch_fn": batch_torch_fn, + } + torch.cuda.synchronize() + for _ in range(warmup): + simulate(seeds, n_decode, **kit) + torch.cuda.synchronize() + accum: dict[str, float] = defaultdict(float) + for _ in range(iterations): + t = simulate(seeds, n_decode, **kit) + for k, v in t.items(): + accum[k] += v + torch.cuda.synchronize() + return {k: accum[k] / iterations / 1000.0 for k in STAGES} # ns -> us + + +def print_breakdown(title: str, results: dict[str, dict[str, float]]) -> None: + """Print per-stage timings with delta us vs the first (baseline) column.""" + labels = list(results.keys()) + baseline_label = labels[0] + baseline = results[baseline_label] + + width = max(len(s) for s in STAGES) + + header_cells = [f"{baseline_label + ' us':>11s}"] + for lbl in labels[1:]: + header_cells.append(f"{lbl + ' us':>11s}") + header_cells.append(f"{'delta':>10s}") + + print(f"=== {title} ===") + print(f"{'Stage':<{width}s} " + " ".join(header_cells)) + print("-" * (width + 2 + sum(len(c) + 2 for c in header_cells))) + + for s in STAGES: + cells = [f"{baseline[s]:>11.3f}"] + for lbl in labels[1:]: + v = results[lbl][s] + cells.append(f"{v:>11.3f}") + d = v - baseline[s] + cells.append(f"{d:>+10.3f}") + print(f"{s:<{width}s} " + " ".join(cells)) + + print("-" * (width + 2 + sum(len(c) + 2 for c in header_cells))) + + base_total = sum(baseline.values()) + total_cells = [f"{base_total:>11.3f}"] + for lbl in labels[1:]: + v = sum(results[lbl].values()) + total_cells.append(f"{v:>11.3f}") + d = v - base_total + total_cells.append(f"{d:>+10.3f}") + print(f"{'TOTAL':<{width}s} " + " ".join(total_cells)) + + print() + for lbl in labels[1:]: + v = sum(results[lbl].values()) + d = v - base_total + speedup = base_total / v if v > 0 else 0.0 + verdict = "LOSES" if d > 0 else "WINS" + print( + f" {lbl:<14s} vs {baseline_label}: {verdict} by {abs(d):>8.2f} us ({speedup:.2f}x)" + ) + print() + + +def microbench_torch_tensor_paths( + sizes: tuple[int, ...] = (1_000, 10_000, 100_000) +) -> None: + """Compare three CPU-buffer -> pinned cuda tensor paths. + + A. torch.tensor(list, pin) -> cuda + B. torch.tensor(array('q'), pin) -> cuda + C. torch.from_numpy(np.frombuffer(array('q'))).pin() -> cuda + """ + + def t(fn, iterations: int) -> float: + for _ in range(20): + fn() + torch.cuda.synchronize() + t0 = time.monotonic_ns() + for _ in range(iterations): + fn() + torch.cuda.synchronize() + return (time.monotonic_ns() - t0) / iterations / 1000.0 + + print("=== microbench: CPU-buffer -> pinned cuda tensor (us/op) ===\n") + width = 56 + print(f"{'Path':<{width}s} " + " ".join(f"{f'N={n}':>10s}" for n in sizes)) + print("-" * (width + 2 + 12 * len(sizes))) + + for label, build in [ + ( + "(A) torch.tensor(list, pin) -> cuda", + lambda x: torch.tensor(x, dtype=torch.int64, pin_memory=True).to( + "cuda", non_blocking=True + ), + ), + ( + "(B) torch.tensor(array('q'), pin) -> cuda (naive)", + lambda x: torch.tensor(x, dtype=torch.int64, pin_memory=True).to( + "cuda", non_blocking=True + ), + ), + ( + "(C) from_numpy(frombuf(array('q'))).pin() -> cuda", + lambda x: torch.from_numpy(np.frombuffer(x, dtype=np.int64)) + .pin_memory() + .to("cuda", non_blocking=True), + ), + ]: + cells = [] + for n in sizes: + iters = max(50, 200_000 // max(n, 1)) + if "(A)" in label: + src = list(range(n)) + else: + src = array("q", range(n)) + us = t(lambda src=src, build=build: build(src), iters) + cells.append(f"{us:>10.2f}") + print(f"{label:<{width}s} " + " ".join(cells)) + print() + + +def main() -> None: + microbench_torch_tensor_paths() + + n_reqs = 2 + cases = [ + ("short prompt N_origin=1K N_decode=1K", 1_000, 1_000, 1_000), + ("medium prompt N_origin=10K N_decode=1K", 10_000, 1_000, 200), + ("long prompt N_origin=100K N_decode=1K", 100_000, 1_000, 30), + ] + print(f"Batch size = {n_reqs} reqs/batch (per-req stages accumulate)\n") + for label, n_origin, n_decode, iters in cases: + seeds = [list(range(n_origin)) for _ in range(n_reqs)] + results = { + "list": bench_lifecycle(seeds, n_decode, iters, **LIST_KIT), + "pyarray": bench_lifecycle(seeds, n_decode, iters, **PYARRAY_KIT), + } + print_breakdown(label, results) + + +if __name__ == "__main__": + main() diff --git a/docker/Dockerfile b/docker/Dockerfile index ad03168e6cef..36404a7926c3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ ARG UBUNTU_MIRROR ARG GITHUB_ARTIFACTORY=github.com ARG INSTALL_FLASHINFER_JIT_CACHE=0 ARG FLASHINFER_VERSION=0.6.11.post1 -ARG MOONCAKE_VERSION=0.3.10.post2 +ARG MOONCAKE_VERSION=0.3.11.post1 #if need other arg please add in MOONCAKE_COMPILE_ARG ARG MOONCAKE_COMPILE_ARG="-DUSE_HTTP=ON -DUSE_MNNVL=ON -DUSE_CUDA=ON -DWITH_EP=ON" diff --git a/docker/npu.Dockerfile b/docker/npu.Dockerfile index bf135b293e2f..44a8cbaf3fc2 100644 --- a/docker/npu.Dockerfile +++ b/docker/npu.Dockerfile @@ -13,6 +13,7 @@ ARG PIP_INDEX_URL="https://pypi.org/simple/" ARG APTMIRROR="" ARG PYTORCH_VERSION="2.8.0" ARG TORCHVISION_VERSION="0.23.0" +ARG TORCHAUDIO_VERSION="2.8.0" ARG PTA_URL_ARM64="https://gitcode.com/Ascend/pytorch/releases/download/v7.3.0-pytorch2.8.0/torch_npu-2.8.0.post2-cp311-cp311-manylinux_2_28_aarch64.whl" ARG PTA_URL_AMD64="https://gitcode.com/Ascend/pytorch/releases/download/v7.3.0-pytorch2.8.0/torch_npu-2.8.0.post2-cp311-cp311-manylinux_2_28_x86_64.whl" ARG SGLANG_TAG=main @@ -78,7 +79,7 @@ RUN ${PIP_INSTALL} sglang-router ### Install PyTorch and PTA RUN . /etc/environment_new && \ - (${PIP_INSTALL} torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION} --index-url https://download.pytorch.org/whl/cpu) \ + (${PIP_INSTALL} torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION} torchaudio==${TORCHAUDIO_VERSION} --index-url https://download.pytorch.org/whl/cpu) \ && (${PIP_INSTALL} ${PTA_URL}) diff --git a/docker/sgl-router.Dockerfile b/docker/sgl-router.Dockerfile new file mode 100644 index 000000000000..40de83aee68a --- /dev/null +++ b/docker/sgl-router.Dockerfile @@ -0,0 +1,83 @@ +# Multi-stage build for sgl-router. +# +# Three stages, each scoped to its caching contract: +# 1. chef — generate a `recipe.json` describing the dep graph. +# 2. builder — compile deps from the recipe, then the workspace. +# 3. runtime — distroless cc-debian12 with the stripped binary. +# +# The `cargo-chef` indirection is the canonical Rust multi-stage cache +# pattern: the recipe step's inputs are JUST `Cargo.toml` + `Cargo.lock`, +# so a source-only change produces a recipe-layer cache hit and the +# heavy `cook --release` step is reused untouched. A naive "copy +# manifests → cargo fetch → copy src" approach caches only the fetched +# registry; every source change still recompiles every dep. +# +# `Cargo.lock` is gitignored repo-wide (root .gitignore "# Rust lib" +# block), so we generate it inside the chef stage with `cargo +# generate-lockfile` and propagate that lockfile to the builder via +# `COPY --from=chef`. Both stages thus build against the same lockfile, +# preserving --locked semantics within a single Docker build. +# +# Build (from the repo root): +# docker build -f docker/sgl-router.Dockerfile -t sgl-router:dev . +# Run: +# docker run --rm -p 8090:8090 \ +# -v $(pwd)/docker/sgl-router.sample.yaml:/etc/sgl-router/sgl-router.yaml \ +# sgl-router:dev --config /etc/sgl-router/sgl-router.yaml +# +# Image budget: < 100 MB stripped (M6 acceptance). Verify with +# `docker image inspect sgl-router:dev --format '{{.Size}}'`. + +ARG RUST_VERSION=1.90 +ARG DEBIAN_VERSION=bookworm + +######################## STAGE 1 — chef recipe ########################## +FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS chef +RUN cargo install cargo-chef --locked --version ^0.1 +WORKDIR /work +COPY experimental/sgl-router/Cargo.toml ./ +COPY experimental/sgl-router/rust-toolchain.toml ./ +# Stub a minimal src tree so cargo can resolve the workspace, generate +# the lockfile (gitignored upstream), then prepare the chef recipe. +RUN mkdir -p src && echo "fn main() {}" > src/main.rs \ + && echo "" > src/lib.rs \ + && cargo generate-lockfile \ + && cargo chef prepare --recipe-path recipe.json \ + && rm -rf src + +######################## STAGE 2 — builder ############################## +FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS builder +RUN cargo install cargo-chef --locked --version ^0.1 +WORKDIR /work +COPY --from=chef /work/recipe.json ./recipe.json +COPY --from=chef /work/Cargo.lock ./Cargo.lock +COPY experimental/sgl-router/rust-toolchain.toml ./ + +# Cook (compile + cache) the dep graph from the recipe. This layer's +# inputs are recipe.json + the toolchain — code changes in src/ do NOT +# invalidate it. +RUN cargo chef cook --release --recipe-path recipe.json + +# Now bring in the real sources and the manifest they need. +COPY experimental/sgl-router/Cargo.toml ./ +COPY experimental/sgl-router/src ./src + +# --locked is intentionally omitted: the lockfile is generated in-container +# (gitignored upstream) and `cargo chef cook` may have mutated it during the +# dep-cook step, so a strict --locked check would spuriously fail. +RUN cargo build --release --bin sgl-router \ + && strip target/release/sgl-router + +######################## STAGE 3 — runtime ############################## +FROM gcr.io/distroless/cc-debian12:nonroot AS runtime + +COPY --from=builder /work/target/release/sgl-router /usr/local/bin/sgl-router + +# Default config path; mount your own via `-v :/etc/sgl-router`. +ENV SGL_ROUTER_CONFIG=/etc/sgl-router/sgl-router.yaml +EXPOSE 8090 + +# distroless `nonroot` runs as uid 65532. The router doesn't need root. +USER nonroot:nonroot + +ENTRYPOINT ["/usr/local/bin/sgl-router"] diff --git a/docker/xeon.Dockerfile b/docker/xeon.Dockerfile index 98e443a1f023..c29e2ad45443 100644 --- a/docker/xeon.Dockerfile +++ b/docker/xeon.Dockerfile @@ -42,7 +42,8 @@ RUN source $HOME/.local/bin/env && \ uv pip install . && \ cd ../sgl-kernel && \ cp pyproject_cpu.toml pyproject.toml && \ - uv pip install . + uv pip install . && \ + uv pip install pytest ENV SGLANG_USE_CPU_ENGINE=1 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4:/usr/lib/x86_64-linux-gnu/libtbbmalloc.so:/opt/.venv/lib/libiomp5.so diff --git a/docker/xpu.Dockerfile b/docker/xpu.Dockerfile index 1a3de356b011..832df26f3d2d 100644 --- a/docker/xpu.Dockerfile +++ b/docker/xpu.Dockerfile @@ -68,6 +68,7 @@ RUN --mount=type=secret,id=github_token \ cp pyproject_xpu.toml pyproject.toml && \ pip install . --extra-index-url https://download.pytorch.org/whl/xpu && \ pip install --no-deps xgrammar==0.1.33 && \ + pip install apache-tvm-ffi && \ pip install triton-xpu==3.7.1 --index-url https://download.pytorch.org/whl/xpu --force-reinstall && \ # Add environment setup commands to .bashrc again (in case it was overwritten) echo ". /home/sdp/miniforge3/bin/activate; conda activate py${PYTHON_VERSION}; cd /home/sdp" >> /home/sdp/.bashrc diff --git a/docs/advanced_features/hisparse_guide.md b/docs/advanced_features/hisparse_guide.md index 941318c869bc..675bbb05e722 100644 --- a/docs/advanced_features/hisparse_guide.md +++ b/docs/advanced_features/hisparse_guide.md @@ -2,7 +2,7 @@ HiSparse reduces per-request GPU memory consumption during the decode phase by maintaining only a small "hot" KV buffer on GPU while keeping complete KV data in CPU pinned memory. Combined with PD disaggregation, it enables significantly higher decode concurrency. -> **Prerequisites**: HiSparse only works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5). These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. +> **Prerequisites**: HiSparse works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5.1) and **DeepSeek V4**. These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. ## Why HiSparse? @@ -40,6 +40,8 @@ Prefill GPU ──RDMA──▶ Decode Host Pool (CPU pinned memory) swap-in kernel (on-demand top-k) ``` +For DeepSeek V4, the direct-to-host path writes only C4 KV into the decode host pool. The c4_indexer and C128 KV remain device-to-device transfers. + ## Server Arguments | Argument | Type / Default | Description | @@ -89,8 +91,7 @@ python3 -m sglang.launch_server \ --context-length 81920 \ --tp-size 8 --dp-size 8 --enable-dp-attention \ --mem-fraction-static 0.85 \ - --kv-cache-dtype bfloat16 \ - --dsa-decode-backend flashmla_sparse \ + --disable-radix-cache \ --disaggregation-mode decode \ --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ --dist-init-addr 127.0.0.1:5757 \ @@ -99,6 +100,8 @@ python3 -m sglang.launch_server \ --hisparse-config='{"top_k": 2048, "device_buffer_size": 6144, "host_to_device_ratio": 10}' ``` +> **Note**: For DSA models, `--kv-cache-dtype` defaults to `auto`, which resolves to `fp8_e4m3` on SM100+ (Blackwell) and `bfloat16` on older architectures. The DSA decode backend is automatically selected based on KV dtype (`bfloat16` → `flashmla_sparse`, `fp8_e4m3` → `flashmla_kv`). DSA backend flags apply only to DSA models; DeepSeek V4 uses its own `dsv4` attention backend. + ### Benchmark ```bash @@ -121,14 +124,12 @@ python3 -m sglang.bench_serving \ ### Key Notes - The prefill instance does not need `--enable-hisparse`; it is unaware of HiSparse. -- On the decode instance, the following flags are **required** for HiSparse: - - `--kv-cache-dtype bfloat16` — currently only bfloat16 KV cache is supported (more dtypes planned). - - `--dsa-decode-backend flashmla_sparse` — currently only `flashmla_sparse` backend is supported. - - `--enable-hisparse` — enables HiSparse. - - `--hisparse-config` — HiSparse configuration (top_k, device_buffer_size, host_to_device_ratio). - - `host_to_device_ratio` should be configured based on the host machine's available memory. For example: - - **~1 TB** host memory → `host_to_device_ratio: 5` - - **~2 TB** host memory → `host_to_device_ratio: 10` +- On the decode instance, `--enable-hisparse` and `--hisparse-config` are required for HiSparse. +- For DSA models, `--kv-cache-dtype bfloat16` uses `flashmla_sparse`, and `--kv-cache-dtype fp8_e4m3` uses `flashmla_kv`. +- For DeepSeek V4, DSA backend flags are not applicable. DeepSeek V4 uses the `dsv4` attention backend and `fp8_e4m3` KV cache by default. +- `host_to_device_ratio` should be configured based on the host machine's available memory. For example: + - **~1 TB** host memory → `host_to_device_ratio: 5` + - **~2 TB** host memory → `host_to_device_ratio: 10` ## Acknowledgments diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md index a880f518c13b..8a88b0ff2ea6 100644 --- a/docs/advanced_features/server_arguments.md +++ b/docs/advanced_features/server_arguments.md @@ -273,6 +273,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s | `--mm-attention-backend` | Set multimodal attention backend. | `None` | `sdpa`, `fa3`, `fa4`, `triton_attn`, `ascend_attn`, `aiter_attn` | | `--dsa-prefill-backend` | Choose the DSA backend for the prefill stage (overrides `--attention-backend` when running DeepSeek DSA-style attention). `--nsa-prefill-backend` is a deprecated alias. | `flashmla_sparse` | `flashmla_sparse`, `flashmla_kv`, `flashmla_auto`, `fa3`, `tilelang`, `aiter`, `trtllm` | | `--dsa-decode-backend` | Choose the DSA backend for the decode stage when running DeepSeek DSA-style attention. Overrides `--attention-backend` for decoding. `--nsa-decode-backend` is a deprecated alias. | `fa3` | `flashmla_sparse`, `flashmla_kv`, `fa3`, `tilelang`, `aiter`, `trtllm` | +| `--dsa-topk-backend` | Choose the DSA indexer top-k backend. The `torch` backend currently requires `SGLANG_DSA_FUSE_TOPK=false`. | `sgl-kernel` | `sgl-kernel`, `torch`, `flashinfer` | | `--fp8-gemm-backend` | Choose the runner backend for Blockwise FP8 GEMM operations. Options: 'auto' (default, auto-selects based on hardware), 'deep_gemm' (JIT-compiled; enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) when DeepGEMM is installed), 'flashinfer_trtllm' (FlashInfer TRTLLM backend; SM100/SM103 only), 'flashinfer_cutlass' (FlashInfer CUTLASS backend, SM120 only), 'flashinfer_deepgemm' (Hopper SM90 only, uses swapAB optimization for small M dimensions in decoding), 'cutlass' (optimal for Hopper/Blackwell GPUs and high-throughput), 'triton' (fallback, widely compatible), 'aiter' (ROCm only).| `auto` | `auto`, `deep_gemm`, `flashinfer_trtllm`, `flashinfer_cutlass`, `flashinfer_deepgemm`, `cutlass`, `triton`, `aiter` | | `--fp4-gemm-backend` | Choose the runner backend for NVFP4 GEMM operations. Options: 'flashinfer_cutlass' (default), 'auto' (auto-selects between flashinfer_cudnn/flashinfer_cutlass based on CUDA/cuDNN version), 'flashinfer_cudnn' (FlashInfer cuDNN backend, optimal on CUDA 13+ with cuDNN 9.15+), 'flashinfer_trtllm' (FlashInfer TensorRT-LLM backend, requires different weight preparation with shuffling). All backends are from FlashInfer; when FlashInfer is unavailable, sgl-kernel CUTLASS is used as an automatic fallback.| `flashinfer_cutlass` | `auto`, `flashinfer_cudnn`, `flashinfer_cutlass`, `flashinfer_trtllm` | | `--disable-flashinfer-autotune` | Flashinfer autotune is enabled by default. Set this flag to disable the autotune. | `False` | bool flag (set to enable) | @@ -346,7 +347,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s | `--max-mamba-cache-size` | The maximum size of the mamba cache. | `None` | Type: int | | `--mamba-ssm-dtype` | The data type of the SSM states in mamba cache. | `float32` | `float32`, `bfloat16`, `float16` | | `--mamba-full-memory-ratio` | The ratio of mamba state memory to full kv cache memory. | `0.9` | Type: float | -| `--mamba-scheduler-strategy` | The strategy to use for mamba scheduler. `auto` currently defaults to `no_buffer`. 1. `no_buffer` does not support overlap scheduler due to not allocating extra mamba state buffers. Branching point caching support is feasible but not implemented. 2. `extra_buffer` supports overlap schedule by allocating extra mamba state buffers to track mamba state for caching (mamba state usage per running req becomes `2x` for non-spec; `1+(1/(2+speculative_num_draft_tokens))x` for spec dec (e.g. 1.16x if speculative_num_draft_tokens==4)). 2a. `extra_buffer` is strictly better for non-KV-cache-bound cases; for KV-cache-bound cases, the tradeoff depends on whether enabling overlap outweighs reduced max running requests. 2b. mamba caching at radix cache branching point is strictly better than non-branch but requires kernel support (currently only FLA backend), currently only extra_buffer supports branching. | `auto` | `auto`, `no_buffer`, `extra_buffer` | +| `--mamba-scheduler-strategy` | The strategy to use for mamba scheduler. `auto` currently defaults to `no_buffer`. 1. `no_buffer` does not support overlap scheduler due to not allocating extra mamba state buffers. Branching point caching support is feasible but not implemented. 2. `extra_buffer` supports overlap schedule by allocating extra mamba state buffers to track mamba state for caching (mamba state usage per running req becomes `2x` for non-spec; `1+(1/(2+speculative_num_draft_tokens))x` for spec dec (e.g. 1.16x if speculative_num_draft_tokens==4)). 2a. `extra_buffer` is strictly better for non-KV-cache-bound cases; for KV-cache-bound cases, the tradeoff depends on whether enabling overlap outweighs reduced max running requests. 2b. mamba caching at radix cache branching point is strictly better than non-branch but requires kernel support, currently only extra_buffer supports branching. | `auto` | `auto`, `no_buffer`, `extra_buffer` | | `--mamba-track-interval` | The interval (in tokens) to track the mamba state during decode. Only used when `--mamba-scheduler-strategy` is `extra_buffer`. Must be divisible by page_size if set, and must be >= speculative_num_draft_tokens when using speculative decoding. | `256` | Type: int | ## Hierarchical cache diff --git a/docs/diffusion/environment_variables.md b/docs/diffusion/environment_variables.md index 745c84af27f6..a9ba2d2500f8 100644 --- a/docs/diffusion/environment_variables.md +++ b/docs/diffusion/environment_variables.md @@ -35,7 +35,7 @@ | Environment Variable | Default | Description | |----------------------|---------|-------------| -| `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND` | not set | FlashInfer FP4 GEMM backend for generic NVFP4 fallback | +| `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND` | not set | Optional FlashInfer FP4 GEMM backend override for diffusion NVFP4. When unset, SGLang defaults to `flashinfer_trtllm`. | ## Caching Acceleration diff --git a/docs/diffusion/quantization.md b/docs/diffusion/quantization.md index 4f2e988cfc4c..dfcd3b3f347a 100644 --- a/docs/diffusion/quantization.md +++ b/docs/diffusion/quantization.md @@ -125,7 +125,7 @@ official `black-forest-labs/FLUX.2-dev-NVFP4` repo. | `FP8` | `Qwen/Qwen-Image-Edit-2511` | `--transformer-path` | `lmsys/qwen-image-edit-modelopt-fp8-sglang-transformer` | TI2I edit path, BF16-vs-FP8 image comparison, H100 benchmark | shares `QwenImageTransformer2DModel` with Qwen Image and uses the same Qwen Image FP8 fallback preset | | `NVFP4` | `black-forest-labs/FLUX.1-dev` | `--transformer-path` | `lmsys/flux1-dev-modelopt-nvfp4-sglang-transformer` | mixed BF16+NVFP4 transformer override, correctness validation, 4x RTX 5090 benchmark, torch-profiler trace | use `build_modelopt_nvfp4_transformer.py`; validated builder keeps selected FLUX.1 modules in BF16 and sets `swap_weight_nibbles=false` | | `NVFP4` | `black-forest-labs/FLUX.2-dev` | `--transformer-weights-path` | `black-forest-labs/FLUX.2-dev-NVFP4` | packed-QKV load path | official raw export repo; validated packed export detection and runtime layout handling | -| `NVFP4` | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `--transformer-path` | `lmsys/wan22-t2v-a14b-modelopt-nvfp4-sglang-transformer` | primary `transformer` quantized with ModelOpt NVFP4, `transformer_2` kept BF16 | primary-transformer-only path; keep `transformer_2` on the base checkpoint, and current B200/Blackwell bring-up uses `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=cudnn` | +| `NVFP4` | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `--transformer-path` | `lmsys/wan22-t2v-a14b-modelopt-nvfp4-sglang-transformer` | primary `transformer` quantized with ModelOpt NVFP4, `transformer_2` kept BF16 | primary-transformer-only path; keep `transformer_2` on the base checkpoint; the default FP4 GEMM backend is `flashinfer_trtllm` | These nine checkpoints are also the intended case set for the B200 diffusion CI job (`multimodal-gen-test-1-b200`). @@ -261,7 +261,6 @@ For a dual-transformer Wan2.2 export where only the primary `transformer` was quantized: ```bash -SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=cudnn \ sglang generate \ --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ --transformer-path lmsys/wan22-t2v-a14b-modelopt-nvfp4-sglang-transformer \ @@ -279,20 +278,15 @@ sglang generate \ primary `--transformer-path` override targets only `transformer`. Use a per-component override such as `--transformer-2-path` only when you intentionally want a non-default `transformer_2`. -- On Blackwell, the validated Wan2.2 ModelOpt NVFP4 path currently prefers - FlashInfer FP4 GEMM via - `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=cudnn`. -- This environment-variable override is a current workaround for NVFP4 cases - where the default sglang JIT/CUTLASS `sm100` path rejects a large-M shape at - `can_implement()`. The intended long-term fix is to add a validated CUTLASS - fallback for those shapes rather than rely on the override. +- On Blackwell, the diffusion ModelOpt NVFP4 path defaults to FlashInfer + TensorRT-LLM FP4 GEMM (`flashinfer_trtllm`). - Direct `--model-path` loading is a compatibility path for FLUX.2 NVFP4-style repos or local directories. - If `--transformer-weights-path` is provided explicitly, it takes precedence over the compatibility `--model-path` flow. - For local directories, SGLang first looks for `*-mixed.safetensors`, then falls back to loading from the directory. -- To force the generic diffusion ModelOpt FP4 path onto a specific FlashInfer +- To force the diffusion ModelOpt FP4 path onto a different FlashInfer backend, set `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND`. Supported values include `flashinfer_cudnn`, `flashinfer_cutlass`, and `flashinfer_trtllm`. - On disk, the quantization config stays `quant_method=modelopt` with diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md index 87e085880231..63cb9c837d5a 100644 --- a/docs/references/environment_variables.md +++ b/docs/references/environment_variables.md @@ -95,6 +95,8 @@ SGLang supports various environment variables that can be used to configure its | Environment Variable | Description | Default Value | | --- | --- | --- | | `SGLANG_DSA_FUSE_TOPK` | Fuse the operation of picking topk logits and picking topk indices from page table (`SGLANG_NSA_FUSE_TOPK` is a deprecated alias) | `true` | +| `SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTIC` | Use deterministic FlashInfer topk kernels when `--dsa-topk-backend=flashinfer` | `false` | +| `SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK` | Tie-break mode for FlashInfer DSA topk when `--dsa-topk-backend=flashinfer`: unset disables explicit tie-breaking, `small` prefers the smaller candidate index for equal scores, and `large` prefers the larger candidate index for equal scores. Setting this variable makes FlashInfer use deterministic topk. | `unset` | | `SGLANG_DSA_ENABLE_MTP_PRECOMPUTE_METADATA` | Precompute metadata that can be shared among different draft steps when MTP is enabled (`SGLANG_NSA_ENABLE_MTP_PRECOMPUTE_METADATA` is a deprecated alias) | `true` | | `SGLANG_USE_FUSED_METADATA_COPY` | Control whether to use fused metadata copy kernel for cuda graph replay | `true` | | `SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` | When the maximum kv len in current prefill batch exceeds this value, the sparse mla kernel will be applied, else it falls back to dense MHA implementation. Default to the index topk of model (2048 for DeepSeek V3.2) (`SGLANG_NSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD` is a deprecated alias) | `2048` | diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 392282aaa9f0..f779ed26a8ef 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -35,7 +35,7 @@ tag: NEW DeepSeek-V4-Pro 1.6T 49B - high-capacity: B200 8 GPU / GB200 8 GPU (2 nodes) / GB300 4 GPU / H200 8 GPU(fp4)/16 GPU(fp8) + high-capacity: B200 8 GPU / GB200 8 GPU (2 nodes) / GB300 4 GPU / H200 8 GPU (FP4) or 16 GPU (SGLang FP8) @@ -120,7 +120,7 @@ The generator currently picks values on the **conservative** side (mirroring an **Hopper (H200) note** We provide two different options for running DeepSeek-V4 models on Hopper devices (H200) -- Original FP4 checkpoints: To run original FP4 checkpoints, we provide two different options for w4a16 MoE kernels: Marlin (`--moe-runner-backend marlin`) and Flashinfer (`--moe-runner-backend flashinfer_mxfp4). For this variant we only support Tensor Parallelism. Complete Pro model can be run on a single H200 node with this option. +- Original FP4 checkpoints: To run original FP4 checkpoints, we provide two different options for w4a16 MoE kernels: Marlin (`--moe-runner-backend marlin`) and Flashinfer (`--moe-runner-backend flashinfer_mxfp4`). For this variant we only support Tensor Parallelism. Complete Pro model can be run on a single H200 node with this option. - Converted FP8 checkpoints: We also provide pre-converted FP8 checkpoints (`sgl-project/DeepSeek-V4-Flash-FP8`, `sgl-project/DeepSeek-V4-Pro-FP8`), which support more parallelism and features. PD-Disagg recipes on H200 may require `docker run --privileged --ulimit memlock=-1` @@ -182,7 +182,7 @@ curl http://localhost:30000/v1/chat/completions \ Enable the `deepseek-v4` reasoning parser (check the box in the [command panel above](#3-model-deployment)) to separate thinking from the final answer into `reasoning_content` vs `content`. -**Streaming with Thinking Process:** + ```python Example from openai import OpenAI @@ -227,17 +227,36 @@ for chunk in response: print() ``` -**Output Example:** + + + ```text Output -Pending update — replace with real server output after deployment. +We are asked: "What is 15% of 240?" This is a simple percentage problem. I need to provide a step-by-step solution. The user wants the solution explained step by step. I'll calculate 15% of 240: 0.15 * 240 = 36. I'll break it down into steps: understand what percent means, convert percentage to decimal or fraction, then multiply. I'll present the answer clearly.To find 15% of 240, follow these steps: + +**Step 1: Understand the meaning of percent** +"Percent" means "per hundred," so 15% means 15 out of every100, or \( \frac{15}{100} \). + +**Step2: Convert the percentage to a decimal or fraction** +\( 15\% = \frac{15}{100} = 0.15 \) + +**Step3: Multiply by the given number** +Multiply the decimal form by 240: +\( 0.15 \times 240 \) + +**Step4: Perform the multiplication** +\( 0.15 \times 240 = 36 \) + +**Answer:** 15% of 240 is **36**. ``` + + #### 4.2.2 Tool Calling Enable the `deepseekv4` tool-call parser (check the box in the [command panel above](#3-model-deployment)) to surface structured tool calls via `message.tool_calls`. -**Python Example (with Thinking Process):** + ```python Example from openai import OpenAI @@ -313,12 +332,22 @@ for index, tool_call in sorted(tool_calls_accumulator.items()): print() ``` -**Output Example:** + + + ```text Output -Pending update — replace with real server output after deployment. +The user wants to know the weather in Beijing. I'll use the get_weather function with Beijing as the location. I don't need to specify a unit, so I'll just use the default. + +<|DSML|tool_calls> +<|DSML|invoke name="get_weather"> +<|DSML|parameter name="location" string="true">Beijing + + ``` + + #### 4.2.3 HiCache (Hierarchical KV Caching) HiCache enables multi-tier KV cache offloading (GPU → CPU → Storage), significantly expanding effective context capacity for long-context and multi-turn scenarios. Combined with UnifiedRadixTree, it provides intelligent prefix caching across all tiers. @@ -332,20 +361,86 @@ For more details, see the [HiCache documentation](../../../docs/advanced_feature ## 5. Benchmark -### 5.1 Speed Benchmark on Blackwell +### 5.1 Accuracy Benchmark + +#### 5.1.1 GSM8K Benchmark + +- **Benchmark Command:** + +```shell Command +python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000 +``` + +- **Test Results:** + - DeepSeek-V4-Pro (FP4, B300, low-latency) + ``` + Accuracy: 0.965 + Invalid: 0.000 + ``` + - DeepSeek-V4-Pro (FP4, H200, low-latency) + ``` + Accuracy: 0.975 + Invalid: 0.000 + ``` + +#### 5.1.2 MMLU Benchmark + +- **Benchmark Command:** + +```shell Command +cd sglang +bash benchmark/mmlu/download_data.sh +python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 30000 +``` + +- **Test Results:** + - DeepSeek-V4-Pro (FP4, B300, low-latency) + ``` + subject: abstract_algebra, #q:100, acc: 0.820 + subject: anatomy, #q:135, acc: 0.881 + subject: astronomy, #q:152, acc: 0.934 + subject: business_ethics, #q:100, acc: 0.840 + subject: clinical_knowledge, #q:265, acc: 0.913 + subject: college_biology, #q:144, acc: 0.972 + subject: college_chemistry, #q:100, acc: 0.680 + subject: college_computer_science, #q:100, acc: 0.890 + subject: college_mathematics, #q:100, acc: 0.870 + subject: college_medicine, #q:173, acc: 0.873 + Total latency: 14.903 + Average accuracy: 0.879 + ``` + - DeepSeek-V4-Pro (FP4, H200, low-latency) + ``` + subject: abstract_algebra, #q:100, acc: 0.850 + subject: anatomy, #q:135, acc: 0.889 + subject: astronomy, #q:152, acc: 0.947 + subject: business_ethics, #q:100, acc: 0.860 + subject: clinical_knowledge, #q:265, acc: 0.932 + subject: college_biology, #q:144, acc: 0.972 + subject: college_chemistry, #q:100, acc: 0.710 + subject: college_computer_science, #q:100, acc: 0.910 + subject: college_mathematics, #q:100, acc: 0.830 + subject: college_medicine, #q:173, acc: 0.896 + Total latency: 42.004 + Average accuracy: 0.893 + ``` + +### 5.2 Speed Benchmark + +We use SGLang's built-in benchmarking tool with its `random` dataset — real prompts sampled from [ShareGPT_Vicuna_unfiltered](https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered) and then truncated/padded to a controlled length. This dataset contains real conversation data and can better reflect performance in actual use scenarios. To simulate real-world usage patterns, we configure each request with 1024 input tokens and 1024 output tokens, representing typical medium-length conversations with detailed responses. + +#### 5.2.1 Hopper **Test Environment:** -- Hardware: NVIDIA B200 GPU (4x) +- Hardware: NVIDIA H200 GPU (4x) - Model: DeepSeek-V4-Flash (FP4) - Tensor Parallelism: 4 -- sglang version: Pending update +- sglang version: 0.5.12 -We use SGLang's built-in benchmarking tool to conduct performance evaluation on the [ShareGPT_Vicuna_unfiltered](https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered) dataset. This dataset contains real conversation data and can better reflect performance in actual use scenarios. To simulate real-world usage patterns, we configure each request with 1024 input tokens and 1024 output tokens, representing typical medium-length conversations with detailed responses. +##### Latency-Sensitive Benchmark -#### 5.1.1 Latency-Sensitive Benchmark - -- **Model Deployment Command:** see the [command panel above](#3-model-deployment). +- **Model Deployment Command:** H200 · DeepSeek-V4-Flash · FP4 · Low-Latency. See the [command panel above](#3-model-deployment). - Benchmark Command: @@ -355,6 +450,7 @@ python3 -m sglang.bench_serving \ --host 127.0.0.1 \ --port 30000 \ --model deepseek-ai/DeepSeek-V4-Flash \ + --dataset-name random \ --random-input-len 1024 \ --random-output-len 1024 \ --num-prompts 10 \ @@ -364,12 +460,49 @@ python3 -m sglang.bench_serving \ - **Test Results:** ```text Output -Pending update — replace with real bench_serving output after the latency run. +============ Serving Benchmark Result ============ +Backend: sglang +Traffic request rate: inf +Max request concurrency: 1 +Successful requests: 10 +Benchmark duration (s): 15.98 +Total input tokens: 6101 +Total input text tokens: 6101 +Total generated tokens: 4220 +Total generated tokens (retokenized): 4220 +Request throughput (req/s): 0.63 +Input token throughput (tok/s): 381.86 +Output token throughput (tok/s): 264.13 +Peak output token throughput (tok/s): 324.00 +Peak concurrent requests: 3 +Total token throughput (tok/s): 645.98 +Concurrency: 1.00 +Accept length: 2.96 +----------------End-to-End Latency---------------- +Mean E2E Latency (ms): 1596.65 +Median E2E Latency (ms): 1274.48 +P90 E2E Latency (ms): 2950.70 +P99 E2E Latency (ms): 3333.18 +---------------Time to First Token---------------- +Mean TTFT (ms): 147.26 +Median TTFT (ms): 132.22 +P99 TTFT (ms): 181.37 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 3.50 +Median TPOT (ms): 3.48 +P99 TPOT (ms): 4.18 +---------------Inter-Token Latency---------------- +Mean ITL (ms): 3.44 +Median ITL (ms): 3.36 +P95 ITL (ms): 5.06 +P99 ITL (ms): 5.15 +Max ITL (ms): 35.31 +================================================== ``` -#### 5.1.2 Throughput-Sensitive Benchmark +##### Throughput-Sensitive Benchmark -- **Model Deployment Command:** see the [command panel above](#3-model-deployment). +- **Model Deployment Command:** H200 · DeepSeek-V4-Flash · FP4 · Max-Throughput. See the [command panel above](#3-model-deployment). - Benchmark Command: @@ -379,6 +512,7 @@ python3 -m sglang.bench_serving \ --host 127.0.0.1 \ --port 30000 \ --model deepseek-ai/DeepSeek-V4-Flash \ + --dataset-name random \ --random-input-len 1024 \ --random-output-len 1024 \ --num-prompts 1000 \ @@ -388,61 +522,57 @@ python3 -m sglang.bench_serving \ - **Test Results:** ```text Output -Pending update — replace with real bench_serving output after the throughput run. -``` - -### 5.2 Accuracy Benchmark - -#### 5.2.1 GSM8K Benchmark - -- **Benchmark Command:** - -```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000 -``` - -- **Test Results:** - - DeepSeek-V4-Flash (FP4, Blackwell) - ``` - Pending update - ``` - - DeepSeek-V4-Flash (FP8, Hopper) - ``` - Pending update - ``` - -#### 5.2.2 MMLU Benchmark - -- **Benchmark Command:** - -```shell Command -cd sglang -bash benchmark/mmlu/download_data.sh -python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 30000 +============ Serving Benchmark Result ============ +Backend: sglang +Traffic request rate: inf +Max request concurrency: 100 +Successful requests: 1000 +Benchmark duration (s): 198.42 +Total input tokens: 512842 +Total input text tokens: 512842 +Total generated tokens: 510855 +Total generated tokens (retokenized): 510765 +Request throughput (req/s): 5.04 +Input token throughput (tok/s): 2584.65 +Output token throughput (tok/s): 2574.64 +Peak output token throughput (tok/s): 4400.00 +Peak concurrent requests: 110 +Total token throughput (tok/s): 5159.28 +Concurrency: 96.21 +----------------End-to-End Latency---------------- +Mean E2E Latency (ms): 19090.29 +Median E2E Latency (ms): 18328.71 +P90 E2E Latency (ms): 35698.68 +P99 E2E Latency (ms): 39161.43 +---------------Time to First Token---------------- +Mean TTFT (ms): 302.41 +Median TTFT (ms): 131.35 +P99 TTFT (ms): 2172.03 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 37.46 +Median TPOT (ms): 37.72 +P99 TPOT (ms): 55.72 +---------------Inter-Token Latency---------------- +Mean ITL (ms): 36.85 +Median ITL (ms): 21.75 +P95 ITL (ms): 107.64 +P99 ITL (ms): 134.58 +Max ITL (ms): 1930.74 +================================================== ``` -- **Test Results:** - - DeepSeek-V4-Flash (FP4, Blackwell) - ``` - Pending update - ``` - - DeepSeek-V4-Flash (FP8, Hopper) - ``` - Pending update - ``` - -### 5.3 Speed Benchmark on Hopper +#### 5.2.2 Blackwell **Test Environment:** -- Hardware: NVIDIA H200 GPU (4x) -- Model: DeepSeek-V4-Flash (FP8) +- Hardware: NVIDIA B200 GPU (4x) +- Model: DeepSeek-V4-Flash (FP4) - Tensor Parallelism: 4 -- sglang version: Pending update +- sglang version: 0.5.12 -#### 5.3.1 Latency-Sensitive Benchmark +##### Latency-Sensitive Benchmark -- **Model Deployment Command:** see the [command panel above](#3-model-deployment). +- **Model Deployment Command:** B200 · DeepSeek-V4-Flash · FP4 · Low-Latency. See the [command panel above](#3-model-deployment). - Benchmark Command: @@ -452,6 +582,7 @@ python3 -m sglang.bench_serving \ --host 127.0.0.1 \ --port 30000 \ --model deepseek-ai/DeepSeek-V4-Flash \ + --dataset-name random \ --random-input-len 1024 \ --random-output-len 1024 \ --num-prompts 10 \ @@ -461,12 +592,49 @@ python3 -m sglang.bench_serving \ - **Test Results:** ```text Output -Pending update — replace with real bench_serving output after the latency run. +============ Serving Benchmark Result ============ +Backend: sglang +Traffic request rate: inf +Max request concurrency: 1 +Successful requests: 10 +Benchmark duration (s): 15.25 +Total input tokens: 6101 +Total input text tokens: 6101 +Total generated tokens: 4220 +Total generated tokens (retokenized): 4220 +Request throughput (req/s): 0.66 +Input token throughput (tok/s): 400.06 +Output token throughput (tok/s): 276.72 +Peak output token throughput (tok/s): 308.00 +Peak concurrent requests: 2 +Total token throughput (tok/s): 676.78 +Concurrency: 1.00 +Accept length: 2.73 +----------------End-to-End Latency---------------- +Mean E2E Latency (ms): 1523.83 +Median E2E Latency (ms): 1173.50 +P90 E2E Latency (ms): 2770.33 +P99 E2E Latency (ms): 3233.82 +---------------Time to First Token---------------- +Mean TTFT (ms): 102.72 +Median TTFT (ms): 85.94 +P99 TTFT (ms): 134.79 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 3.40 +Median TPOT (ms): 3.42 +P99 TPOT (ms): 4.00 +---------------Inter-Token Latency---------------- +Mean ITL (ms): 3.38 +Median ITL (ms): 3.06 +P95 ITL (ms): 4.60 +P99 ITL (ms): 4.95 +Max ITL (ms): 34.64 +================================================== ``` -#### 5.3.2 Throughput-Sensitive Benchmark +##### Throughput-Sensitive Benchmark -- **Model Deployment Command:** see the [command panel above](#3-model-deployment). +- **Model Deployment Command:** B200 · DeepSeek-V4-Flash · FP4 · Max-Throughput (MegaMoE W4A4). See the [command panel above](#3-model-deployment) — flip the **MegaMoE** toggle to **W4A4** to reproduce these numbers; the default Max-Throughput recipe uses `--moe-a2a-backend deepep` and runs slower. - Benchmark Command: @@ -476,6 +644,7 @@ python3 -m sglang.bench_serving \ --host 127.0.0.1 \ --port 30000 \ --model deepseek-ai/DeepSeek-V4-Flash \ + --dataset-name random \ --random-input-len 1024 \ --random-output-len 1024 \ --num-prompts 1000 \ @@ -485,5 +654,41 @@ python3 -m sglang.bench_serving \ - **Test Results:** ```text Output -Pending update — replace with real bench_serving output after the throughput run. +============ Serving Benchmark Result ============ +Backend: sglang +Traffic request rate: inf +Max request concurrency: 100 +Successful requests: 1000 +Benchmark duration (s): 105.10 +Total input tokens: 512842 +Total input text tokens: 512842 +Total generated tokens: 510855 +Total generated tokens (retokenized): 510682 +Request throughput (req/s): 9.51 +Input token throughput (tok/s): 4879.44 +Output token throughput (tok/s): 4860.54 +Peak output token throughput (tok/s): 6600.00 +Peak concurrent requests: 117 +Total token throughput (tok/s): 9739.98 +Concurrency: 94.34 +----------------End-to-End Latency---------------- +Mean E2E Latency (ms): 9915.50 +Median E2E Latency (ms): 9521.19 +P90 E2E Latency (ms): 17726.66 +P99 E2E Latency (ms): 24910.72 +---------------Time to First Token---------------- +Mean TTFT (ms): 349.95 +Median TTFT (ms): 68.23 +P99 TTFT (ms): 4581.26 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 19.86 +Median TPOT (ms): 17.96 +P99 TPOT (ms): 61.58 +---------------Inter-Token Latency---------------- +Mean ITL (ms): 18.76 +Median ITL (ms): 13.23 +P95 ITL (ms): 44.79 +P99 ITL (ms): 88.25 +Max ITL (ms): 2499.49 +================================================== ``` diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx index 8c4c140dd63b..79eb5a446675 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.mdx @@ -26,13 +26,15 @@ SGLang offers multiple installation methods. You can choose the most suitable in Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. +For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation). + ## 3. Model Deployment This section provides deployment configurations optimized for different hardware platforms and use cases. ### 3.1 Basic Configuration -The Qwen3 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA and AMD GPUs. The recommended launch configurations vary by hardware and model size. +The Qwen3 series offers models in various sizes and architectures, optimized for different hardware platforms including NVIDIA GPUs, AMD GPUs, and Intel Xeon CPUs. The recommended launch configurations vary by hardware and model size. **Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, model size, quantization method, and thinking capabilities. @@ -51,6 +53,7 @@ import { Qwen3Deployment } from "/src/snippets/autoregressive/qwen3-deployment.j - `--speculative-eagle-topk 1`: Top-k sampling for draft tokens - `--speculative-num-draft-tokens 4`: Number of draft tokens per step - `--speculative-draft-model-path`: The path of the draft model weights. This can be a local folder or a Hugging Face repo ID such as [`lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan`](https://huggingface.co/lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan). +- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings. ## 4. Model Invocation diff --git a/docs_new/docs.json b/docs_new/docs.json index d6a0a054e11a..91c5aafcc1b7 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -905,9 +905,10 @@ "docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples", "docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples", "docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables", + "docs/hardware-platforms/ascend-npus/ascend_npu_faq", + "docs/hardware-platforms/ascend-npus/ascend_npu_operator_performance_optimizing", "docs/hardware-platforms/ascend-npus/ascend_npu_profiling", - "docs/hardware-platforms/ascend-npus/ascend_npu_operator_development", - "docs/hardware-platforms/ascend-npus/ascend_npu_faq" + "docs/hardware-platforms/ascend-npus/ascend_npu_operator_development" ] }, "docs/hardware-platforms/cpu_server", diff --git a/docs_new/docs/advanced_features/hisparse_guide.mdx b/docs_new/docs/advanced_features/hisparse_guide.mdx index 2c62b0b0afd9..78b71288ab10 100644 --- a/docs_new/docs/advanced_features/hisparse_guide.mdx +++ b/docs_new/docs/advanced_features/hisparse_guide.mdx @@ -6,7 +6,7 @@ metatags: HiSparse reduces per-request GPU memory consumption during the decode phase by maintaining only a small "hot" KV buffer on GPU while keeping complete KV data in CPU pinned memory. Combined with PD disaggregation, it enables significantly higher decode concurrency. -> **Prerequisites**: HiSparse only works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5). These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. +> **Prerequisites**: HiSparse works with models that use **DeepSeek Sparse Attention (DSA)** architectures (e.g., DeepSeek-V3.2, GLM-5.1) and **DeepSeek V4**. These models natively select a subset of tokens for attention, making it possible to keep only the top-k KV on GPU while storing the full KV in host memory — without accuracy loss. Additionally, HiSparse currently requires **PD disaggregation mode** and is enabled on the **decode instance** only. ## Why HiSparse? @@ -44,6 +44,8 @@ Prefill GPU ──RDMA──▶ Decode Host Pool (CPU pinned memory) swap-in kernel (on-demand top-k) ``` +For DeepSeek V4, the direct-to-host path writes only C4 KV into the decode host pool. The c4_indexer and C128 KV remain device-to-device transfers. + ## Server Arguments @@ -141,8 +143,7 @@ python3 -m sglang.launch_server \ --context-length 81920 \ --tp-size 8 --dp-size 8 --enable-dp-attention \ --mem-fraction-static 0.85 \ - --kv-cache-dtype bfloat16 \ - --dsa-decode-backend flashmla_sparse \ + --disable-radix-cache \ --disaggregation-mode decode \ --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 \ --dist-init-addr 127.0.0.1:5757 \ @@ -151,6 +152,8 @@ python3 -m sglang.launch_server \ --hisparse-config='{"top_k": 2048, "device_buffer_size": 6144, "host_to_device_ratio": 10}' ``` +> **Note**: For DSA models, `--kv-cache-dtype` defaults to `auto`, which resolves to `fp8_e4m3` on SM100+ (Blackwell) and `bfloat16` on older architectures. The DSA decode backend is automatically selected based on KV dtype (`bfloat16` → `flashmla_sparse`, `fp8_e4m3` → `flashmla_kv`). DSA backend flags apply only to DSA models; DeepSeek V4 uses its own `dsv4` attention backend. + ### Benchmark ```bash Command @@ -173,14 +176,12 @@ python3 -m sglang.bench_serving \ ### Key Notes - The prefill instance does not need `--enable-hisparse`; it is unaware of HiSparse. -- On the decode instance, the following flags are **required** for HiSparse: - - `--kv-cache-dtype bfloat16` — currently only bfloat16 KV cache is supported (more dtypes planned). - - `--dsa-decode-backend flashmla_sparse` — currently only `flashmla_sparse` backend is supported. - - `--enable-hisparse` — enables HiSparse. - - `--hisparse-config` — HiSparse configuration (top_k, device_buffer_size, host_to_device_ratio). - - `host_to_device_ratio` should be configured based on the host machine's available memory. For example: - - **~1 TB** host memory → `host_to_device_ratio: 5` - - **~2 TB** host memory → `host_to_device_ratio: 10` +- On the decode instance, `--enable-hisparse` and `--hisparse-config` are required for HiSparse. +- For DSA models, `--kv-cache-dtype bfloat16` uses `flashmla_sparse`, and `--kv-cache-dtype fp8_e4m3` uses `flashmla_kv`. +- For DeepSeek V4, DSA backend flags are not applicable. DeepSeek V4 uses the `dsv4` attention backend and `fp8_e4m3` KV cache by default. +- `host_to_device_ratio` should be configured based on the host machine's available memory. For example: + - **~1 TB** host memory → `host_to_device_ratio: 5` + - **~2 TB** host memory → `host_to_device_ratio: 10` ## Acknowledgments diff --git a/docs_new/docs/advanced_features/server_arguments.mdx b/docs_new/docs/advanced_features/server_arguments.mdx index d40f0dce0d68..68f19455fdca 100644 --- a/docs_new/docs/advanced_features/server_arguments.mdx +++ b/docs_new/docs/advanced_features/server_arguments.mdx @@ -1206,6 +1206,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s + + + + + + diff --git a/docs_new/docs/get-started/install.mdx b/docs_new/docs/get-started/install.mdx index 8543e840fcbb..6bc70d421615 100644 --- a/docs_new/docs/get-started/install.mdx +++ b/docs_new/docs/get-started/install.mdx @@ -25,6 +25,16 @@ pip install uv uv pip install sglang ``` +The major version of Cuda is 13 by default. To install sglang under Cuda 12 with pip or uv, please try the following commands: +```bash Command +pip install --upgrade pip +pip install uv +uv pip install sglang +uv pip install --force-reinstall torch==2.11.0 torchaudio==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu129 +uv pip install --force-reinstall sglang-kernel --index-url https://docs.sglang.ai/whl/cu129/ +uv pip install --force-reinstall sgl-deep-gemm --index-url https://docs.sglang.ai/whl/cu129/ --no-deps +``` + ### Quick fixes to common problems - If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions: 1. Use `export CUDA_HOME=/usr/local/cuda-` to set the `CUDA_HOME` environment variable. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx index c78c3998b94b..a4ee1eb73c1c 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu.mdx @@ -82,13 +82,33 @@ docker pull quay.io/ascend/cann:8.5.0-910b-ubuntu22.04-py3.11 #### Python Version -Only `python==3.11` is supported currently. If you don't want to break system pre-installed python, try installing with [conda](https://github.com/conda/conda). +**Only `python==3.11` is supported currently**. If you don't want to break system pre-installed python, try installing with [conda](https://github.com/conda/conda). ```bash Command conda create --name sglang_npu python=3.11 conda activate sglang_npu ``` +Note on Anaconda repository restrictions +If you encounter an error like “Terms of Service have not been accepted” during the conda create step, the default Anaconda repository is blocking package downloads. To resolve this, configure a mirror (e.g., Tsinghua Open Source Mirror): +```bash Command +# Add Tsinghua mirrors +conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/ +conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/ +conda config --set show_channel_urls yes + +# Edit the system-level conda config to remove any hardcoded defaults +vi /root/miniconda3/.condarc +``` +Inside /root/miniconda3/.condarc, delete or comment out any lines containing defaults or official Anaconda URLs. +Then remove the failed environment and recreate it: +```bash Command +conda clean -i +conda env remove -n sglang_npu +conda create --name sglang_npu python=3.11 +conda activate sglang_npu +``` + #### CANN Prior to start work with SGLang on Ascend you need to install CANN Toolkit, Kernels operator package and NNAL version 8.5.0, check the [installation guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/850/softwareinst/instg/instg_0008.html?Mode=PmIns&InstallType=local&OS=openEuler&Software=cannToolKit) diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_operator_performance_optimizing.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_operator_performance_optimizing.mdx new file mode 100644 index 000000000000..c751c6fca9ec --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_operator_performance_optimizing.mdx @@ -0,0 +1,113 @@ +--- +title: "Operator Performance Optimizing Guidance" +metatags: + description: "Operator Performance Optimizing Guidance for NPU" +--- + +## Performance_benchmark + +### Obtaining Performance Data + +Before optimizing the performance, you need to obtain accurate performance data, understand the current performance status, and analyze the next optimization direction based on the performance status. MindStudio provides realistic methods for testing the performance of Triton operators. + +#### Device-end + +The msProf tool is used to collect and analyze key performance indicators of operators running on the Ascend AI Processor. You can use the output performance data to quickly locate the software and hardware performance bottlenecks of operators and improve the efficiency of operator performance analysis. + +```bash +msprof op --kernel-name=xxxxx python3 test_xxxxx.py +``` + +| Attribute | Value | +|------|------| +| Name | DequantSwigluQuant_int32_high_performance_100000000 | +| Type | DequantSwigluQuant | +| OP State | static | +| Accelerator Core | AI_VECTOR_CORE | +| Start Time(us) | 1774489226717521.715 | +| Duration(us) | 102.824 | +| Wait Time(us) | 0 | +| Block Dim | 36 | +| Mix Block Dim | 0 | +| HF32 Eligible | NO | +| Input Shapes | 163840,1024;128,1024;163840;;;;128 | +| Input Data Types | INT32;FLOAT;FLOAT;DT_UNDEFINED;DT_UNDEFINED;DT_UNDEFINED;INT64 | +| Input Formats | ND;ND;ND;NULL;NULL;NULL;ND | +| Output Shapes | 163840,512;163840 | +| Output Data Types | INT8;FLOAT | +| Output Formats | ND;ND | +| Context ID | N/A | +| aicore_time(us) | 0 | +| aic_total_cycles | 0 | +| aic_mac_time(us) | 0 | +| aic_mac_ratio | 0 | +| aic_scalar_time(us) | 0 | +| aic_scalar_ratio | 0 | +| aic_mte1_time(us) | 0 | +| aic_mte1_ratio | 0 | +| aic_mte2_time(us) | 0 | +| aic_mte2_ratio | 0 | +| aic_fixpipe_time(us) | 0 | +| aic_fixpipe_ratio | 0 | +| aic_icache_miss_rate | 0 | +| aiv_time(us) | 59.128 | +| aiv_total_cycles | 3512188 | +| aiv_vec_time(us) | 36.708 | +| aiv_vec_ratio | 0.621 | +| aiv_scalar_time(us) | 41.403 | +| aiv_scalar_ratio | 0.7 | +| aiv_mte2_time(us) | 11.975 | +| aiv_mte2_ratio | 0.203 | +| aiv_mte3_time(us) | 9.738 | +| aiv_mte3_ratio | 0.165 | +| aiv_icache_miss_rate | 0.005 | +| cube_utilization(%) | 0 | + + +The Task Duration field indicates the time consumed by each operator. You can sort the operators by Task Duration to find the operators that consume the most time, or sort the operators by Task Type to view the operators that consume the most time on the AI Core or AI CPU. + +For some operators, the execution time is too long. As a result, the metric data is inaccurate and no longer has reference value. Such data is set to N/A and is not displayed. + +Input Shapes set to an empty value indicates that when the format is "; ; ; ;", the current input is a scalar. Here, ";" serves as the separator for each dimension. The output dimension of the operator follows the same principle. + +- **Task Duration (us)**: Time required for running a task, including the time for scheduling a task to the accelerator, execution time on the accelerator, and response end time. The unit is μs. + +- **Task Wait Time (us)**: Interval between the end time of the previous task and the start time of the current task. The unit is μs. + +- **Block Dim**: Number of blocks into which a task is divided, which corresponds to the number of cores used for running the task. If task_time is L0, this field is not collected and is displayed as 0. + +## Optimization + +### Specification + +#### 1. Ascend core compute units + +- AI Core: the core that actually performs matrix/vector computation +- Vector Unit: responsible for SIMD computation (similar to CUDA Core) +- Scalar Unit: responsible for control/loop +- L0/L1/L2 cache: The smaller the size, the faster the speed. L0 is only 64 KB, L1 is 256 KB, and L2 is shared. + +#### 2. Ascend memory hierarchy (from fastest to slowest) + +- Register → Fastest +- L0/L1 cache → Very fast +- On-chip cache (L2) → Fast +- DDR (host memory) → Slowest + +### 3. Characteristics of Ascend instructions + +Good at accessing large contiguous memory blocks + +Dislikes discrete access, stride access, and random access + +Must be 128-bit/256-bit aligned + +Must be vectorized. + +### Tips + +1. Ascend 910 series usually has only 40 or 48 vector cores. If the number of grids exceeds 40 or 48 vector cores, the grids will be delivered in a queue, resulting in a long waiting time. Therefore, the number of cores for high-performance implementation does not exceed the number of vector cores. +2. Try to use up all the UB as much as possible. Move a large block size at a time to ensure that the bound is in the MTE. No Redundant Copy. +3. If the offset is a negative number, the current triton-ascend considers it as a discrete memory access scenario. As a result, the performance severely deteriorates, and the data is read from the entire DMA block instead of being read in scalar mode. +4. The UB of the Ascend hardware requires that the size of the tail axis of the tensor can be exactly divided by 32 bytes. If the length of the tail axis is insufficient, the length of the tail axis is automatically supplemented. For example, the performance deteriorates exponentially due to automatic supplementation for the Tensor whose shape is (2048, 3). In this situation, you can perform the transposition operation to change the alignment axis to a lower dimension. In addition, the transposition operation is affected by the automatic supplement rule. Therefore, special skills are also required to avoid supplementation. +5. Use Double Buffer, parallelizes computation and data transfer. While computing one block of data, another block of data is being transferred to L1. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx index 71c704424430..e6c33e75ba90 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_quantization.mdx @@ -52,6 +52,14 @@ SGLang support **mix-bits** quantization (independently defines and loads each l + + + + + + + + diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx index eb054f7780aa..ec78a7b3290a 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx @@ -271,6 +271,77 @@ python3 -m sglang.launch_server \ --mm-attention-backend ascend_attn ``` +### Multi-node Deployment + + +Recommended model: [`Qwen/Qwen3.5-35B-A3B`](https://www.modelscope.cn/models/Qwen/Qwen3.5-35B-A3B) + +Other Qwen3.5 series models can also be deployed in multi-node configurations following this workflow. Simply change `--model-path` to the corresponding model, and adjust parameters like `--tp-size`, `--nnodes`, and `--mem-fraction-static` according to the model size and available resources. + + +**A2 series** + +Modify the IP of 2 nodes, then run the same scripts on two nodes. + +**node 0/1** + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export STREAMS_PER_DEVICE=32 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_NPU_USE_MULTI_STREAM=1 +export HCCL_BUFFSIZE=1000 + +# Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + + +P_IP=('your ip1' 'your ip2') +P_MASTER="${P_IP[0]}:your port" + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 --nnodes 2 --node-rank $i --dist-init-addr $P_MASTER \ + --chunked-prefill-size 16384 --max-prefill-tokens 131072 \ + --trust-remote-code \ + --host 127.0.0.1 \ + --mem-fraction-static 0.8\ + --port 8000 \ + --served-model-name qwen3.5 \ + --cuda-graph-max-bs 16 \ + --disable-radix-cache + NODE_RANK=$i + break + fi +done +``` ### Prefill-Decode Disaggregation Not tested yet. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/diffusion/disaggregation.mdx b/docs_new/docs/hardware-platforms/ascend-npus/diffusion/disaggregation.mdx new file mode 100644 index 000000000000..d3adf0bb23a5 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/diffusion/disaggregation.mdx @@ -0,0 +1,33 @@ +--- +title: "Disaggregation of Diffusion Pipeline on Ascend NPU" +--- + +## Quick Start + +Please follow the [NPU installation guide](../ascend_npu.mdx) first. + +Then install Mooncake from sources. + +```bash +git clone https://github.com/kvcache-ai/Mooncake.git +cd Mooncake +git checkout v0.3.10.post2 +bash dependencies.sh +mkdir build +cd build +export GLOG_logtostderr=1 +cmake -DUSE_ASCEND_DIRECT=ON .. +make -j +make install +cd ../mooncake-wheel/ +pip install . +``` + +Before run servers ensure that *.so files of mooncake in LD_LIBRARY_PATH +```bash +export LD_LIBRARY_PATH=/usr/local/python3.11.14/lib/python3.11/site-packages/mooncake:$LD_LIBRARY_PATH +``` + +## Examples + +For usage examples please follow the [main disaggregation guide](../../../sglang-diffusion/disaggregation.mdx) diff --git a/docs_new/docs/hardware-platforms/xpu.mdx b/docs_new/docs/hardware-platforms/xpu.mdx index b5a6c8a2cce1..4ec4d4253b2b 100644 --- a/docs_new/docs/hardware-platforms/xpu.mdx +++ b/docs_new/docs/hardware-platforms/xpu.mdx @@ -55,6 +55,7 @@ conda activate sgl-xpu # Set PyTorch XPU as primary pip install channel to avoid installing the larger CUDA-enabled version and prevent potential runtime issues. pip3 install torch==2.11.0+xpu torchao torchvision torchaudio==2.11.0+xpu --index-url https://download.pytorch.org/whl/xpu pip3 install xgrammar --no-deps # xgrammar will introduce CUDA-enabled triton which might conflict with XPU +pip3 install apache-tvm-ffi # xgrammar requires apache-tvm-ffi # Clone the SGLang code git clone https://github.com/sgl-project/sglang.git diff --git a/docs_new/docs/references/environment_variables.mdx b/docs_new/docs/references/environment_variables.mdx index 61dae0e07095..3efc356a1c59 100644 --- a/docs_new/docs/references/environment_variables.mdx +++ b/docs_new/docs/references/environment_variables.mdx @@ -416,6 +416,16 @@ SGLang supports various environment variables that can be used to configure its + + + + + + + + + + diff --git a/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx b/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx index 9f234bfa2cad..5091df562c57 100644 --- a/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx +++ b/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx @@ -435,6 +435,14 @@ Optimization columns are abbreviated to keep the matrix readable: + + + + + + + + diff --git a/docs_new/docs/sglang-diffusion/dynamic_batching.mdx b/docs_new/docs/sglang-diffusion/dynamic_batching.mdx index b05a6eb892f2..795536a49f68 100644 --- a/docs_new/docs/sglang-diffusion/dynamic_batching.mdx +++ b/docs_new/docs/sglang-diffusion/dynamic_batching.mdx @@ -65,6 +65,8 @@ An initial implementation of dynamic batching for T2I and T2V models can be foun + + diff --git a/docs_new/docs/sglang-diffusion/environment_variables.mdx b/docs_new/docs/sglang-diffusion/environment_variables.mdx index 8ade9a7ca47a..b864521e91f1 100644 --- a/docs_new/docs/sglang-diffusion/environment_variables.mdx +++ b/docs_new/docs/sglang-diffusion/environment_variables.mdx @@ -150,7 +150,7 @@ description: "Configure SGLang diffusion behavior with environment variables." - +
`fa3` flashmla_sparse, flashmla_kv, fa3, tilelang, aiter, trtllm
`--dsa-topk-backend`Choose the DSA indexer top-k backend. The `torch` backend currently requires `SGLANG_DSA_FUSE_TOPK=false`.`sgl-kernel`sgl-kernel, torch, flashinfer
`--fp8-gemm-backend` Choose the runner backend for Blockwise FP8 GEMM operations. Options: 'auto' (default, auto-selects based on hardware), 'deep_gemm' (JIT-compiled; enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) when DeepGEMM is installed), 'flashinfer_trtllm' (FlashInfer TRTLLM backend; SM100/SM103 only), 'flashinfer_cutlass' (FlashInfer CUTLASS backend, SM120 only), 'flashinfer_deepgemm' (Hopper SM90 only, uses swapAB optimization for small M dimensions in decoding), 'cutlass' (optimal for Hopper/Blackwell GPUs and high-throughput), 'triton' (fallback, widely compatible), 'aiter' (ROCm only).
MXFP4Linearxx
W4A4 dynamic MoEFuse the operation of picking topk logits and picking topk indices from page table. SGLANG_NSA_FUSE_TOPK is a deprecated alias. true
SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTICUse deterministic FlashInfer topk kernels when --dsa-topk-backend=flashinfer.false
SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAKTie-break mode for FlashInfer DSA topk when --dsa-topk-backend=flashinfer: unset disables explicit tie-breaking, small prefers the smaller candidate index for equal scores, and large prefers the larger candidate index for equal scores. Setting this variable makes FlashInfer use deterministic topk.unset
SGLANG_DSA_ENABLE_MTP_PRECOMPUTE_METADATA Precompute metadata that can be shared among different draft steps when MTP is enabled. SGLANG_NSA_ENABLE_MTP_PRECOMPUTE_METADATA is a deprecated alias.FLUX.2-Klein-9B black-forest-labs/FLUX.2-klein-9B
FLUX.2-Klein-Base-4Bblack-forest-labs/FLUX.2-klein-base-4B
FLUX.2-Klein-Base-9Bblack-forest-labs/FLUX.2-klein-base-9B
Z-Image Tongyi-MAI/Z-Image
FLUX.2-dev-NVFP4??
FLUX.2-Klein-4B
FLUX.2-Klein-9B??
FLUX.2-Klein-Base-4B??
FLUX.2-Klein-Base-9B??
Z-Image?-
Z-Image-Turbo-
GLM-Image-
SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND not setFlashInfer FP4 GEMM backend for generic NVFP4 fallbackOptional FlashInfer FP4 GEMM backend override for diffusion NVFP4. When unset, SGLang defaults to flashinfer_trtllm.
diff --git a/docs_new/docs/sglang-diffusion/quantization.mdx b/docs_new/docs/sglang-diffusion/quantization.mdx index 043ef28ce60c..8fedb80b2453 100644 --- a/docs_new/docs/sglang-diffusion/quantization.mdx +++ b/docs_new/docs/sglang-diffusion/quantization.mdx @@ -102,7 +102,7 @@ backend. --model-path Wan2.2 family None - Currently only compatible with the Ascend NPU family and supports mxfp8, w8a8, and w4a4 + Currently only compatible with the Ascend NPU family and supports mxfp8, mxfp4, w8a8, and w4a4 @@ -210,7 +210,7 @@ official full Diffusers repos, and the FLUX.2 NVFP4 entry keeps the official --model-path nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4 full Diffusers repo with ModelOpt NVFP4 Wan2.2 components - current B200/Blackwell bring-up uses SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=trtllm + default FP4 GEMM backend is flashinfer_trtllm @@ -327,7 +327,6 @@ sglang generate \ For Wan2.2 NVFP4: ```bash -SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=trtllm \ sglang generate \ --model-path nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4 \ --prompt "a fox walking through neon rain" \ @@ -340,23 +339,19 @@ sglang generate \ directories that already include `config.json`. - Use `--transformer-weights-path` for raw NVFP4 exports, individual safetensors files, or repo layouts that should be treated as weights first. -- For legacy mixed Wan2.2 transformer overrides, the primary - `--transformer-path` override targets only `transformer`. Use a per-component - override such as `--transformer-2-path` only when you intentionally want a - non-default `transformer_2`. -- On Blackwell, the validated Wan2.2 ModelOpt NVFP4 path currently prefers - FlashInfer FP4 GEMM via - `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=trtllm`. -- This environment-variable override selects the validated Wan2.2 NVFP4 - full-repo path on Blackwell while the other NVFP4 CI cases continue to use - the generic `cudnn` backend. +- For dual-transformer pipelines such as `Wan2.2-T2V-A14B-Diffusers`, the + primary `--transformer-path` override targets only `transformer`. Use a + per-component override such as `--transformer-2-path` only when you + intentionally want a non-default `transformer_2`. +- On Blackwell, the diffusion ModelOpt NVFP4 path defaults to FlashInfer + TensorRT-LLM FP4 GEMM (`flashinfer_trtllm`). - Direct `--model-path` loading is a compatibility path for FLUX.2 NVFP4-style repos or local directories. - If `--transformer-weights-path` is provided explicitly, it takes precedence over the compatibility `--model-path` flow. - For local directories, SGLang first looks for `*-mixed.safetensors`, then falls back to loading from the directory. -- To force the generic diffusion ModelOpt FP4 path onto a specific FlashInfer +- To force the diffusion ModelOpt FP4 path onto a different FlashInfer backend, set `SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND`. Supported values include `flashinfer_cudnn`, `flashinfer_cutlass`, and `flashinfer_trtllm`. - On disk, the quantization config stays `quant_method=modelopt` with @@ -601,6 +596,8 @@ MindStudio-ModelSlim (msModelSlim) is a model offline quantization compression t - [x] ```W8A8_DYNAMIC``` linear with online quantization of activations - [x] ```W8A8_MXFP8``` linear with offline quantization (msmodelslim pre-quantized weights) - [x] ```mxfp8``` linear with online quantization (`--quantization mxfp8`) + - [x] ```W4A4_MXFP4``` / ```W4A4_MXFP4_DUALSCALE``` linear with offline quantization (msmodelslim pre-quantized weights) + - [x] ```mxfp4_npu``` linear with online quantization (`--quantization mxfp4_npu`) ## MXFP8 Online Quantization @@ -630,3 +627,46 @@ sglang generate \ --prompt "a beautiful sunset" \ --save-output ``` + +## MXFP4 Online Quantization + +For online MXFP4 quantization on Ascend NPU, load the original FP16/BF16 model and add +`--quantization mxfp4_npu`. The `mxfp4_npu` key is used for Ascend because `mxfp4` +is reserved for the ROCm/aiter backend. + +Weights are quantized at load time via `npu_dynamic_dual_level_mx_quant`, and activations +are quantized per-token during inference before `npu_dual_level_quant_matmul`. MXFP4 uses +dual-level block scales with an L1 block size of 32 and an L0 block size of 512. + +```bash +sglang generate \ + --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --quantization mxfp4_npu \ + --prompt "a fox walking through neon rain" \ + --save-output +``` + +> **Hardware requirement:** Ascend A5 series or newer. `npu_dynamic_dual_level_mx_quant` +> and `npu_dual_level_quant_matmul` are not available on A2/A3. +> +> **Note:** Online MXFP4 weight quantization is experimental. The offline msmodelslim +> flow uses pre-quantized weights and may produce different numerical results. + +## MXFP4 Offline Quantization (msmodelslim) + +Pre-quantized MXFP4 weights exported by msmodelslim are auto-detected via +`quant_model_description.json` (`W4A4_MXFP4` / `W4A4_MXFP4_DUALSCALE` scheme). +Use `wan_repack.py` to convert the quantized weights to Diffusers format, then load +the converted model with `--model-path`: + +```bash +sglang generate \ + --model-path {path_to_converted_mxfp4_model} \ + --prompt "a beautiful sunset" \ + --save-output +``` + +The offline MXFP4 checkpoint stores weights in an FP8 container and includes dual-level +scales (`weight_scale`, `weight_dual_scale`). If exported with smooth quantization, +`mul_scale` is loaded and applied before activation quantization to keep activations +aligned with the calibrated weights. diff --git a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx index 455fcd14a4e0..006e28d73e12 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-v4-deployment.jsx @@ -27,13 +27,12 @@ export const DeepSeekV4Deployment = () => { name: "hardware", title: "Hardware Platform", items: [ - { id: "b200", label: "B200 (FP4)", default: true }, - { id: "b300", label: "B300 (FP4)", default: false }, - { id: "gb200", label: "GB200 (FP4)", default: false }, - { id: "gb300", label: "GB300 (FP4)", default: false }, - { id: "h200", label: "H200 (FP8)", default: false }, - { id: "h200-fp4", label: "H200 (FP4)", default: false }, - { id: "h100", label: "H100 (FP4)", default: false }, + { id: "b200", label: "B200", default: true }, + { id: "b300", label: "B300", default: false }, + { id: "gb200", label: "GB200", default: false }, + { id: "gb300", label: "GB300", default: false }, + { id: "h200", label: "H200", default: false }, + { id: "h100", label: "H100", default: false }, ], }, modelSize: { @@ -44,6 +43,14 @@ export const DeepSeekV4Deployment = () => { { id: "big", label: "Pro", default: false, subtitle: "1.6T" }, ], }, + quantization: { + name: "quantization", + title: "Quantization", + items: [ + { id: "fp4", label: "FP4", default: true }, + { id: "fp8", label: "FP8", default: false, subtitle: "H100/H200 only" }, + ], + }, recipe: { name: "recipe", title: "Recipe", @@ -90,37 +97,100 @@ export const DeepSeekV4Deployment = () => { }, }; + // Hopper GPUs supporting the SGLang FP8 repackaging path. + const FP8_SUPPORTED_HARDWARE = new Set(["h100", "h200"]); + + // Internal "effective hardware" id used by HW_SIZE_SPEC / VERIFIED_RECIPES. + // Combines the user-facing hardware choice with the Quantization axis: + // h200 + fp4 → h200-fp4 (Marlin FP4 path on H200) + // h200 + fp8 → h200 (sgl-project FP8 ckpts on H200) + // h100 + fp4 → h100 (Marlin FP4 path on H100) + // h100 + fp8 → h100-fp8 (Flash-only FP8 path on H100) + // anything else → hardware unchanged + const effHw = (hardware, quantization) => { + if (hardware === "h200") return quantization === "fp8" ? "h200" : "h200-fp4"; + if (hardware === "h100") return quantization === "fp8" ? "h100-fp8" : "h100"; + return hardware; + }; + // Recipes that are not supported on the Marlin (FP4) Hopper paths // (H200 FP4, H100 FP4). const MARLIN_UNSUPPORTED_RECIPES = new Set(["cp", "pd-disagg"]); - const MARLIN_HARDWARE = new Set(["h200-fp4", "h100"]); + const MARLIN_EFFHW = new Set(["h200-fp4", "h100"]); const MARLIN_LABEL = { "h200-fp4": "H200 (FP4)", h100: "H100 (FP4)" }; - // MegaMoE is only supported on Blackwell with DeepEP-based recipes - // (balanced / max-throughput / pd-disagg). It's disabled on Hopper - // (H100 / H200 / H200-FP4) and on low-latency / cp recipes. - const MEGAMOE_UNSUPPORTED_RECIPES = new Set(["low-latency", "cp"]); - const MEGAMOE_UNSUPPORTED_HARDWARE = new Set(["h100", "h200", "h200-fp4"]); + // MegaMoE is only wired into the deepep-replacing recipes on Blackwell + // (balanced / max-throughput). Disabled on Hopper (H100 / H200, both FP4 + // and FP8), on low-latency / cp recipes, and on PD-Disagg (the cookbook's + // PD command builder doesn't emit the megamoe backend / env vars yet). + const MEGAMOE_UNSUPPORTED_RECIPES = new Set(["low-latency", "cp", "pd-disagg"]); + const MEGAMOE_UNSUPPORTED_HARDWARE = new Set(["h100", "h200"]); const isMegamoeUnsupported = (vals) => MEGAMOE_UNSUPPORTED_HARDWARE.has(vals.hardware) || MEGAMOE_UNSUPPORTED_RECIPES.has(vals.recipe); + // HiCache works on PD-Disagg in SGLang itself (prefill worker only, per + // mooncake_store/README.md), but the cookbook generator doesn't yet emit + // the hicache flags into buildPDDisaggCommand. Grey it out for now. + const HICACHE_UNSUPPORTED_RECIPES = new Set(["pd-disagg"]); + const isHicacheUnsupported = (vals) => + HICACHE_UNSUPPORTED_RECIPES.has(vals.recipe); + + // H100 + SGLang FP8 only ships a Flash variant — Pro FP8 on H100 isn't + // covered by the generator yet, so the Pro radio is greyed out there. + const isProDisabledFp8H100 = (vals) => + vals.hardware === "h100" && vals.quantization === "fp8"; + const resolveItems = (option, vals) => { - if (option.name === "recipe" && vals && MARLIN_HARDWARE.has(vals.hardware)) { + const eff = vals ? effHw(vals.hardware, vals.quantization) : null; + if (option.name === "recipe" && eff && MARLIN_EFFHW.has(eff)) { return option.items.map((it) => MARLIN_UNSUPPORTED_RECIPES.has(it.id) - ? { ...it, disabled: true, disabledReason: `Not supported on ${MARLIN_LABEL[vals.hardware]}` } + ? { ...it, disabled: true, disabledReason: `Not supported on ${MARLIN_LABEL[eff]}` } + : it + ); + } + if (option.name === "recipe" && eff === "h100-fp8") { + // H100 SGLang FP8 only has low-latency / balanced / max-throughput + // commands verified — cp and pd-disagg fall back to the Marlin + // "not supported" message. + return option.items.map((it) => + MARLIN_UNSUPPORTED_RECIPES.has(it.id) + ? { ...it, disabled: true, disabledReason: "Not supported on H100 (SGLang FP8)" } : it ); } if (option.name === "megamoe" && vals && isMegamoeUnsupported(vals)) { const reason = MEGAMOE_UNSUPPORTED_HARDWARE.has(vals.hardware) ? "MegaMoE is only supported on Blackwell" + : vals.recipe === "pd-disagg" + ? "MegaMoE is not yet wired into the PD-Disagg cookbook command" : "MegaMoE is not supported on this recipe"; return option.items.map((it) => it.id === "disabled" ? it : { ...it, disabled: true, disabledReason: reason } ); } + if (option.name === "hicache" && vals && isHicacheUnsupported(vals)) { + return option.items.map((it) => + it.id === "disabled" + ? it + : { ...it, disabled: true, disabledReason: "HiCache is not yet wired into the PD-Disagg cookbook command" } + ); + } + if (option.name === "quantization" && vals && !FP8_SUPPORTED_HARDWARE.has(vals.hardware)) { + return option.items.map((it) => + it.id === "fp8" + ? { ...it, disabled: true, disabledReason: "SGLang FP8 is only available on H100 / H200" } + : it + ); + } + if (option.name === "modelSize" && vals && isProDisabledFp8H100(vals)) { + return option.items.map((it) => + it.id === "big" + ? { ...it, disabled: true, disabledReason: "H100 SGLang FP8 only ships a Flash variant" } + : it + ); + } return option.items; }; @@ -158,12 +228,31 @@ export const DeepSeekV4Deployment = () => { const handleRadioChange = (optionName, value) => { setValues((prev) => { const next = { ...prev, [optionName]: value }; + // Switching to a hardware that doesn't support FP8 while FP8 is + // selected: fall back to FP4. + if ( + optionName === "hardware" && + next.quantization === "fp8" && + !FP8_SUPPORTED_HARDWARE.has(value) + ) { + next.quantization = "fp4"; + } + // H100 + SGLang FP8 only supports Flash; auto-flip Pro → Flash when + // entering that combo (via hardware or quantization switch). + if ( + (optionName === "hardware" || optionName === "quantization") && + isProDisabledFp8H100(next) && + next.modelSize === "big" + ) { + next.modelSize = "small"; + } // Switching to a Marlin (FP4) Hopper path while cp / pd-disagg is // selected: fall back to low-latency since those recipes are not // supported on Marlin. + const nextEff = effHw(next.hardware, next.quantization); if ( - optionName === "hardware" && - MARLIN_HARDWARE.has(value) && + (optionName === "hardware" || optionName === "quantization") && + (MARLIN_EFFHW.has(nextEff) || nextEff === "h100-fp8") && MARLIN_UNSUPPORTED_RECIPES.has(next.recipe) ) { next.recipe = "low-latency"; @@ -177,6 +266,25 @@ export const DeepSeekV4Deployment = () => { ) { next.megamoe = "disabled"; } + // Switching to a recipe that doesn't support HiCache (pd-disagg) while + // L2 is selected: fall back to disabled. + if ( + optionName === "recipe" && + next.hicache !== "disabled" && + isHicacheUnsupported(next) + ) { + next.hicache = "disabled"; + } + // Switching to max-throughput on supported hardware: default MegaMoE to + // W4A8 if it's currently disabled (best throughput config). + if ( + (optionName === "recipe" || optionName === "hardware") && + next.recipe === "max-throughput" && + next.megamoe === "disabled" && + !isMegamoeUnsupported(next) + ) { + next.megamoe = "w4a8"; + } return next; }); }; @@ -230,6 +338,10 @@ export const DeepSeekV4Deployment = () => { // a higher TP: Flash fits at TP=8 single-node, Pro needs TP=16 across 2 nodes. "h100|small": { slug: "deepseek-ai/DeepSeek-V4-Flash", tp: 8, multinode: false }, "h100|big": { slug: "deepseek-ai/DeepSeek-V4-Pro", tp: 16, multinode: true, nnodes: 2 }, + // H100 (SGLang FP8) ships Flash only — Pro FP8 on H100 is not exposed by + // the generator. TP=8 single-node uses the same sgl-project FP8 ckpt as + // H200; the Flash/balanced/max-throughput recipes use TP=8 DP=8 + DeepEP. + "h100-fp8|small": { slug: "sgl-project/DeepSeek-V4-Flash-FP8", tp: 8, multinode: false }, }; // Per (hardware, modelSize) PD role TP (from allinone _PD_SPEC). const PD_TP_SPEC = { @@ -297,6 +409,9 @@ export const DeepSeekV4Deployment = () => { "h100|big|low-latency", "h100|big|balanced", "h100|big|max-throughput", + "h100-fp8|small|low-latency", + "h100-fp8|small|balanced", + "h100-fp8|small|max-throughput", ]); // Recipes whose command is intentionally not yet provided (e.g. blocked by an // upstream limitation). Showing a minimal placeholder is friendlier to users @@ -337,10 +452,20 @@ export const DeepSeekV4Deployment = () => { `${cmd}`; // === SHARED END === + // Hopper FP8 paths (effHw values that share the sgl-project FP8 codepath): + // both H200 FP8 and H100 SGLang FP8 go through the same "general" command + // branch, skipping flashinfer_mxfp4 / chunked-prefill / mem-frac flags that + // are Blackwell-FP4-specific. + const isHopperFp8 = (effHwId) => effHwId === "h200" || effHwId === "h100-fp8"; + const generateCommand = () => { - const { hardware: rawHardware, modelSize, recipe, reasoningParser, toolcall, hicache, megamoe } = values; + const { hardware: userHardware, modelSize, quantization, recipe, reasoningParser, toolcall, hicache, megamoe } = values; // B300 usage is identical to B200 — alias so we don't duplicate every spec entry. - const hardware = rawHardware === "b300" ? "b200" : rawHardware; + const rawHardware = userHardware === "b300" ? "b200" : userHardware; + // Translate (hardware, quantization) into the internal "effective hw" id + // that HW_SIZE_SPEC / VERIFIED_RECIPES are keyed by. See the effHw helper + // at the top of this component for the full mapping. + const hardware = effHw(rawHardware, quantization); const specKey = `${hardware}|${modelSize}`; const spec = HW_SIZE_SPEC[specKey]; const { slug, tp, multinode, nnodes } = spec; @@ -350,22 +475,33 @@ export const DeepSeekV4Deployment = () => { return buildPDDisaggCommand(hardware, modelSize); } - // H200 (FP4) Marlin path: dedicated branch — Hopper runs the FP4-mixed - // Instruct repos through the Marlin MoE runner, so it doesn't share envs - // or flags with either the FP8 H200 path or the Blackwell paths. + // H200 (FP4) path: dedicated branch — Hopper runs the FP4-mixed Instruct + // repos through one of two w4a16 MoE runners (Marlin or Flashinfer mxfp4), + // so it doesn't share envs or flags with either the FP8 H200 path or the + // Blackwell paths. // Flash: TP=4, single node Pro: TP=8, single node // low-latency: MTP 3 / 1 / 4 (steps / topk / draft-tokens) // balanced: MTP 1 / 1 / 2 // max-throughput: MTP disabled + // + // MoE runner selection (verified on 2026-05-20): + // - Pro: flashinfer_mxfp4 for all recipes + // - Flash Balanced: flashinfer_mxfp4 (~1.5x faster output throughput vs + // Marlin in the balanced throughput benchmark). + // - Flash Low-Latency / Max-Throughput: Marlin (faster than + // flashinfer_mxfp4 in those benchmarks). if (hardware === "h200-fp4") { const verifyKey = `${hardware}|${modelSize}|${recipe}`; if (TBD_RECIPES.has(verifyKey)) return TBD_PLACEHOLDER; + const useFlashinferMxfp4 = isBig || recipe === "balanced"; const fp4Flags = [ " --trust-remote-code", ` --model-path ${slug}`, ` --tp ${tp}`, - " --moe-runner-backend marlin", + useFlashinferMxfp4 + ? " --moe-runner-backend flashinfer_mxfp4" + : " --moe-runner-backend marlin", ]; if (recipe === "low-latency") { fp4Flags.push(" --speculative-algo EAGLE"); @@ -378,7 +514,14 @@ export const DeepSeekV4Deployment = () => { fp4Flags.push(" --speculative-eagle-topk 1"); fp4Flags.push(" --speculative-num-draft-tokens 2"); } - if (isBig) fp4Flags.push(" --mem-fraction-static 0.88"); + // H200 Pro (FP4) low-latency runs MTP 3/1/4 with flashinfer_mxfp4, which + // needs more headroom for the draft model + MTP buffers than the + // balanced / max-throughput recipes — drop mem-frac to 0.83. + if (isBig) { + fp4Flags.push(recipe === "low-latency" + ? " --mem-fraction-static 0.83" + : " --mem-fraction-static 0.88"); + } if (toolcall === "enabled") fp4Flags.push(" --tool-call-parser deepseekv4"); if (reasoningParser === "enabled") fp4Flags.push(" --reasoning-parser deepseek-v4"); if (hicache === "l2") { @@ -459,11 +602,12 @@ export const DeepSeekV4Deployment = () => { // _LAUNCH_HEAD always prepends these: // Per-hardware env (whitelist #1: NVSHMEM removed for B200). const HW_ENV = { - h200: ["SGLANG_DSV4_FP4_EXPERTS=0"], // allinone _ENV_H200 - b200: [], // _ENV_B200 minus NVSHMEM - gb300: [], // _ENV_GB300 + h200: ["SGLANG_DSV4_FP4_EXPERTS=0"], // allinone _ENV_H200 + "h100-fp8": ["SGLANG_DSV4_FP4_EXPERTS=0"], // H100 SGLang FP8 shares H200's FP8 env + b200: [], // _ENV_B200 minus NVSHMEM + gb300: [], // _ENV_GB300 // GB200 multinode needs NCCL MNNVL for cross-node NVLink communication. - gb200: multinode ? ["NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1"] : [], + gb200: multinode ? ["NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1"] : [], }[hardware]; // Recipe-specific env (matches allinone exactly, taking size into account). @@ -476,7 +620,7 @@ export const DeepSeekV4Deployment = () => { recipeEnv.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); } } else if (recipe === "balanced") { - if (hardware === "h200") { + if (isHopperFp8(hardware)) { recipeEnv.push(isBig ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128" : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); @@ -488,7 +632,7 @@ export const DeepSeekV4Deployment = () => { : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=1024"); } } else if (recipe === "max-throughput") { - if (hardware === "h200") { + if (isHopperFp8(hardware)) { recipeEnv.push(isBig ? "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128" : "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); @@ -533,7 +677,7 @@ export const DeepSeekV4Deployment = () => { if (hardware === "h200" && isBig) { flags.push(" --moe-a2a-backend deepep"); } - if (hardware !== "h200") { + if (!isHopperFp8(hardware)) { flags.push(" --moe-runner-backend flashinfer_mxfp4"); } if (hardware === "h200" && isBig) { @@ -545,14 +689,14 @@ export const DeepSeekV4Deployment = () => { flags.push(" --speculative-num-steps 3"); flags.push(" --speculative-eagle-topk 1"); flags.push(" --speculative-num-draft-tokens 4"); - if (hardware !== "h200") { + if (!isHopperFp8(hardware)) { // B200/B300 Pro accuracy-verified: chunked-prefill-size 8192 flags.push(isBig ? " --chunked-prefill-size 8192" : " --chunked-prefill-size 4096"); flags.push(" --disable-flashinfer-autotune"); flags.push(" --swa-full-tokens-ratio 0.1"); } // B200/B300 Pro accuracy-verified: mem-fraction-static 0.90 - if (isBig && hardware !== "h200") { + if (isBig && !isHopperFp8(hardware)) { flags.push(" --mem-fraction-static 0.90"); } else if (isBig) { flags.push(" --mem-fraction-static 0.88"); @@ -605,7 +749,8 @@ export const DeepSeekV4Deployment = () => { } // allinone H200 gates DEEPEP_LARGE_SMS_FLAG on !multinode — only H200 big // is multi-node; all Blackwell cells get the flag unconditionally. - if (!multinode) flags.push(DEEPEP_LARGE_SMS_FLAG); + // Skip when MegaMoE is enabled (uses its own backend, not DeepEP). + if (!multinode && megamoe === "disabled") flags.push(DEEPEP_LARGE_SMS_FLAG); } else if (recipe === "max-throughput") { // allinone max-throughput: TP + DP + DP-attn + DeepEP (NO MTP). // H200 small: cg=128 max-run=256 | H200 big: cg=128 max-run=256 (same) @@ -642,7 +787,7 @@ export const DeepSeekV4Deployment = () => { flags.push(" --cuda-graph-max-bs 64"); flags.push(" --max-running-requests 256"); } - if (!multinode) flags.push(DEEPEP_LARGE_SMS_FLAG); + if (!multinode && megamoe === "disabled") flags.push(DEEPEP_LARGE_SMS_FLAG); } else if (recipe === "cp") { // allinone cp: TP (NO --dp) + DeepEP + _CP_FLAGS (mem-frac 0.78, max-run 1024). // Blackwell big additionally: mem-frac 0.70 (overrides), cg=256, max-run=256. @@ -650,8 +795,12 @@ export const DeepSeekV4Deployment = () => { flags.push(` --tp ${tp}`); if (multinode) flags.push(...multiNodeFlags(nnodes)); flags.push(" --moe-a2a-backend deepep"); - flags.push(" --enable-dsa-prefill-context-parallel"); - flags.push(" --dsa-prefill-cp-mode round-robin-split"); + // PR #25821 (merged 2026-05-20) renamed these flags from --enable-nsa-* / + // --nsa-prefill-cp-mode to --enable-dsa-* / --dsa-prefill-cp-mode. The + // :latest release image predates that PR, so we emit the old nsa-* names + // here and surface a note above the command for main-branch users. + flags.push(" --enable-nsa-prefill-context-parallel"); + flags.push(" --nsa-prefill-cp-mode round-robin-split"); flags.push(" --chunked-prefill-size 16384"); // GB300 big CP needs higher mem-fraction-static: Pro 1.6T weights at // tp=4 are ~224 GB/card on a 273 GB GB300, so 0.78 leaves a negative @@ -715,6 +864,12 @@ export const DeepSeekV4Deployment = () => { if (megamoe !== "disabled" && recipe === "max-throughput") { megamoeEnv.push("SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320"); } + // Blackwell balanced always runs with MTP (1/1/2) — when MegaMoE is layered on + // top, cap the per-rank dispatch buffer at 4096 to keep MoE memory in budget. + // (megamoe is gated to Blackwell by MEGAMOE_UNSUPPORTED_HARDWARE.) + if (megamoe !== "disabled" && recipe === "balanced") { + megamoeEnv.push("SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096"); + } if (megamoe === "w4a4") { megamoeEnv.push("SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1"); megamoeEnv.push("SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1"); @@ -737,6 +892,15 @@ export const DeepSeekV4Deployment = () => { // GB200 multinode may need machine-specific NVSHMEM / Gloo env vars; // emit them as commented hints above the env block so users know to check. let cmd = base; + if (recipe === "cp") { + cmd = + `# NOTE: --enable-nsa-prefill-context-parallel / --nsa-prefill-cp-mode were\n` + + `# renamed to --enable-dsa-prefill-context-parallel / --dsa-prefill-cp-mode\n` + + `# in PR #25821 (merged 2026-05-20). The cookbook emits the old nsa-* names\n` + + `# because the :latest release image predates that PR. If you're running\n` + + `# SGLang built from main, replace nsa- with dsa- in the two flags below.\n` + + cmd; + } if (hardware === "gb200" && multinode) { cmd = `# The following env vars may be needed depending on your cluster:\n` + @@ -745,6 +909,20 @@ export const DeepSeekV4Deployment = () => { `# NVSHMEM_HCA_LIST=\n` + cmd; } + // GB200 Pro with MegaMoE disabled runs the DeepEP a2a backend, which is + // currently only packaged in the CUDA 12.9 image — the default `:latest` + // ships CUDA 13 and does not include a compatible DeepEP build. + if ( + hardware === "gb200" && + isBig && + megamoe === "disabled" && + flags.some((f) => f.includes("--moe-a2a-backend deepep")) + ) { + cmd = + `# NOTE: for the DeepEP backend, use the cu129 docker image\n` + + `# (lmsysorg/sglang:latest-cu129) instead of the default \`:latest\`.\n` + + cmd; + } const withMultinode = multinode ? prependMultiNodeNote(cmd, nnodes) : cmd; // H200 Pro low-latency: show BOTH a single-node (TP=8 marlin) variant @@ -899,6 +1077,13 @@ export const DeepSeekV4Deployment = () => { } else { flags.push(" --max-running-requests 256"); } + // Reasoning / tool-call parsers are the OpenAI response formatter + // running in serving_chat.py. The PD HTTP router (sgl-model-gateway + // pd_router.rs::execute_dual_dispatch) returns the decode server's + // response to the client and discards prefill's, so the parsers only + // need to be set on decode. Adding them to prefill would be inert. + if (values.toolcall === "enabled") flags.push(" --tool-call-parser deepseekv4"); + if (values.reasoningParser === "enabled") flags.push(" --reasoning-parser deepseek-v4"); } flags.push(" --host 0.0.0.0"); flags.push(` --port ${port}`); diff --git a/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx index ee99a57938b2..0d4209c3e6e0 100644 --- a/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx @@ -9,7 +9,8 @@ export const Qwen3Deployment = () => { b200: { tp: 8, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 4, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 4, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 4, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 4, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 6, ep: 0, bf16: true, fp8: true } }, '30b': { baseName: '30B-A3B', @@ -19,7 +20,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '32b': { baseName: '32B', @@ -29,7 +31,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 6, ep: 0, bf16: true, fp8: true } }, '14b': { baseName: '14B', @@ -39,7 +42,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '8b': { baseName: '8B', @@ -49,7 +53,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '4b': { baseName: '4B', @@ -59,7 +64,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '1.7b': { baseName: '1.7B', @@ -69,7 +75,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } }, '0.6b': { baseName: '0.6B', @@ -79,7 +86,8 @@ export const Qwen3Deployment = () => { b200: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, - mi355x: { tp: 1, ep: 0, bf16: true, fp8: true } + mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, + xeon: { tp: 3, ep: 0, bf16: true, fp8: true } } }; @@ -94,7 +102,8 @@ export const Qwen3Deployment = () => { { id: 'h200', label: 'H200', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, - { id: 'mi355x', label: 'MI355X', default: false } + { id: 'mi355x', label: 'MI355X', default: false }, + { id: 'xeon', label: 'XEON', default: false } ] }, modelsize: { @@ -261,6 +270,10 @@ export const Qwen3Deployment = () => { let cmd = 'python -m sglang.launch_server \\\n'; cmd += ` --model ${modelName}`; + if (hardware === 'xeon') { + cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule`; + } + if (hwConfig.tp > 1) { cmd += ` \\\n --tp ${hwConfig.tp}`; } diff --git a/experimental/sgl-router/.gitignore b/experimental/sgl-router/.gitignore new file mode 100644 index 000000000000..99e6c61096d6 --- /dev/null +++ b/experimental/sgl-router/.gitignore @@ -0,0 +1,3 @@ +target/ +*.rs.bk +.DS_Store diff --git a/experimental/sgl-router/BENCHMARKS.md b/experimental/sgl-router/BENCHMARKS.md new file mode 100644 index 000000000000..f3bf2bda1f0f --- /dev/null +++ b/experimental/sgl-router/BENCHMARKS.md @@ -0,0 +1,79 @@ +# sgl-router microbench harness + SMG comparison + +This file pairs with `experimental/sgl-router/benches/` and the SMG +Criterion harnesses at: + +- `~/smg_workspace/smg/model_gateway/benches/radix_tree_benchmark.rs` +- `~/smg_workspace/smg/model_gateway/benches/manual_policy_benchmark.rs` +- `~/smg_workspace/smg/model_gateway/benches/router_registry_bench.rs` +- `sgl-model-gateway/benches/*` (in-tree mirror of SMG, same code) + +## Scope + +These are CPU-bound microbenches that don't need GPUs — they target +routing-decision latency only. The full E2E throughput comparison +(genai-bench at 4×H200 against a real SGLang fleet) is **not** part of +this file; it requires a real GPU cluster and is tracked separately. + +## How to run + +sgl-router: +```bash +cd experimental/sgl-router +cargo bench --bench tree_lookup -- --sample-size 30 --measurement-time 3 +cargo bench --bench policy_select -- --sample-size 30 --measurement-time 3 +``` + +SMG (the gateway being deprecated): +```bash +cd ~/smg_workspace/smg/model_gateway +cargo bench --bench radix_tree_benchmark -- --sample-size 30 --measurement-time 3 \ + 'token_match_10w_4096tok|token_insert_10w_4096tok' +cargo bench --bench manual_policy_benchmark +``` + +For the quick smoke runs whose numbers are reproduced below: drop +`--sample-size` to 10 and `--measurement-time` to 2 (Criterion will +warn about reduced statistical confidence but the order-of-magnitude +comparison stands). + +## Smoke-run data points (M1 MacBook, release profile) + +These are NOT the real acceptance numbers — they're a sanity check +that the sgl-router routing primitives are in the same ballpark as the +SMG ones they replace. Real targets come from the cluster-scale +comparison and are tracked separately. + +### Cache-aware lookup (`HashTree` vs SMG `TokenTree`) + +| Bench | sgl-router | SMG TokenTree | Notes | +|---|---|---|---| +| Insert 64 blocks for 1 worker (medium case) | `hashtree_insert/128` ≈ 21.5 µs | `token_insert_10w_4096tok` ≈ 1.05 µs | Numbers not directly comparable — SMG counts per-token insert, sgl-router counts per-block insert. SMG inserts 4096 tokens at a fixed `block_size`; sgl-router inserts 128 pre-hashed `i64` block-hashes. The hashing step (`compute_block_hashes`) is upstream of `HashTree` and not measured here. | +| Match request prefix | `hashtree_match_prefix/w64_bpw128_q64` ≈ 47 ns | `token_match_10w_4096tok` ≈ 1.24 µs | sgl-router's match is a short-circuit walk over `i64` hashes; SMG's match tokenizes + hashes per-call. The fair comparison includes `compute_block_hashes` cost (~ tens of µs depending on prompt length). | + +**Read carefully.** The 26× difference at the match step is not the +end-to-end speedup an operator should expect — `compute_block_hashes` +upstream dominates in real traffic. The number proves that sgl-router's +tree walk is no slower than SMG's, which is what the `routing-decision +latency p50 ≤ 1.10× SMG` acceptance criterion targets. + +### Policy selection (non-cache-aware) + +| Policy | n=4 workers | n=16 | n=64 | n=256 | SMG equivalent | +|---|---|---|---|---|---| +| `round_robin` | 2.5 ns | 2.5 ns | 2.5 ns | 2.5 ns | SMG round-robin is O(1) — same shape. | +| `random` | 16 ns | 36 ns | 137 ns | 471 ns | SMG random is also O(1) per `rand::random()` call; sgl-router's variant grows with n because it `Vec::iter().nth(idx)`. **Action item:** drop sgl-router to O(1) by indexing the slice directly. | +| `power_of_two` | … | … | … | 1.75 µs at n=256 | SMG power-of-two-choices is identical in shape (2× rand + 2× load read). | + +The `random` finding (linear in worker count) is a real follow-up — file +an issue and pair it with a Criterion regression-guard in the same +bench. + +## Pre-deprecation calibration runbook + +Before deleting SMG, every routing-latency metric in the slim-design +spec needs a real-cluster measurement. The bench-harness here is the +small-scale, CPU-only complement; it catches algorithmic regressions in +the routing primitives without burning GPU time. Pair both: this file +in pre-commit / CI tier-2, the real-cluster e2e in the +`pr-test-rust.yml` matrix entry. diff --git a/experimental/sgl-router/Cargo.toml b/experimental/sgl-router/Cargo.toml new file mode 100644 index 000000000000..74368ab7e899 --- /dev/null +++ b/experimental/sgl-router/Cargo.toml @@ -0,0 +1,105 @@ +[workspace] +resolver = "2" +members = ["."] + +[package] +name = "sgl-router" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +description = "Slim KV-aware OpenAI-compatible router for SGLang workers" +publish = false # binary-only; not published to crates.io + +[lib] +name = "sgl_router" +crate-type = ["rlib"] + +[[bin]] +name = "sgl-router" +path = "src/main.rs" + +[lints.rust] +unused_qualifications = "warn" + +[dependencies] +# Dynamo crates — pinned by SHA. Bumps are manual PRs. +dynamo-protocols = { git = "https://github.com/ai-dynamo/dynamo", rev = "1efdd4dcb901caeae636131321094090d252c8d6" } +dynamo-tokenizers = { git = "https://github.com/ai-dynamo/dynamo", rev = "1efdd4dcb901caeae636131321094090d252c8d6" } +dynamo-parsers = { git = "https://github.com/ai-dynamo/dynamo", rev = "1efdd4dcb901caeae636131321094090d252c8d6" } + +# Async runtime + http +tokio = { version = "1.42", features = ["full"] } +axum = { version = "0.8", features = ["macros", "tracing"] } +tower = { version = "0.5", features = ["full"] } +tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "request-id"] } +reqwest = { version = "0.12", features = ["stream", "json", "rustls-tls"], default-features = false } + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +# `humantime-serde` lets `WorkerConfig.request_timeout` accept human-readable +# durations like `"60s"` / `"500ms"` / `"2m"` in YAML / TOML, rather than +# forcing operators to write raw milliseconds. +humantime-serde = "1" + +# Utilities +anyhow = "1" +thiserror = "2" +clap = { version = "4", features = ["derive", "env"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +futures = "0.3" +bytes = "1" +rand = "0.8" +tokio-stream = "0.1" +dashmap = "6" +serde_yaml = "0.9" +toml = "0.8" +kube = { version = "0.96", features = ["runtime", "derive"] } +k8s-openapi = { version = "0.23", features = ["v1_31"] } +tokio-util = "0.7" +uuid = { version = "1", features = ["v4"] } + +# KV-event subsystem — msgpack-encoded events over ZMQ and sha256-based +# block hashing matching SGLang's `radix_cache`. Wire format authority is +# `python/sglang/srt/disaggregation/kv_events.py`. +parking_lot = "0.12" +rmp-serde = "1" +sha2 = "0.10" +url = "2" +zeromq = { version = "0.6", default-features = false, features = ["tokio-runtime", "tcp-transport"] } + +[dev-dependencies] +dirs = "5" +http-body-util = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tempfile = "3" +tower = { version = "0.5", features = ["util"] } +tokio = { version = "1.42", features = ["test-util"] } +# Low-level msgpack encoder used to hand-construct wire bytes in +# kv_events golden-bytes tests (decode-only path uses rmp-serde). +rmp = "0.8" +# Criterion benches that mirror the SMG `radix_tree_benchmark` + +# `manual_policy_benchmark` so routing-decision latency can be compared +# apples-to-apples against the gateway being deprecated. +criterion = { version = "0.5", features = ["html_reports"] } +rand = "0.8" + +[[test]] +name = "component" +path = "tests/component/main.rs" + +[[test]] +name = "proxy" +path = "tests/proxy/main.rs" + +[[bench]] +name = "tree_lookup" +harness = false +path = "benches/tree_lookup.rs" + +[[bench]] +name = "policy_select" +harness = false +path = "benches/policy_select.rs" diff --git a/experimental/sgl-router/README.md b/experimental/sgl-router/README.md new file mode 100644 index 000000000000..0231d7952fd0 --- /dev/null +++ b/experimental/sgl-router/README.md @@ -0,0 +1,20 @@ +# sgl-router + +Slim, KV-aware, OpenAI-compatible router for SGLang workers. + +**Status:** functional single-worker HTTP proxy. Exposes `/v1/tokenize`, +`/v1/detokenize`, `/v1/models`, `/v1/chat/completions` (buffered and SSE), +plus `/healthz` / `/readyz`. Forwards to one configured worker via reqwest; +parity-tested against `transformers.AutoTokenizer`. Multi-worker routing, +service discovery, and observability still pending. + +## Building + +```bash +cd experimental/sgl-router +cargo build --release +``` + +## License + +Apache-2.0. diff --git a/experimental/sgl-router/benches/policy_select.rs b/experimental/sgl-router/benches/policy_select.rs new file mode 100644 index 000000000000..6b0e97458dd9 --- /dev/null +++ b/experimental/sgl-router/benches/policy_select.rs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Policy-selection throughput microbench. +//! +//! Mirrors `sgl-model-gateway/benches/manual_policy_benchmark.rs` — +//! measures how fast the routing layer returns a worker for a given +//! request context, across the policies sgl-router actually ships +//! (round-robin, random, power-of-two-choices). The cache-aware-zmq +//! policy lives in `tree_lookup.rs`; this file targets the non-tree +//! policies' steady-state hot path. + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::power_of_two::PowerOfTwoChoicesPolicy; +use sgl_router::policies::random::RandomPolicy; +use sgl_router::policies::round_robin::RoundRobinPolicy; +use sgl_router::policies::{Policy, SelectionContext}; +use sgl_router::workers::{Worker, WorkerRegistry}; +use std::sync::Arc; + +fn workers(n: usize, model: &str) -> Vec> { + let registry = WorkerRegistry::default(); + for i in 0..n { + registry + .add(WorkerSpec { + id: WorkerId(format!("w{i}")), + url: format!("http://w{i}:30000"), + mode: WorkerMode::Plain, + model_ids: vec![ModelId(model.into())], + bootstrap_port: None, + }) + .expect("test workers are unmixed"); + } + registry.workers_for(&ModelId(model.into())) +} + +fn bench_policy(c: &mut Criterion, name: &str, policy: Arc) { + let mut group = c.benchmark_group(format!("policy_select::{name}")); + for &n in &[4usize, 16, 64, 256] { + let workers = workers(n, "tiny"); + let model = ModelId("tiny".into()); + // Same body across iterations — measures the policy's per-call + // cost rather than body-parsing overhead. + let body = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hello world"}], + })) + .unwrap(); + group.throughput(Throughput::Elements(1)); + group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { + b.iter(|| { + let ctx = SelectionContext::new(&model, Some(&body)); + let chosen = policy.select(black_box(&workers), &ctx); + black_box(chosen); + }); + }); + } + group.finish(); +} + +fn bench_round_robin(c: &mut Criterion) { + bench_policy(c, "round_robin", Arc::new(RoundRobinPolicy::new())); +} + +fn bench_random(c: &mut Criterion) { + bench_policy(c, "random", Arc::new(RandomPolicy::new())); +} + +fn bench_power_of_two(c: &mut Criterion) { + bench_policy(c, "power_of_two", Arc::new(PowerOfTwoChoicesPolicy::new())); +} + +criterion_group!(benches, bench_round_robin, bench_random, bench_power_of_two); +criterion_main!(benches); diff --git a/experimental/sgl-router/benches/tree_lookup.rs b/experimental/sgl-router/benches/tree_lookup.rs new file mode 100644 index 000000000000..892fcd02819e --- /dev/null +++ b/experimental/sgl-router/benches/tree_lookup.rs @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Cache-aware tree-lookup microbench. +//! +//! Mirrors the shape of `sgl-model-gateway/benches/radix_tree_benchmark.rs` +//! (specifically the `TokenTree` / `PositionalIndexer` paths — which serve +//! the same role as sgl-router's `HashTree`). The bench measures: +//! +//! * `insert` — populate one worker's prefix. +//! * `match_prefix` — score an incoming request against the tree. +//! +//! Output is `criterion`'s default (target/criterion/...). To run: +//! +//! cargo bench --bench tree_lookup +//! cargo bench --bench tree_lookup -- --sample-size 30 # faster +//! +//! See `BENCHMARKS.md` for the SMG↔sgl-router comparison table. + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use sgl_router::policies::kv_events::tree::{HashTree, KvWorkerId}; + +fn build_tree(num_workers: usize, blocks_per_worker: usize, seed: u64) -> HashTree { + let tree = HashTree::new(); + let mut rng = StdRng::seed_from_u64(seed); + for w in 0..num_workers { + let worker = KvWorkerId::new(format!("http://w{w}:30000"), 0); + // Each worker holds a distinct (random) prefix so the trees fan + // out — this is the realistic case for cache-aware routing. + let hashes: Vec = (0..blocks_per_worker).map(|_| rng.gen::()).collect(); + tree.insert(&worker, None, &hashes); + } + tree +} + +fn bench_insert(c: &mut Criterion) { + let mut group = c.benchmark_group("hashtree_insert"); + for &n_blocks in &[8usize, 32, 128, 512] { + group.throughput(Throughput::Elements(n_blocks as u64)); + group.bench_with_input(BenchmarkId::from_parameter(n_blocks), &n_blocks, |b, &n| { + let mut rng = StdRng::seed_from_u64(0xC0FFEE); + let hashes: Vec = (0..n).map(|_| rng.gen::()).collect(); + b.iter_batched( + HashTree::new, + |tree| { + let worker = KvWorkerId::new("http://w:30000".to_string(), 0); + tree.insert(&worker, None, black_box(&hashes)); + tree + }, + criterion::BatchSize::SmallInput, + ); + }); + } + group.finish(); +} + +fn bench_match_prefix(c: &mut Criterion) { + let mut group = c.benchmark_group("hashtree_match_prefix"); + // (workers, blocks_per_worker, query_len) cases that span the + // realistic operating window: small fleet w/ moderate prefixes, + // medium fleet w/ long prefixes, and a stress case. + let cases = [ + (4usize, 32usize, 8usize), + (16, 64, 32), + (64, 128, 64), + (128, 256, 128), + ]; + for (workers, bpw, query_len) in cases { + let label = format!("w{workers}_bpw{bpw}_q{query_len}"); + group.throughput(Throughput::Elements(query_len as u64)); + let tree = build_tree(workers, bpw, 0xDEADBEEF); + // Pull one real worker's prefix so the query has a non-trivial + // partial match — closer to the production hot path. + let mut rng = StdRng::seed_from_u64(0x12345); + let probe: Vec = (0..query_len).map(|_| rng.gen::()).collect(); + group.bench_function(label, |b| { + b.iter(|| { + let m = tree.match_prefix(None, black_box(&probe)); + black_box(m.matched_blocks) + }); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_insert, bench_match_prefix); +criterion_main!(benches); diff --git a/experimental/sgl-router/deny.toml b/experimental/sgl-router/deny.toml new file mode 100644 index 000000000000..8a5bd4ba6ee3 --- /dev/null +++ b/experimental/sgl-router/deny.toml @@ -0,0 +1,75 @@ +[graph] +all-features = true + +[advisories] +yanked = "warn" +# Unmaintained advisories are demoted to warnings: the affected crates +# (unic-*, paste, number_prefix) are all transitive through dynamo-parsers +# and have no available upgrades. They pose no security risk; revisit +# if/when dynamo-parsers feature-flags rustpython-parser off upstream. +unmaintained = "none" +# `ignore` left empty — we want to be notified of new CVEs. +ignore = [] + +[licenses] +allow = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "MIT", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Unicode-DFS-2016", + "Unicode-3.0", + "Zlib", + "CC0-1.0", + "MPL-2.0", + # Both added per the initial license review: + "NCSA", # libfuzzer-sys (transitive via rav1e) — BSD-equivalent permissive. + "CDLA-Permissive-2.0", # webpki-roots — Linux Foundation permissive license. +] +confidence-threshold = 0.93 + +# LGPL-3.0-only is accepted on a per-crate exception basis. Rationale: +# - sgl-router is Apache-2.0 and ships full source on the public sglang +# repo, so the LGPL "users must be able to relink" requirement is +# satisfied by the conventional Rust-ecosystem interpretation (anyone +# can git-clone the repo, bump a malachite version, rebuild). +# - The malachite-* crates are pure-Rust arbitrary-precision math, used +# four levels deep through dynamo-parsers → rustpython-parser → malachite-bigint. +# They are NOT on the routing hot path; dynamo-parsers is only wired +# into chat-completions for tool-call parsing. +# - Revisit if/when: (a) a regulated-enterprise customer objects, or +# (b) dynamo-parsers feature-flags rustpython-parser off upstream. +[[licenses.exceptions]] +name = "malachite" +allow = ["LGPL-3.0-only"] + +[[licenses.exceptions]] +name = "malachite-base" +allow = ["LGPL-3.0-only"] + +[[licenses.exceptions]] +name = "malachite-nz" +allow = ["LGPL-3.0-only"] + +[[licenses.exceptions]] +name = "malachite-q" +allow = ["LGPL-3.0-only"] + +[[licenses.exceptions]] +name = "malachite-bigint" +allow = ["LGPL-3.0-only"] + +[bans] +multiple-versions = "warn" +wildcards = "deny" +# Git deps (e.g. dynamo-* pinned by SHA) have no semver version req and would +# otherwise trip the wildcard check. allow-wildcard-paths exempts non-registry +# (git + path) sources so we only deny bare '*' on crates.io deps. +allow-wildcard-paths = true + +[sources] +unknown-registry = "deny" +unknown-git = "allow" # dynamo git dep pinned by SHA in Cargo.toml. +allow-git = ["https://github.com/ai-dynamo/dynamo"] diff --git a/experimental/sgl-router/rust-toolchain.toml b/experimental/sgl-router/rust-toolchain.toml new file mode 100644 index 000000000000..2739b9054bff --- /dev/null +++ b/experimental/sgl-router/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.90" +profile = "minimal" +components = ["clippy", "rustfmt"] diff --git a/experimental/sgl-router/src/config/mod.rs b/experimental/sgl-router/src/config/mod.rs new file mode 100644 index 000000000000..79f5ffb02dbb --- /dev/null +++ b/experimental/sgl-router/src/config/mod.rs @@ -0,0 +1,540 @@ +pub mod types; +pub use types::*; + +use anyhow::Context as _; +use anyhow::{anyhow, Result}; +use std::path::Path; + +impl Config { + pub fn from_path(p: &Path) -> Result { + let raw = + std::fs::read_to_string(p).with_context(|| format!("read config {}", p.display()))?; + let ext = p.extension().and_then(|s| s.to_str()).unwrap_or(""); + let cfg: Config = match ext { + "yaml" | "yml" => serde_yaml::from_str(&raw) + .map_err(|e| anyhow!("parse yaml {}: {e}", p.display()))?, + "toml" => { + toml::from_str(&raw).map_err(|e| anyhow!("parse toml {}: {e}", p.display()))? + } + other => { + return Err(anyhow!( + "unsupported config extension {other:?}; want yaml/yml/toml" + )) + } + }; + cfg.validate()?; + Ok(cfg) + } + + fn validate(&self) -> Result<()> { + // Unknown policy names are rejected by serde via `PolicyKind`'s + // `rename_all = "snake_case"`; threshold = 0 is rejected by + // `NonZeroU32`. Only fields without a type-system constraint are + // checked here. + for m in &self.models { + if m.id.is_empty() { + return Err(anyhow!("model.id must be non-empty")); + } + } + match &self.discovery.backend { + DiscoveryBackend::StaticUrls(s) => { + if s.urls.is_empty() { + return Err(anyhow!( + "discovery.static_urls.urls must be a non-empty list" + )); + } + // Validate every entry up front so typos surface at + // config-load with a precise diagnostic instead of as + // per-worker introspect failures or as two registry + // entries pointing at the same SGLang (trailing-slash + // near-duplicates). Dedupe runs against a normalized + // form (trimmed + trailing `/` stripped) so + // `"http://x:30000"` and `"http://x:30000/"` collide. + let mut seen = std::collections::HashSet::new(); + for raw in &s.urls { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(anyhow!( + "discovery.static_urls.urls contains an empty or whitespace-only entry" + )); + } + let parsed = url::Url::parse(trimmed).map_err(|e| { + anyhow!("discovery.static_urls.urls entry {raw:?} is not a valid URL: {e}") + })?; + match parsed.scheme() { + "http" | "https" => {} + other => { + return Err(anyhow!( + "discovery.static_urls.urls entry {raw:?} has unsupported scheme {other:?}; only http and https are supported" + )); + } + } + let normalized = parsed.as_str().trim_end_matches('/').to_string(); + if !seen.insert(normalized.clone()) { + return Err(anyhow!( + "discovery.static_urls.urls contains duplicate entry {raw:?} (normalized: {normalized:?})" + )); + } + } + } + DiscoveryBackend::K8s(k) => { + // Empty namespace is intentional: triggers `Api::all(client)` + // for cluster-wide EndpointSlice watch (see + // `discovery::k8s::spawn`). Only validate the selector + // combination here. + let _ = &k.namespace; + k.mode().map_err(|e| anyhow!("{e}"))?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Write `body` to a temp file with the given extension and load it + /// through `Config::from_path`. Failures still surface the offending + /// config because each call site passes its body inline. + fn load(ext: &str, body: &str) -> Result { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join(format!("c.{ext}")); + std::fs::write(&p, body).unwrap(); + Config::from_path(&p) + } + + #[test] + fn loads_minimal_yaml() { + let c = load( + "yaml", + r#" +server: + host: "0.0.0.0" + port: 8090 +models: + - id: "qwen3-0.6b" + tokenizer_path: "/tmp/qwen.json" +discovery: + backend: static_urls + static_urls: + urls: + - "http://10.0.0.1:30000" +"#, + ) + .unwrap(); + assert_eq!(c.server.port, 8090); + assert_eq!(c.models[0].id, "qwen3-0.6b"); + match &c.discovery.backend { + DiscoveryBackend::StaticUrls(s) => { + assert_eq!(s.urls, vec!["http://10.0.0.1:30000".to_string()]) + } + _ => panic!("expected static_urls backend"), + } + } + + #[test] + fn loads_minimal_toml() { + let c = load( + "toml", + r#" +[server] +host = "0.0.0.0" +port = 8090 +[[models]] +id = "qwen3-0.6b" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["http://10.0.0.1:30000"] +"#, + ) + .unwrap(); + assert_eq!(c.server.port, 8090); + match &c.discovery.backend { + DiscoveryBackend::StaticUrls(s) => { + assert_eq!(s.urls, vec!["http://10.0.0.1:30000".to_string()]) + } + _ => panic!("expected static_urls backend"), + } + } + + #[test] + fn rejects_missing_discovery_section() { + let err = load( + "yaml", + "server:\n host: \"0.0.0.0\"\n port: 8090\nmodels: []\n", + ) + .unwrap_err(); + let msg = err.to_string().to_lowercase(); + assert!( + msg.contains("discovery") || msg.contains("missing"), + "got: {err}" + ); + } + + #[test] + fn rejects_unknown_extension() { + let err = load("txt", "").unwrap_err(); + assert!(err.to_string().contains("yaml") && err.to_string().contains("toml")); + } + + #[test] + fn loads_static_urls_discovery() { + let c = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "qwen3-0.6b" +tokenizer_path = "/tmp/qwen.json" +policy = "round_robin" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["http://10.0.0.1:30000", "http://10.0.0.2:30000"] +"#, + ) + .unwrap(); + match &c.discovery.backend { + DiscoveryBackend::StaticUrls(s) => { + assert_eq!( + s.urls, + vec![ + "http://10.0.0.1:30000".to_string(), + "http://10.0.0.2:30000".to_string(), + ], + ); + } + _ => panic!("expected static_urls backend"), + } + assert_eq!(c.models[0].policy, PolicyKind::RoundRobin); + } + + #[test] + fn rejects_static_urls_with_empty_list() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "m" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = [] +"#, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("non-empty"), "got: {err}"); + } + + #[test] + fn rejects_static_urls_with_duplicate_entry() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "m" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["http://x:30000", "http://x:30000"] +"#, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("duplicate"), "got: {err}"); + } + + #[test] + fn rejects_static_urls_with_empty_entry() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "m" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["http://x:30000", ""] +"#, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("empty"), "got: {err}"); + } + + /// Whitespace-only entries are user typos that previously slipped + /// through `is_empty()` checks and surfaced as "introspect against + /// ` /server_info` failed" at runtime. Catch at load. + #[test] + fn rejects_static_urls_with_whitespace_only_entry() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "m" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["http://x:30000", " "] +"#, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("whitespace"), "got: {err}"); + } + + /// `"10.0.0.1:30000"` (missing scheme) used to pass validation; the + /// scheme/`http://` would only fail (or worse, silently degrade + /// because of the `parse_bootstrap_host` localhost fallback) at + /// introspect time. Reject at load. + #[test] + fn rejects_static_urls_with_schemeless_entry() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "m" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["10.0.0.1:30000"] +"#, + ) + .unwrap_err() + .to_string(); + assert!( + err.contains("not a valid URL") || err.contains("unsupported scheme"), + "got: {err}" + ); + } + + /// Non-http(s) schemes are rejected. The router speaks HTTP to + /// workers; a `tcp://` or `ws://` entry is almost certainly an + /// operator typo. + #[test] + fn rejects_static_urls_with_non_http_scheme() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "m" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["ws://x:30000"] +"#, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("unsupported scheme"), "got: {err}"); + } + + /// Trailing-slash near-duplicates collide in the registry but used + /// to pass byte-equality dedupe. Normalize before checking so two + /// pointers at the same SGLang surface as a config error. + #[test] + fn rejects_static_urls_with_trailing_slash_near_duplicate() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "m" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["http://x:30000", "http://x:30000/"] +"#, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("duplicate"), "got: {err}"); + } + + #[test] + fn loads_k8s_discovery() { + let c = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "qwen3-0.6b" +tokenizer_path = "/tmp/qwen.json" +policy = "round_robin" +[discovery] +backend = "k8s" +[discovery.k8s] +namespace = "default" +label_selector = "app=sglang" +"#, + ) + .unwrap(); + match &c.discovery.backend { + DiscoveryBackend::K8s(k) => { + assert_eq!(k.namespace, "default"); + assert_eq!(k.label_selector.as_deref(), Some("app=sglang")); + assert!(k.prefill_selector.is_none()); + assert!(k.decode_selector.is_none()); + } + _ => panic!("expected k8s backend"), + } + } + + /// K8s PD selectors drive slice-classification only; per-worker + /// bootstrap_port comes from `/server_info` post-discovery + /// (`crate::workers::introspect`). This test pins the wire-shape; + /// the selector grammar itself is covered in `types.rs`. + #[test] + fn loads_k8s_pd_discovery_with_prefill_and_decode_selectors() { + let c = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "qwen3-0.6b" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "k8s" +[discovery.k8s] +namespace = "default" +prefill_selector = "app=sglang,role=prefill" +decode_selector = "app=sglang,role=decode" +"#, + ) + .expect("k8s PD config must load"); + match &c.discovery.backend { + DiscoveryBackend::K8s(k) => { + assert_eq!(k.namespace, "default"); + assert_eq!( + k.prefill_selector.as_deref(), + Some("app=sglang,role=prefill") + ); + assert_eq!(k.decode_selector.as_deref(), Some("app=sglang,role=decode")); + assert!(k.label_selector.is_none()); + } + _ => panic!("expected k8s backend"), + } + } + + #[test] + fn rejects_k8s_config_with_no_selector() { + let err = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "qwen" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "k8s" +[discovery.k8s] +namespace = "default" +"#, + ) + .unwrap_err(); + // Pin the specific variant: `ConfigError::NoSelector` ("none were + // set"). A bare `contains("selector")` would also pass for + // EmptyPdSelector / PartialPdSelectors / IdenticalPdSelectors / + // UnsupportedSelectorGrammar — variants that have semantically + // different error wording but all mention "selector". A future + // regression that returned, say, `PartialPdSelectors` for the + // all-None input would be caught here. + let msg = err.to_string().to_lowercase(); + assert!( + msg.contains("none were set"), + "expected NoSelector wording (\"none were set\"); got: {err}", + ); + } + + // Direct `K8sDiscoveryConfig::mode()` unit tests live alongside the + // type in `src/config/types.rs::k8s_discovery_config_tests`. + // The tests in this module exercise the `Config::from_path` ↔ K8s + // selector wiring, not the selector grammar itself. + + #[test] + fn rejects_unknown_policy_name() { + let err = load( + "yaml", + " +server: + host: 0.0.0.0 + port: 8090 +discovery: + backend: static_urls + static_urls: + urls: + - http://x:30000 +models: + - id: qwen + tokenizer_path: /tmp/qwen.json + policy: bogus_policy +", + ) + .unwrap_err(); + let msg = err.to_string().to_lowercase(); + assert!( + msg.contains("bogus_policy") || msg.contains("policy"), + "got: {err}" + ); + } + + #[test] + fn defaults_policy_to_round_robin() { + let c = load( + "toml", + r#" +[server] +host = "127.0.0.1" +port = 8090 +[[models]] +id = "qwen" +tokenizer_path = "/tmp/qwen.json" +[discovery] +backend = "static_urls" +[discovery.static_urls] +urls = ["http://x:30000"] +"#, + ) + .unwrap(); + assert_eq!(c.models[0].policy, PolicyKind::RoundRobin); + } +} diff --git a/experimental/sgl-router/src/config/types.rs b/experimental/sgl-router/src/config/types.rs new file mode 100644 index 000000000000..0b55df66c2cb --- /dev/null +++ b/experimental/sgl-router/src/config/types.rs @@ -0,0 +1,919 @@ +use serde::{Deserialize, Serialize}; +use std::num::NonZeroU32; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Config { + pub server: ServerConfig, + #[serde(default)] + pub observability: ObservabilityConfig, + pub models: Vec, + pub discovery: DiscoveryConfig, + #[serde(default)] + pub proxy: ProxyConfig, + #[serde(default)] + pub active_load: ActiveLoadConfig, +} + +/// Outbound proxy tuning. Default mirrors SGLang's typical prefill / +/// decode latency budget; e2e tests lower it so per-request failures +/// trip the circuit breaker within the test's wall-time. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct ProxyConfig { + /// Maximum time to wait for a single upstream HTTP request to + /// return headers + body. Default 300 s. The circuit breaker + /// records a failure when this fires. + #[serde(default = "default_proxy_request_timeout_secs")] + pub request_timeout_secs: u64, +} + +fn default_proxy_request_timeout_secs() -> u64 { + 300 +} + +impl Default for ProxyConfig { + fn default() -> Self { + Self { + request_timeout_secs: default_proxy_request_timeout_secs(), + } + } +} + +/// Active-load (per-request) tracking. Production default (10 min) +/// sits above `proxy.request_timeout_secs` so the proxy timeout is the +/// one users hit first for normal slow upstreams; tests lower it to +/// let the janitor fire within their wall-time budget. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct ActiveLoadConfig { + /// How long a request entry can live in the registry before the + /// janitor fires its `cancel_token` and the chat handler returns + /// 504 `stale_request_expired`. Default 600 s. + #[serde(default = "default_stale_request_timeout_secs")] + pub stale_request_timeout_secs: u64, +} + +fn default_stale_request_timeout_secs() -> u64 { + 600 +} + +impl Default for ActiveLoadConfig { + fn default() -> Self { + Self { + stale_request_timeout_secs: default_stale_request_timeout_secs(), + } + } +} + +/// Routing policy selector — the enum form lets serde reject unknown +/// values at deserialization time and removes the runtime string match in +/// the policy factory. +/// +/// Serialised as `"round_robin"` / `"random"` / `"power_of_two"` / +/// `"cache_aware_zmq"`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PolicyKind { + #[default] + RoundRobin, + Random, + PowerOfTwo, + /// Cache-aware routing fed by SGLang's ZMQ KV-cache event publisher. + /// Requires the model to have a tokenizer loaded; cache_aware tuning + /// lives on `ModelConfig::cache_aware`. + CacheAwareZmq, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServerConfig { + pub host: String, + pub port: u16, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ObservabilityConfig { + #[serde(default = "default_log_level")] + pub log_level: String, + /// Selects the tracing-subscriber output format. Serde rejects + /// unrecognized values at config-load (`"jsonl"` and similar + /// plausible typos surface as an error instead of silently + /// degrading to text), matching the discoverability pattern used + /// by `policy` and `discovery.backend`. + #[serde(default)] + pub log_format: LogFormat, +} + +/// `text` for human-readable dev output, `json` for one-line-per-record +/// JSON suitable for k8s log aggregators (fluent-bit / vector / Loki). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum LogFormat { + #[default] + Text, + Json, +} + +fn default_log_level() -> String { + "info".to_string() +} + +impl Default for ObservabilityConfig { + fn default() -> Self { + Self { + log_level: default_log_level(), + log_format: LogFormat::default(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelConfig { + pub id: String, + pub tokenizer_path: String, + #[serde(default)] + pub policy: PolicyKind, + #[serde(default)] + pub circuit_breaker: Option, + /// Tuning for the cache-aware ZMQ policy. Ignored unless + /// `policy = "cache_aware_zmq"`. `None` falls back to defaults at + /// policy construction time. + #[serde(default)] + pub cache_aware: Option, +} + +/// Per-model cache-aware-ZMQ tuning. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct CacheAwareConfig { + /// Lower bound on `matched_blocks / total_blocks` for the tree match + /// to win the selection. Below this, the policy falls back to + /// min-load. Default 0.5 — a half-cached prompt is still a strong + /// signal but not so weak that random hash collisions could trigger + /// affinity to an arbitrary worker. + #[serde(default = "default_cache_threshold")] + pub cache_threshold: f32, + /// Absolute load spread (`max - min`) above which the cache check is + /// skipped in favour of min-load. Default 32 — picked to dominate + /// over typical batch-of-8 effect. + #[serde(default = "default_balance_abs")] + pub balance_abs_threshold: usize, + /// Multiplicative load spread (`max > min * balance_rel_threshold`) + /// that the absolute check is gated on. Default 1.1 — 10 % relative + /// difference triggers re-balancing. + #[serde(default = "default_balance_rel")] + pub balance_rel_threshold: f32, +} + +impl Default for CacheAwareConfig { + fn default() -> Self { + Self { + cache_threshold: default_cache_threshold(), + balance_abs_threshold: default_balance_abs(), + balance_rel_threshold: default_balance_rel(), + } + } +} + +fn default_cache_threshold() -> f32 { + 0.5 +} +fn default_balance_abs() -> usize { + 32 +} +fn default_balance_rel() -> f32 { + 1.1 +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircuitBreakerConfig { + /// Consecutive failures required before the breaker opens. Encoded + /// as `NonZeroU32` so a config setting `threshold = 0` (which would + /// open the breaker before any failure) is rejected at deserialization + /// rather than silently behaving as "always open". + #[serde(default = "default_cb_threshold")] + pub threshold: NonZeroU32, + #[serde(default = "default_cb_cool_down")] + pub cool_down_secs: u64, +} + +fn default_cb_threshold() -> NonZeroU32 { + NonZeroU32::new(3).unwrap() +} +fn default_cb_cool_down() -> u64 { + 30 +} + +/// Config-level discovery section. Deserialized from: +/// +/// TOML: +/// ```toml +/// [discovery] +/// backend = "static_urls" +/// [discovery.static_urls] +/// urls = ["http://10.0.0.1:30000", "http://10.0.0.2:30000"] +/// ``` +/// +/// YAML: +/// ```yaml +/// discovery: +/// backend: static_urls +/// static_urls: +/// urls: +/// - http://10.0.0.1:30000 +/// - http://10.0.0.2:30000 +/// ``` +/// +/// The custom `Deserialize` impl on [`DiscoveryConfig`] converts the +/// raw fields into the resolved `DiscoveryBackend` enum via `try_from`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DiscoveryConfigRaw { + pub backend: String, + pub static_urls: Option, + pub k8s: Option, +} + +/// Post-validation discovery config with a resolved `DiscoveryBackend` enum. +/// Constructed by `Config::from_path` after `validate()`. +#[derive(Debug, Clone)] +pub struct DiscoveryConfig { + pub backend: DiscoveryBackend, +} + +impl<'de> Deserialize<'de> for DiscoveryConfig { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = DiscoveryConfigRaw::deserialize(deserializer)?; + raw.try_into().map_err(serde::de::Error::custom) + } +} + +impl Serialize for DiscoveryConfig { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let raw: DiscoveryConfigRaw = self.clone().into(); + raw.serialize(serializer) + } +} + +impl TryFrom for DiscoveryConfig { + type Error = String; + + fn try_from(raw: DiscoveryConfigRaw) -> Result { + let backend = match raw.backend.as_str() { + "static_urls" => { + let s = raw.static_urls.ok_or( + "discovery.backend = \"static_urls\" requires [discovery.static_urls] section", + )?; + DiscoveryBackend::StaticUrls(s) + } + "k8s" => { + let k = raw + .k8s + .ok_or("discovery.backend = \"k8s\" requires [discovery.k8s] section")?; + DiscoveryBackend::K8s(k) + } + other => { + return Err(format!( + "unknown discovery.backend = {other:?}; valid: \"static_urls\", \"k8s\"" + )) + } + }; + Ok(DiscoveryConfig { backend }) + } +} + +impl From for DiscoveryConfigRaw { + fn from(cfg: DiscoveryConfig) -> Self { + match cfg.backend { + DiscoveryBackend::StaticUrls(s) => DiscoveryConfigRaw { + backend: "static_urls".to_string(), + static_urls: Some(s), + k8s: None, + }, + DiscoveryBackend::K8s(k) => DiscoveryConfigRaw { + backend: "k8s".to_string(), + static_urls: None, + k8s: Some(k), + }, + } + } +} + +#[derive(Debug, Clone)] +pub enum DiscoveryBackend { + StaticUrls(StaticUrlsDiscoveryConfig), + K8s(K8sDiscoveryConfig), +} + +/// Fixed list of worker URLs. Each URL is registered once at startup; +/// `mode`, `model_ids`, and `bootstrap_port` are resolved per-worker +/// from `/server_info` (see [`crate::workers::introspect`]). +/// +/// No file watcher, no hot-reload: topology change requires a restart. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StaticUrlsDiscoveryConfig { + pub urls: Vec, +} + +/// Configuration for the Kubernetes `EndpointSlice` discovery backend. +/// +/// Two operating modes, distinguished by which selector fields are set: +/// +/// 1. **Plain** — all matched workers share the same role: +/// ```toml +/// [discovery.k8s] +/// namespace = "default" +/// label_selector = "app=sglang" +/// ``` +/// +/// 2. **PD disaggregation** — prefill and decode workers are separated by +/// different selectors: +/// ```toml +/// [discovery.k8s] +/// namespace = "default" +/// prefill_selector = "app=sglang,role=prefill" +/// decode_selector = "app=sglang,role=decode" +/// ``` +/// +/// In PD mode, the selectors drive **slice-classification** (which +/// EndpointSlices feed the prefill pool vs the decode pool). The actual +/// `WorkerMode` and `bootstrap_port` for each worker are filled in by +/// the worker manager from each worker's `/server_info` introspection, +/// so PD works without any pod-level annotations — see +/// [`crate::workers::introspect`] for the `disaggregation_mode` and +/// `disaggregation_bootstrap_port` extraction. +/// +/// `mode()` validates the combination and returns the resolved +/// [`K8sDiscoveryMode`]; any other selector combination is rejected. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct K8sDiscoveryConfig { + pub namespace: String, + #[serde(default)] + pub label_selector: Option, + #[serde(default)] + pub prefill_selector: Option, + #[serde(default)] + pub decode_selector: Option, +} + +/// Resolved discovery mode derived from a [`K8sDiscoveryConfig`]. +/// +/// The discovery backend uses this to: +/// * pick the server-side `LIST` label selector (Plain: the single selector; +/// PD: empty, with classification done client-side per slice), and +/// * assign each `EndpointSlice` a [`crate::discovery::WorkerMode`] in +/// `extract_workers`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum K8sDiscoveryMode { + /// One global label selector; every matched EndpointSlice becomes a + /// `WorkerMode::Plain` worker. + Plain { label_selector: String }, + /// Two label selectors; an EndpointSlice's labels are matched against + /// each to classify it as `WorkerMode::Prefill` or `WorkerMode::Decode`. + PdDisaggregation { + prefill_selector: String, + decode_selector: String, + }, +} + +/// Error returned by [`K8sDiscoveryConfig::mode`] when the selector +/// combination is invalid. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("discovery.k8s requires either `label_selector` (plain) or both `prefill_selector` and `decode_selector` (PD); none were set")] + NoSelector, + #[error("discovery.k8s: `label_selector` (plain) and `prefill_selector`/`decode_selector` (PD) are mutually exclusive — set one or the other, not both")] + MixedModes, + #[error("discovery.k8s: PD mode requires BOTH `prefill_selector` and `decode_selector`")] + PartialPdSelectors, + #[error( + "discovery.k8s: {selector}_selector `{value}` uses unsupported syntax — \ + only equality terms (`key=value` or `key==value`) joined by `,` are accepted. \ + Set-based operators (`in`, `notin`), presence tests, and `!=` silently match \ + zero endpoints at runtime and are rejected at config-load time." + )] + UnsupportedSelectorGrammar { + selector: &'static str, + value: String, + }, + #[error( + "discovery.k8s: PD `{selector}_selector` is empty (or only whitespace/commas) — \ + it would match every EndpointSlice, and since classify_mode checks prefill before \ + decode, the opposite role's pool would stay empty. Set non-empty equality terms \ + distinguishing the two roles." + )] + EmptyPdSelector { selector: &'static str }, + #[error( + "discovery.k8s: `prefill_selector` and `decode_selector` resolve to the same set \ + of equality terms — classify_mode would tag every matching slice as Prefill and \ + leave the decode pool empty. The two selectors must differ." + )] + IdenticalPdSelectors, +} + +/// Returns `true` when `selector` has zero non-empty terms after +/// trimming and splitting on `,`. `labels_match_selector` then returns +/// `true` for every label set, which is the "matches everything" +/// degenerate case PD mode must reject. +fn is_selector_empty(selector: &str) -> bool { + selector.split(',').all(|t| t.trim().is_empty()) +} + +/// Canonicalize a comma-separated equality selector to a sorted list of +/// parsed `(key, value)` tuples. Comparison happens at the parsed-term +/// level — *not* the raw string level — because `labels_match_selector` +/// already strips whitespace and treats `key=value` and `key==value` as +/// the same equality test. Comparing raw strings would let +/// `"app=sglang"` vs `"app==sglang"` (and `"app = sglang"` vs +/// `"app=sglang"`) past the identical-selector check, even though +/// `classify_mode` would treat them identically at runtime — exactly +/// the silent decode-pool-empty failure mode this check exists to +/// prevent. +/// +/// Returns an empty `Vec` for selectors with no parseable terms +/// (whitespace-only, comma-only, or any term that doesn't match the +/// `key=value` / `key==value` grammar). Callers must run +/// [`is_equality_selector`] before this to surface malformed +/// selectors as `UnsupportedSelectorGrammar`. +fn canonical_selector(selector: &str) -> Vec<(String, String)> { + let mut terms: Vec<(String, String)> = selector + .split(',') + .filter_map(|raw| { + let term = raw.trim(); + if term.is_empty() { + return None; + } + // Mirror `labels_match_selector`: prefer the `==` alias so a + // term like `key==value` parses to `(key, value)` instead of + // `(key, =value)`. + let (k, v) = term.split_once("==").or_else(|| term.split_once('='))?; + Some((k.trim().to_string(), v.trim().to_string())) + }) + .collect(); + terms.sort(); + terms +} + +/// Returns `true` when `selector` parses as a comma-separated equality +/// selector — every term has the shape `key=value` or `key==value`. +/// See [`ConfigError::UnsupportedSelectorGrammar`] for rationale. +fn is_equality_selector(selector: &str) -> bool { + for term in selector.split(',') { + let term = term.trim(); + if term.is_empty() { + // Treat lone trailing commas / whitespace as fine; the runtime + // splitter ignores empty terms. + continue; + } + if let Some((k, _)) = term.split_once("==") { + if k.trim().is_empty() { + return false; + } + continue; + } + if let Some((k, _value)) = term.split_once('=') { + // Reject `!=` (rendered as `key!` + `=value` by split_once). + // Empty value is legal in K8s — `label_selector = "tier="` + // matches pods with `tier=""` — so we don't constrain it. + if k.trim().is_empty() || k.trim().ends_with('!') { + return false; + } + continue; + } + // No `=` at all → set-based operator, presence test, or garbage. + return false; + } + true +} + +impl K8sDiscoveryConfig { + /// Validate the selector combination and return the resolved mode. + pub fn mode(&self) -> Result { + let plain = self.label_selector.as_deref(); + let prefill = self.prefill_selector.as_deref(); + let decode = self.decode_selector.as_deref(); + + match (plain, prefill, decode) { + (Some(label), None, None) => { + // Plain mode pushes `label` to the K8s API as the + // server-side `labelSelector` of the EndpointSlice + // watcher (`watcher::Config::default().labels(&label)` + // in `discovery::k8s::spawn`). K8s itself parses the + // full label-selector grammar — equality, set-based + // (`in` / `notin`), presence (`key` / `!key`), and + // `!=` — and rejects malformed selectors at + // watch-start time. So at config-load we don't + // grammar-check `label` and let the K8s API be the + // syntax authority (README.md:25 and the multi-model + // e2e in tests/e2e/k8s_integration/test_multi_model.py + // depend on this). PD mode, in contrast, evaluates + // selectors client-side via `labels_match_selector` + // which only understands equality — so PD selectors + // are still grammar-checked below. + Ok(K8sDiscoveryMode::Plain { + label_selector: label.to_string(), + }) + } + (None, Some(prefill), Some(decode)) => { + // Both selectors validated individually so the operator + // sees which one is malformed. WorkerMode + bootstrap_port + // for each prefill pod are filled in by the worker + // manager from each worker's `/server_info` — these + // selectors only drive client-side classification per + // EndpointSlice (see `classify_mode` in discovery/k8s.rs). + if !is_equality_selector(prefill) { + return Err(ConfigError::UnsupportedSelectorGrammar { + selector: "prefill", + value: prefill.to_string(), + }); + } + if !is_equality_selector(decode) { + return Err(ConfigError::UnsupportedSelectorGrammar { + selector: "decode", + value: decode.to_string(), + }); + } + // Empty PD selector matches every EndpointSlice at + // runtime; combined with classify_mode's prefill-first + // ordering, an empty selector would silently funnel all + // workers into one role. Reject up front. + if is_selector_empty(prefill) { + return Err(ConfigError::EmptyPdSelector { + selector: "prefill", + }); + } + if is_selector_empty(decode) { + return Err(ConfigError::EmptyPdSelector { selector: "decode" }); + } + // Identical selectors degrade the same way as an empty + // one: every slice matches both, prefill wins, decode + // stays empty. + if canonical_selector(prefill) == canonical_selector(decode) { + return Err(ConfigError::IdenticalPdSelectors); + } + Ok(K8sDiscoveryMode::PdDisaggregation { + prefill_selector: prefill.to_string(), + decode_selector: decode.to_string(), + }) + } + (None, None, None) => Err(ConfigError::NoSelector), + (None, Some(_), None) | (None, None, Some(_)) => Err(ConfigError::PartialPdSelectors), + (Some(_), _, _) => Err(ConfigError::MixedModes), + } + } +} + +#[cfg(test)] +mod k8s_discovery_config_tests { + use super::*; + + fn cfg(plain: Option<&str>, prefill: Option<&str>, decode: Option<&str>) -> K8sDiscoveryConfig { + K8sDiscoveryConfig { + namespace: "ns".to_string(), + label_selector: plain.map(str::to_string), + prefill_selector: prefill.map(str::to_string), + decode_selector: decode.map(str::to_string), + } + } + + #[test] + fn mode_constructs_pd_disaggregation_from_prefill_and_decode_selectors() { + // K8s PD now works without per-pod annotations: each worker's + // `/server_info` carries `disaggregation_bootstrap_port`, and the + // worker manager applies it post-discovery. The K8s config layer's + // job is just to validate the selector combination. + let m = cfg(None, Some("app=sglang,role=p"), Some("app=sglang,role=d")) + .mode() + .expect("PD mode is now valid"); + assert_eq!( + m, + K8sDiscoveryMode::PdDisaggregation { + prefill_selector: "app=sglang,role=p".to_string(), + decode_selector: "app=sglang,role=d".to_string(), + } + ); + } + + #[test] + fn mode_pd_rejects_set_based_prefill_selector() { + // Both PD selectors get the same equality-only grammar check as + // the plain label_selector. A set-based prefill selector would + // silently match zero pods at runtime → fail-fast at load. + let err = cfg(None, Some("app in (sglang, vllm)"), Some("app=sglang")) + .mode() + .unwrap_err(); + assert!( + matches!( + err, + ConfigError::UnsupportedSelectorGrammar { + selector: "prefill", + .. + }, + ), + "expected UnsupportedSelectorGrammar(prefill), got {err:?}", + ); + } + + #[test] + fn mode_pd_rejects_set_based_decode_selector() { + let err = cfg(None, Some("app=sglang"), Some("app in (sglang, vllm)")) + .mode() + .unwrap_err(); + assert!( + matches!( + err, + ConfigError::UnsupportedSelectorGrammar { + selector: "decode", + .. + }, + ), + "expected UnsupportedSelectorGrammar(decode), got {err:?}", + ); + } + + #[test] + fn mode_accepts_plain_with_equality_selector() { + let m = cfg(Some("app=sglang"), None, None).mode().unwrap(); + assert_eq!( + m, + K8sDiscoveryMode::Plain { + label_selector: "app=sglang".to_string() + } + ); + } + + /// Plain mode pushes its selector to the K8s API server-side + /// (`watcher::Config::default().labels(&selector)` in + /// `discovery::k8s::spawn`), so the full K8s label-selector grammar + /// — including set-based operators — is supported. README.md:25 + /// advertises this, and `tests/e2e/k8s_integration/test_multi_model.py` + /// relies on it (`label_selector = "app in (sglang,sglang-small)"`). + /// Rejecting set-based selectors at config-load broke the documented + /// multi-model k8s path. + #[test] + fn mode_accepts_set_based_selector_in_plain_mode() { + let m = cfg(Some("app in (sglang,sglang-small)"), None, None) + .mode() + .expect("plain mode must accept set-based selectors"); + assert_eq!( + m, + K8sDiscoveryMode::Plain { + label_selector: "app in (sglang,sglang-small)".to_string(), + } + ); + } + + /// `notin`, presence (`key`), absence (`!key`), and inequality (`!=`) + /// are all valid K8s server-side selector grammar — plain mode must + /// pass them through. + #[test] + fn mode_accepts_other_set_based_forms_in_plain_mode() { + for raw in [ + "app notin (vllm,trtllm)", + "tier", + "!deprecated", + "tier!=canary", + ] { + let m = cfg(Some(raw), None, None) + .mode() + .unwrap_or_else(|e| panic!("plain mode must accept `{raw}`, got {e:?}")); + assert_eq!( + m, + K8sDiscoveryMode::Plain { + label_selector: raw.to_string(), + }, + "selector roundtrip mismatch for `{raw}`", + ); + } + } + + /// PD mode evaluates selectors *client-side* via + /// `labels_match_selector`, which only handles equality. A set-based + /// PD selector would silently match zero pods → fail-fast at load. + /// Pins the plain-server-side / PD-client-side asymmetry: relaxing + /// the grammar check for plain (see `mode_accepts_set_based_*` + /// above) must not accidentally relax it for PD selectors. Uses + /// `notin` so this test covers a different set-based form than + /// `mode_pd_rejects_set_based_prefill_selector` (which uses `in`) + /// — both must keep failing. + #[test] + fn mode_pd_rejects_notin_prefill_selector() { + let err = cfg(None, Some("app notin (vllm, trtllm)"), Some("app=sglang")) + .mode() + .unwrap_err(); + assert!( + matches!( + err, + ConfigError::UnsupportedSelectorGrammar { + selector: "prefill", + .. + }, + ), + "expected UnsupportedSelectorGrammar(prefill), got {err:?}", + ); + } + + #[test] + fn mode_accepts_comma_separated_equality_terms() { + // The canonical Plain-mode selector form: `key1=v1,key2=v2`. + let m = cfg(Some("app=sglang,zone=us-east"), None, None) + .mode() + .unwrap(); + assert_eq!( + m, + K8sDiscoveryMode::Plain { + label_selector: "app=sglang,zone=us-east".to_string() + } + ); + } + + #[test] + fn mode_rejects_when_no_selector_is_set() { + let err = cfg(None, None, None).mode().unwrap_err(); + assert!(matches!(err, ConfigError::NoSelector), "got {err:?}"); + } + + #[test] + fn mode_rejects_mixed_plain_and_pd_selectors() { + let err = cfg( + Some("app=sglang"), + Some("role=prefill"), + Some("role=decode"), + ) + .mode() + .unwrap_err(); + assert!(matches!(err, ConfigError::MixedModes), "got {err:?}"); + } + + #[test] + fn mode_rejects_partial_pd_selectors() { + let err = cfg(None, Some("role=prefill"), None).mode().unwrap_err(); + assert!( + matches!(err, ConfigError::PartialPdSelectors), + "got {err:?}" + ); + let err = cfg(None, None, Some("role=decode")).mode().unwrap_err(); + assert!( + matches!(err, ConfigError::PartialPdSelectors), + "got {err:?}" + ); + } + + /// Empty plain `label_selector` is valid — matches every + /// EndpointSlice in the namespace (documented K8s behavior; the + /// operator opts in by setting plain mode at all). + #[test] + fn mode_accepts_empty_plain_label_selector() { + let m = cfg(Some(""), None, None).mode().unwrap(); + assert_eq!( + m, + K8sDiscoveryMode::Plain { + label_selector: String::new() + } + ); + } + + /// PD mode is the *opposite* of plain: an empty selector would match + /// every EndpointSlice, and since `classify_mode` checks prefill + /// before decode, an empty `prefill_selector` would classify + /// everything as Prefill — decode pool stays empty and the resolver + /// surfaces the wrong `no_decode_workers_available` error. Fail-fast + /// at config load. + #[test] + fn mode_pd_rejects_empty_prefill_selector() { + let err = cfg(None, Some(""), Some("role=decode")).mode().unwrap_err(); + assert!( + matches!( + err, + ConfigError::EmptyPdSelector { + selector: "prefill" + }, + ), + "expected EmptyPdSelector(prefill), got {err:?}", + ); + } + + #[test] + fn mode_pd_rejects_empty_decode_selector() { + let err = cfg(None, Some("role=prefill"), Some("")) + .mode() + .unwrap_err(); + assert!( + matches!(err, ConfigError::EmptyPdSelector { selector: "decode" },), + "expected EmptyPdSelector(decode), got {err:?}", + ); + } + + /// Whitespace-only / comma-only PD selector parses to zero terms in + /// `labels_match_selector` and matches every slice at runtime — same + /// failure mode as a literal empty string. + #[test] + fn mode_pd_rejects_whitespace_only_prefill_selector() { + let err = cfg(None, Some(" , "), Some("role=decode")) + .mode() + .unwrap_err(); + assert!( + matches!( + err, + ConfigError::EmptyPdSelector { + selector: "prefill" + }, + ), + "expected EmptyPdSelector(prefill), got {err:?}", + ); + } + + /// Identical prefill and decode selectors degrade silently: every + /// slice matches both, but `classify_mode` returns `Prefill` first, + /// so the decode pool stays empty. + #[test] + fn mode_pd_rejects_identical_prefill_and_decode_selectors() { + let err = cfg(None, Some("app=sglang"), Some("app=sglang")) + .mode() + .unwrap_err(); + assert!( + matches!(err, ConfigError::IdenticalPdSelectors), + "expected IdenticalPdSelectors, got {err:?}", + ); + } + + /// Trailing whitespace must not be a loophole that bypasses the + /// identical-selector check. + #[test] + fn mode_pd_rejects_identical_selectors_under_whitespace_normalization() { + let err = cfg(None, Some("app=sglang"), Some(" app=sglang ")) + .mode() + .unwrap_err(); + assert!( + matches!(err, ConfigError::IdenticalPdSelectors), + "expected IdenticalPdSelectors, got {err:?}", + ); + } + + /// `labels_match_selector` accepts both `key=value` and `key==value` + /// for equality and parses them to the same `(key, value)` tuple. + /// Two selectors that differ only in this alias choice are runtime- + /// equivalent — they'd match the same EndpointSlices, then + /// `classify_mode`'s prefill-first ordering would funnel every slice + /// into Prefill, leaving decode empty. The check must canonicalize + /// at the term level (parsed `(key, value)` tuples), not the raw + /// string level. + #[test] + fn mode_pd_rejects_identical_selectors_under_eq_alias() { + let err = cfg(None, Some("app=sglang"), Some("app==sglang")) + .mode() + .unwrap_err(); + assert!( + matches!(err, ConfigError::IdenticalPdSelectors), + "expected IdenticalPdSelectors, got {err:?}", + ); + } + + /// Inner whitespace inside a term (`"app = sglang"`) is the same + /// label as no whitespace (`"app=sglang"`) — the runtime + /// `labels_match_selector` trims key and value independently + /// (see `key.trim()` / `expected.trim()` in `k8s.rs`). Canonical + /// form must agree. + #[test] + fn mode_pd_rejects_identical_selectors_under_inner_whitespace() { + let err = cfg(None, Some("app=sglang"), Some("app = sglang")) + .mode() + .unwrap_err(); + assert!( + matches!(err, ConfigError::IdenticalPdSelectors), + "expected IdenticalPdSelectors, got {err:?}", + ); + } + + /// Term order doesn't matter for label matching, so `"a=1,b=2"` and + /// `"b=2,a=1"` must be treated as identical. (Implied by the sort + /// in `canonical_selector`, but pinned explicitly so a future + /// "preserve user order for diagnostics" refactor can't silently + /// reintroduce the silent-failure bug.) + #[test] + fn mode_pd_rejects_identical_selectors_under_term_order_permutation() { + let err = cfg(None, Some("role=p,app=sglang"), Some("app=sglang,role=p")) + .mode() + .unwrap_err(); + assert!( + matches!(err, ConfigError::IdenticalPdSelectors), + "expected IdenticalPdSelectors, got {err:?}", + ); + } + + /// Sanity: two selectors that genuinely differ at the term level + /// must still pass validation — the canonicalizer must not be so + /// aggressive that it false-positives on legitimate PD configs. + #[test] + fn mode_pd_accepts_truly_distinct_selectors() { + let m = cfg( + None, + Some("app=sglang,role=prefill"), + Some("app=sglang,role=decode"), + ) + .mode() + .expect("distinct selectors must validate"); + assert!(matches!(m, K8sDiscoveryMode::PdDisaggregation { .. })); + } +} diff --git a/experimental/sgl-router/src/discovery/k8s.rs b/experimental/sgl-router/src/discovery/k8s.rs new file mode 100644 index 000000000000..bf2798954b90 --- /dev/null +++ b/experimental/sgl-router/src/discovery/k8s.rs @@ -0,0 +1,1056 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Kubernetes EndpointSlice discovery backend. +//! +//! Watches `EndpointSlice` resources by label selector in the configured +//! namespace, diffing against in-memory state and emitting +//! [`DiscoveryEvent`]s for the worker manager. + +use crate::config::{K8sDiscoveryConfig, K8sDiscoveryMode}; +use crate::discovery::{DiscoveryEvent, WorkerId, WorkerMode, WorkerSpec}; +use anyhow::{Context, Result}; +use futures::{Stream, StreamExt}; +use k8s_openapi::api::discovery::v1::EndpointSlice; +use kube::{api::Api, runtime::watcher, Client}; +use std::collections::{BTreeMap, HashMap}; +use tokio::sync::mpsc; + +/// Decide which [`WorkerMode`] an `EndpointSlice` should be assigned, based +/// on the configured discovery mode. +/// +/// * `Plain` mode — every slice yields `Some(WorkerMode::Plain)`. The +/// server-side label selector has already filtered to the right set. +/// * `PD` mode — the slice's labels are matched against `prefill_selector` +/// and `decode_selector` (in that order). Returns `Some(Prefill)` / +/// `Some(Decode)` on the first match, `None` if neither matches. The +/// server-side watch in PD mode is unfiltered (label selectors for the +/// two roles may not be mergeable into one Kubernetes selector), so this +/// client-side classification is the gate that drops irrelevant slices. +fn classify_mode(es: &EndpointSlice, mode: &K8sDiscoveryMode) -> Option { + match mode { + K8sDiscoveryMode::Plain { .. } => Some(WorkerMode::Plain), + K8sDiscoveryMode::PdDisaggregation { + prefill_selector, + decode_selector, + } => { + let labels = es.metadata.labels.as_ref().cloned().unwrap_or_default(); + if labels_match_selector(&labels, prefill_selector) { + Some(WorkerMode::Prefill) + } else if labels_match_selector(&labels, decode_selector) { + Some(WorkerMode::Decode) + } else { + None + } + } + } +} + +/// Match a Kubernetes label set against a comma-separated equality +/// selector like `"app=sglang,role=prefill"`. +/// +/// Supports the equality-based subset of the K8s label-selector grammar: +/// `key=value` (and the alias `key==value`). Set-based operators (`in`, +/// `not in`, presence tests) are not supported — fall back to the +/// server-side selector in plain mode if you need them. +fn labels_match_selector(labels: &BTreeMap, selector: &str) -> bool { + for term in selector.split(',') { + let term = term.trim(); + if term.is_empty() { + continue; + } + // Accept both `=` and `==` for equality. + let (key, expected) = if let Some((k, v)) = term.split_once("==") { + (k.trim(), v.trim()) + } else if let Some((k, v)) = term.split_once('=') { + (k.trim(), v.trim()) + } else { + // Set-based or presence-only term — not supported here. + return false; + }; + match labels.get(key) { + Some(v) if v == expected => {} + _ => return false, + } + } + true +} + +/// Convert an `EndpointSlice` into a list of [`WorkerSpec`]s with the +/// supplied [`WorkerMode`]. +/// +/// Skips endpoints whose `conditions.ready` is explicitly `Some(false)`. +/// Per the EndpointSlice API spec, `conditions.ready = None` (absent) means +/// the endpoint should be considered ready. +/// +/// The worker URL is `http://:` where port comes from +/// `EndpointSlice.ports[0].port`, defaulting to `30000` if absent. +/// +/// The [`WorkerId`] is `{ns}/{uid}` where `uid` is the pod's K8s UID +/// from `endpoint.target_ref.uid`. Pod UIDs are globally unique per +/// pod incarnation, so when a pod dies and a new pod gets the same IP +/// (kubelet IP reuse on a busy podCIDR), the router sees a fresh +/// `WorkerId` and emits a clean Removed→Added cycle — old breaker / +/// active-load state is shed instead of being mis-applied to the new +/// pod. For manually-created EndpointSlices that lack `target_ref` +/// (rare in real clusters but common in unit tests), falls back to +/// `{ns}/{slice_name}/{addr}:{port}`. +/// +/// `model_ids` is intentionally left empty — model membership is resolved +/// by the worker manager via `/server_info` introspection after the +/// `Added` event is emitted. +fn extract_workers(es: &EndpointSlice, mode: WorkerMode) -> Vec { + let port = es + .ports + .as_ref() + .and_then(|p| p.first()) + .and_then(|p| p.port) + .unwrap_or(30000); + + let ns = es.metadata.namespace.as_deref().unwrap_or(""); + let slice_name = es.metadata.name.as_deref().unwrap_or(""); + + let mut out = Vec::new(); + for ep in es.endpoints.iter() { + let is_ready = ep.conditions.as_ref().and_then(|c| c.ready).unwrap_or(true); + if !is_ready { + continue; + } + let pod_uid: Option<&str> = ep.target_ref.as_ref().and_then(|r| r.uid.as_deref()); + for addr in &ep.addresses { + let url = format!("http://{addr}:{port}"); + let id = match pod_uid { + Some(uid) => WorkerId(format!("{ns}/{uid}")), + None => WorkerId(format!("{ns}/{slice_name}/{addr}:{port}")), + }; + // bootstrap_port stays `None` here on purpose. The final + // `WorkerMode` and the bootstrap port are both filled in by + // the worker manager from each worker's `/server_info` + // body (`disaggregation_mode` + `disaggregation_bootstrap_port`, + // both fields on SGLang's ServerArgs and already surfaced + // via `**asdict(server_args)` in the response). EndpointSlice + // carries neither, but doesn't need to — see + // `src/workers/introspect.rs` for the extraction and + // `register_one` in `src/workers/manager.rs` for the override. + out.push(WorkerSpec { + id, + url, + mode, + model_ids: Vec::new(), + bootstrap_port: None, + }); + } + } + out +} + +/// Spawn the k8s discovery task. +/// +/// Connects to the cluster via `KUBECONFIG` / in-cluster service account, +/// then watches `EndpointSlice` resources in `cfg.namespace`. Diffs against +/// in-memory state and emits [`DiscoveryEvent`]s to `tx`. +/// +/// In plain mode the configured `label_selector` is pushed to the server +/// side; in PD mode the watch is unfiltered and per-slice classification +/// happens client-side via `classify_mode`. +/// +/// Stable per-slice key. +/// +/// Uses `metadata.uid` when present (the normal case in a real cluster), and +/// falls back to `{ns}/{name}` for slices without a UID (rare: CR shims, +/// tests, certain fake/in-memory backends). The fallback is unique per slice +/// because EndpointSlice names are unique within a namespace. +fn slice_key(es: &EndpointSlice) -> String { + if let Some(uid) = es.metadata.uid.as_deref() { + if !uid.is_empty() { + return uid.to_string(); + } + } + let ns = es.metadata.namespace.as_deref().unwrap_or(""); + let name = es.metadata.name.as_deref().unwrap_or(""); + format!("{ns}/{name}") +} + +/// Send all `Added` / `Removed` / `ModeChanged` events that bring the +/// consumer from `prev_union` to the recomputed union of `per_slice`. +/// +/// Returns `Err` on the first send failure (consumer dropped); the caller is +/// expected to exit the watcher loop. Updates `prev_union` in place to the +/// new union on success. +async fn emit_diff( + tx: &mpsc::Sender, + per_slice: &HashMap>, + prev_union: &mut HashMap, +) -> Result<(), mpsc::error::SendError> { + let union: HashMap = per_slice + .values() + .flat_map(|s| s.iter().map(|(k, v)| (k.clone(), v.clone()))) + .collect(); + + for (id, spec) in &union { + match prev_union.get(id) { + Some(prev) => { + if prev.mode != spec.mode { + tx.send(DiscoveryEvent::ModeChanged { + id: id.clone(), + mode: spec.mode, + }) + .await?; + } + if prev.url != spec.url || prev.model_ids != spec.model_ids { + tx.send(DiscoveryEvent::Removed { id: id.clone() }).await?; + tx.send(DiscoveryEvent::Added(spec.clone())).await?; + } + } + None => { + tx.send(DiscoveryEvent::Added(spec.clone())).await?; + } + } + } + + let dropped: Vec = prev_union + .keys() + .filter(|id| !union.contains_key(id)) + .cloned() + .collect(); + for id in dropped { + tx.send(DiscoveryEvent::Removed { id }).await?; + } + + *prev_union = union; + Ok(()) +} + +/// Drive the event-processing loop for a stream of `watcher::Event`s. +/// +/// Handles the full set of `kube` watcher events: +/// * `Init` / `InitApply` / `InitDone` — full-LIST resync. Objects are +/// buffered until `InitDone`, then swapped in atomically. Any slices that +/// were present before but not seen during the resync are diffed out as +/// `Removed`, which catches deletions that occurred while the watcher was +/// disconnected. +/// * `Apply` — single-object upsert into `per_slice`. +/// * `Delete` — single-object removal from `per_slice`; the diff emits +/// `Removed` for every worker that lived in that slice. +/// +/// On any watcher error the kube-runtime watcher auto-restarts and emits a +/// new `Init` cycle, so the resync logic above is what reconciles state +/// after transient errors — no separate state reset is required. +/// +/// The loop returns when the input stream ends (logged at WARN) or when the +/// consumer drops the receiving end of `tx` (logged at INFO). +async fn process_events(mut stream: S, tx: mpsc::Sender, mode: K8sDiscoveryMode) +where + S: Stream, watcher::Error>> + Unpin, +{ + let mut per_slice: HashMap> = HashMap::new(); + let mut prev_union: HashMap = HashMap::new(); + let mut init_buffer: Option>> = None; + + fn workers_for_slice( + es: &EndpointSlice, + mode: &K8sDiscoveryMode, + ) -> HashMap { + match classify_mode(es, mode) { + Some(wm) => extract_workers(es, wm) + .into_iter() + .map(|w| (w.id.clone(), w)) + .collect(), + None => HashMap::new(), + } + } + + while let Some(event) = stream.next().await { + let result = match event { + Ok(watcher::Event::Init) => { + init_buffer = Some(HashMap::new()); + Ok(()) + } + Ok(watcher::Event::InitApply(es)) => { + let key = slice_key(&es); + let workers = workers_for_slice(&es, &mode); + if let Some(buf) = init_buffer.as_mut() { + buf.insert(key, workers); + Ok(()) + } else { + // Defensive: InitApply outside an Init cycle. Treat as Apply. + per_slice.insert(key, workers); + emit_diff(&tx, &per_slice, &mut prev_union).await + } + } + Ok(watcher::Event::InitDone) => { + if let Some(buf) = init_buffer.take() { + per_slice = buf; + emit_diff(&tx, &per_slice, &mut prev_union).await + } else { + Ok(()) + } + } + Ok(watcher::Event::Apply(es)) => { + let key = slice_key(&es); + let workers = workers_for_slice(&es, &mode); + per_slice.insert(key, workers); + emit_diff(&tx, &per_slice, &mut prev_union).await + } + Ok(watcher::Event::Delete(es)) => { + let key = slice_key(&es); + per_slice.remove(&key); + emit_diff(&tx, &per_slice, &mut prev_union).await + } + Err(e) => { + tracing::warn!(error = ?e, "k8s watcher error; awaiting auto-restart"); + Ok(()) + } + }; + if result.is_err() { + tracing::info!("k8s discovery: event channel closed; exiting watcher"); + return; + } + } + tracing::warn!("k8s watcher stream ended; discovery task exiting"); +} + +/// Empty `cfg.namespace` triggers a cluster-wide watch via `Api::all(client)`. +/// `Api::namespaced(client, "")` is namespace-scoped to the empty-named +/// namespace, which is almost never what callers intend. +/// +/// State is tracked per-slice as `HashMap>`. K8s auto-shards Services with >100 endpoints and CNIs +/// often shard per AZ, so multiple `EndpointSlice` objects can exist per +/// Service; a flat state map would let each slice's event silently drop all +/// workers from sibling slices. The global union is recomputed from all +/// submaps on every event and diffed against `prev_union` to produce +/// `DiscoveryEvent`s. +/// +/// The returned `JoinHandle` runs until the channel is closed or the watcher +/// stream ends (server restart, RBAC change, etc.). +pub async fn spawn( + cfg: K8sDiscoveryConfig, + tx: mpsc::Sender, +) -> Result> { + let mode = cfg.mode().context("validate k8s discovery selectors")?; + + let client = Client::try_default() + .await + .context("kube client default config")?; + + let api: Api = if cfg.namespace.is_empty() { + Api::all(client) + } else { + Api::namespaced(client, &cfg.namespace) + }; + + // Plain mode pushes the single selector to the server side so the LIST + // is already filtered. PD mode leaves the server-side selector empty + // because the prefill/decode selectors may not be expressible as one + // K8s label-selector — classification happens client-side per slice + // via `classify_mode`. + let server_side_selector = match &mode { + K8sDiscoveryMode::Plain { label_selector } => label_selector.clone(), + K8sDiscoveryMode::PdDisaggregation { .. } => String::new(), + }; + let watcher_cfg = watcher::Config::default().labels(&server_side_selector); + + let handle = tokio::spawn(async move { + let stream = watcher(api, watcher_cfg); + tokio::pin!(stream); + process_events(stream, tx, mode).await; + }); + Ok(handle) +} + +#[cfg(test)] +mod tests { + use super::*; + use k8s_openapi::api::discovery::v1::{Endpoint, EndpointConditions, EndpointPort}; + use kube::core::ObjectMeta; + + /// Helper: build a minimal EndpointSlice with predictable metadata. + fn make_slice(addrs: &[&str], port: i32, ready: bool) -> EndpointSlice { + make_slice_full(addrs, port, ready, "testns", "test-slice", &[]) + } + + fn make_slice_ns( + addrs: &[&str], + port: i32, + ready: bool, + ns: &str, + slice_name: &str, + ) -> EndpointSlice { + make_slice_full(addrs, port, ready, ns, slice_name, &[]) + } + + fn make_slice_with_labels( + addrs: &[&str], + port: i32, + ready: bool, + labels: &[(&str, &str)], + ) -> EndpointSlice { + make_slice_full(addrs, port, ready, "testns", "test-slice", labels) + } + + fn make_slice_full( + addrs: &[&str], + port: i32, + ready: bool, + ns: &str, + slice_name: &str, + labels: &[(&str, &str)], + ) -> EndpointSlice { + let label_map: BTreeMap = labels + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + EndpointSlice { + metadata: ObjectMeta { + labels: if label_map.is_empty() { + None + } else { + Some(label_map) + }, + name: if slice_name.is_empty() { + None + } else { + Some(slice_name.into()) + }, + namespace: if ns.is_empty() { None } else { Some(ns.into()) }, + ..Default::default() + }, + address_type: "IPv4".into(), + endpoints: vec![Endpoint { + addresses: addrs.iter().map(|a| (*a).to_string()).collect(), + conditions: Some(EndpointConditions { + ready: Some(ready), + ..Default::default() + }), + ..Default::default() + }], + ports: Some(vec![EndpointPort { + port: Some(port), + ..Default::default() + }]), + } + } + + fn plain_mode() -> K8sDiscoveryMode { + K8sDiscoveryMode::Plain { + label_selector: "app=sglang".into(), + } + } + + fn pd_mode() -> K8sDiscoveryMode { + K8sDiscoveryMode::PdDisaggregation { + prefill_selector: "app=sglang,role=prefill".into(), + decode_selector: "app=sglang,role=decode".into(), + } + } + + #[test] + fn classify_mode_plain_returns_some_plain_for_any_slice() { + let s = make_slice(&["10.0.0.1"], 30000, true); + assert_eq!(classify_mode(&s, &plain_mode()), Some(WorkerMode::Plain)); + } + + #[test] + fn classify_mode_pd_returns_prefill_when_labels_match_prefill_selector() { + let s = make_slice_with_labels( + &["10.0.0.1"], + 30000, + true, + &[("app", "sglang"), ("role", "prefill")], + ); + assert_eq!(classify_mode(&s, &pd_mode()), Some(WorkerMode::Prefill)); + } + + #[test] + fn classify_mode_pd_returns_decode_when_labels_match_decode_selector() { + let s = make_slice_with_labels( + &["10.0.0.1"], + 30000, + true, + &[("app", "sglang"), ("role", "decode")], + ); + assert_eq!(classify_mode(&s, &pd_mode()), Some(WorkerMode::Decode)); + } + + #[test] + fn classify_mode_pd_returns_none_when_no_selector_matches() { + let s = make_slice_with_labels( + &["10.0.0.1"], + 30000, + true, + &[("app", "sglang"), ("role", "router")], + ); + assert!(classify_mode(&s, &pd_mode()).is_none()); + + // No labels at all => still None in PD mode (every selector term + // requires a key/value). + let s = make_slice(&["10.0.0.1"], 30000, true); + assert!(classify_mode(&s, &pd_mode()).is_none()); + } + + #[test] + fn extract_workers_emits_workers_with_supplied_mode_and_empty_model_ids() { + let s = make_slice(&["10.0.0.1"], 30000, true); + let ws = extract_workers(&s, WorkerMode::Plain); + assert_eq!(ws.len(), 1); + assert_eq!(ws[0].mode, WorkerMode::Plain); + assert_eq!(ws[0].url, "http://10.0.0.1:30000"); + assert_eq!(ws[0].id.0, "testns/test-slice/10.0.0.1:30000"); + assert!( + ws[0].model_ids.is_empty(), + "model_ids are resolved via /server_info, not at extract time" + ); + + // The mode argument flows through unchanged. + let ws = extract_workers(&s, WorkerMode::Prefill); + assert_eq!(ws[0].mode, WorkerMode::Prefill); + let ws = extract_workers(&s, WorkerMode::Decode); + assert_eq!(ws[0].mode, WorkerMode::Decode); + } + + #[test] + fn skips_not_ready_endpoints() { + let s = make_slice(&["10.0.0.1"], 30000, false); + assert!(extract_workers(&s, WorkerMode::Plain).is_empty()); + } + + /// `conditions.ready = None` must default to ready=true per EndpointSlice + /// API spec ("undefined → endpoint should be considered ready"). + #[test] + fn is_ready_none_defaults_to_ready() { + let mut s = make_slice(&["10.0.0.1"], 30000, false /* overridden below */); + s.endpoints[0].conditions.as_mut().unwrap().ready = None; + let ws = extract_workers(&s, WorkerMode::Plain); + assert_eq!( + ws.len(), + 1, + "endpoint with None ready should be treated as ready" + ); + } + + /// WorkerId must include namespace + slice name to avoid cross-namespace + /// IP collisions on overlay networks. + #[test] + fn worker_id_includes_namespace_and_slice_name() { + let mut s = make_slice(&["10.0.0.1"], 30000, true); + s.metadata.namespace = Some("prod".to_string()); + s.metadata.name = Some("svc-abc-xyz".to_string()); + let ws = extract_workers(&s, WorkerMode::Plain); + assert_eq!(ws[0].id.0, "prod/svc-abc-xyz/10.0.0.1:30000"); + } + + /// A cluster-scoped slice (no namespace metadata) must still produce a + /// valid `WorkerId` — empty-namespace prefix yields a leading slash. + #[test] + fn worker_id_handles_missing_namespace() { + // make_slice_ns with empty ns leaves metadata.namespace = None. + let s = make_slice_ns(&["10.0.0.1"], 30000, true, "", "my-slice"); + let ws = extract_workers(&s, WorkerMode::Plain); + assert!( + ws[0].id.0.contains("10.0.0.1:30000"), + "id must contain addr:port" + ); + assert!( + ws[0].id.0.starts_with('/'), + "empty ns => id starts with '/', got: {}", + ws[0].id.0 + ); + } + + /// Two slices with no `metadata.uid` must hash to distinct per-slice + /// keys; otherwise an event for slice B would overwrite slice A's state. + #[test] + fn slice_key_falls_back_to_ns_and_name_when_uid_missing() { + let a = make_slice_ns(&["10.0.0.1"], 30000, true, "ns1", "a"); + let b = make_slice_ns(&["10.0.0.2"], 30000, true, "ns1", "b"); + assert_ne!(slice_key(&a), slice_key(&b)); + assert_eq!(slice_key(&a), "ns1/a"); + } + + #[test] + fn slice_key_uses_uid_when_present() { + let mut a = make_slice_ns(&["10.0.0.1"], 30000, true, "ns1", "a"); + a.metadata.uid = Some("uid-abc".into()); + assert_eq!(slice_key(&a), "uid-abc"); + } + + fn with_uid(mut es: EndpointSlice, uid: &str) -> EndpointSlice { + es.metadata.uid = Some(uid.into()); + es + } + + /// Apply → Delete on the same slice removes its workers from the union + /// (the missing variant for `.applied_objects()` before the rewrite). + #[tokio::test] + async fn delete_event_emits_removed_for_workers_in_slice() { + let s = with_uid(make_slice_ns(&["10.0.0.1"], 30000, true, "ns", "svc"), "u1"); + let events = vec![ + Ok(watcher::Event::Apply(s.clone())), + Ok(watcher::Event::Delete(s)), + ]; + let (tx, mut rx) = mpsc::channel(16); + let stream = futures::stream::iter(events); + process_events(stream, tx, plain_mode()).await; + let mut out = Vec::new(); + while let Ok(e) = rx.try_recv() { + out.push(e); + } + assert_eq!(out.len(), 2, "{out:?}"); + assert!(matches!(out[0], DiscoveryEvent::Added(_))); + assert!(matches!(out[1], DiscoveryEvent::Removed { .. })); + } + + /// Init/InitDone replaces state atomically: any worker present before + /// the Init cycle but not seen during it is diffed out as Removed. This + /// covers the "slice deleted while watcher was disconnected" case. + #[tokio::test] + async fn init_cycle_diffs_out_unseen_slices() { + let a = with_uid(make_slice_ns(&["10.0.0.1"], 30000, true, "ns", "a"), "u-a"); + let b = with_uid(make_slice_ns(&["10.0.0.2"], 30000, true, "ns", "b"), "u-b"); + + let events = vec![ + // Initial state: both slices live. + Ok(watcher::Event::Apply(a.clone())), + Ok(watcher::Event::Apply(b.clone())), + // Watcher restart resyncs and only sees `a` (b was deleted offline). + Ok(watcher::Event::Init), + Ok(watcher::Event::InitApply(a.clone())), + Ok(watcher::Event::InitDone), + ]; + let (tx, mut rx) = mpsc::channel(16); + process_events(futures::stream::iter(events), tx, plain_mode()).await; + let mut out = Vec::new(); + while let Ok(e) = rx.try_recv() { + out.push(e); + } + let removed: Vec<_> = out + .iter() + .filter_map(|e| match e { + DiscoveryEvent::Removed { id } => Some(id.0.as_str()), + _ => None, + }) + .collect(); + assert!( + removed.contains(&"ns/b/10.0.0.2:30000"), + "init resync should diff out the deleted slice: out={out:?}" + ); + assert!( + !removed.contains(&"ns/a/10.0.0.1:30000"), + "live slice must not be removed: out={out:?}" + ); + } + + /// When the consumer drops the receiver, the watcher loop exits cleanly + /// rather than continuing to process events into the void. + #[tokio::test] + async fn watcher_exits_when_consumer_drops_receiver() { + use std::time::Duration; + let s = with_uid(make_slice_ns(&["10.0.0.1"], 30000, true, "ns", "a"), "u-a"); + // A stream that never ends — only consumer drop should stop the loop. + let events = futures::stream::iter(std::iter::repeat_with(move || { + Ok(watcher::Event::Apply(s.clone())) + })); + let (tx, rx) = mpsc::channel(1); + drop(rx); + let handle = tokio::spawn(process_events(events, tx, plain_mode())); + tokio::time::timeout(Duration::from_secs(2), handle) + .await + .expect("process_events must exit promptly when consumer drops") + .expect("task should not panic"); + } + + /// Watcher errors are logged but state is preserved; the next successful + /// event continues to diff against the pre-error union. + #[tokio::test] + async fn watcher_error_preserves_state_and_diffs_against_pre_error_union() { + let s = with_uid(make_slice_ns(&["10.0.0.1"], 30000, true, "ns", "a"), "u-a"); + let events = vec![ + Ok(watcher::Event::Apply(s.clone())), + Err(watcher::Error::NoResourceVersion), + // Same slice; nothing should be re-emitted because state survived. + Ok(watcher::Event::Apply(s)), + ]; + let (tx, mut rx) = mpsc::channel(16); + process_events(futures::stream::iter(events), tx, plain_mode()).await; + let mut out = Vec::new(); + while let Ok(e) = rx.try_recv() { + out.push(e); + } + // Exactly one Added — the second Apply is a no-op against the + // existing union. + let added = out + .iter() + .filter(|e| matches!(e, DiscoveryEvent::Added(_))) + .count(); + assert_eq!(added, 1, "out={out:?}"); + } + + /// In PD mode, only slices whose labels match one of the role selectors + /// produce workers; unrelated slices in the same namespace are dropped + /// without registration. + #[tokio::test] + async fn pd_mode_drops_slices_whose_labels_match_no_role_selector() { + let prefill_slice = with_uid( + make_slice_full( + &["10.0.0.1"], + 30000, + true, + "ns", + "p", + &[("app", "sglang"), ("role", "prefill")], + ), + "u-p", + ); + let unrelated_slice = with_uid( + make_slice_full( + &["10.0.0.2"], + 30000, + true, + "ns", + "x", + &[("app", "sglang"), ("role", "router")], + ), + "u-x", + ); + let events = vec![ + Ok(watcher::Event::Apply(prefill_slice)), + Ok(watcher::Event::Apply(unrelated_slice)), + ]; + let (tx, mut rx) = mpsc::channel(16); + process_events(futures::stream::iter(events), tx, pd_mode()).await; + let mut out = Vec::new(); + while let Ok(e) = rx.try_recv() { + out.push(e); + } + let added: Vec<_> = out + .iter() + .filter_map(|e| match e { + DiscoveryEvent::Added(spec) => Some(spec), + _ => None, + }) + .collect(); + assert_eq!( + added.len(), + 1, + "only the prefill-labelled slice should be registered: out={out:?}" + ); + assert_eq!(added[0].mode, WorkerMode::Prefill); + assert_eq!(added[0].id.0, "ns/p/10.0.0.1:30000"); + } + + /// End-to-end K8s + PD integration: synthesize EndpointSlice events + /// for two prefill pods and two decode pods (each backed by a real + /// HTTP listener mounting `/server_info`), pipe them through + /// `process_events` → DiscoveryEvent channel → manager, and assert + /// the resulting registry has: + /// * two `Prefill` workers, each with `bootstrap_port` matching + /// what its own `/server_info` advertised (so per-worker + /// plumbing is verified, not just "some prefill registered"), + /// * two `Decode` workers with `bootstrap_port = None`. + /// + /// This is the load-bearing integration covering the seam this PR + /// just opened: the K8s backend emits `bootstrap_port: None`, the + /// PD-disaggregation classification comes from slice labels, and + /// `WorkerMode` + `bootstrap_port` are re-resolved by the manager + /// from each worker's `/server_info`. A regression at *any* of + /// those three layers (k8s extract → process_events label + /// classification → manager introspect-and-override) fails this + /// test. + #[tokio::test] + async fn k8s_pd_pipeline_registers_workers_with_per_pod_bootstrap_port() { + use crate::workers::introspect::WorkerIntrospector; + use crate::workers::manager::run_with_introspector; + use crate::workers::WorkerRegistry; + use axum::{routing::get, Json, Router}; + use serde_json::{json, Value}; + use std::sync::Arc; + use std::time::Duration; + use tokio::net::TcpListener; + use tokio::sync::oneshot; + + /// Bind axum on an OS-assigned 127.0.0.1 port, mount a + /// `/server_info` returning `body`, return the port + shutdown + /// channel so the test can join cleanly. + async fn spawn_fake_server_info(body: Value) -> (u16, oneshot::Sender<()>) { + let body = Arc::new(body); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/server_info", + get(move || { + let body = body.clone(); + async move { Json((*body).clone()) } + }), + ); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = rx.await; + }) + .await; + }); + (port, tx) + } + + // Four fake SGLang workers — two prefill (each with a distinct + // bootstrap_port to verify per-pod plumbing) and two decode. + let (port_p1, _shut_p1) = spawn_fake_server_info(json!({ + "served_model_name": "m", + "disaggregation_mode": "prefill", + "disaggregation_bootstrap_port": 8998, + })) + .await; + let (port_p2, _shut_p2) = spawn_fake_server_info(json!({ + "served_model_name": "m", + "disaggregation_mode": "prefill", + "disaggregation_bootstrap_port": 8999, + })) + .await; + let (port_d1, _shut_d1) = spawn_fake_server_info(json!({ + "served_model_name": "m", + "disaggregation_mode": "decode", + })) + .await; + let (port_d2, _shut_d2) = spawn_fake_server_info(json!({ + "served_model_name": "m", + "disaggregation_mode": "decode", + })) + .await; + + // One EndpointSlice per worker (one address each, so the slice + // port matches the worker's axum listener port exactly). + // Labels match the PD selectors so `classify_mode` sends each + // slice to the right pool. + let prefill_labels = &[("app", "sglang"), ("role", "prefill")]; + let decode_labels = &[("app", "sglang"), ("role", "decode")]; + let p1 = with_uid( + make_slice_full( + &["127.0.0.1"], + port_p1 as i32, + true, + "ns", + "prefill-1", + prefill_labels, + ), + "u-p1", + ); + let p2 = with_uid( + make_slice_full( + &["127.0.0.1"], + port_p2 as i32, + true, + "ns", + "prefill-2", + prefill_labels, + ), + "u-p2", + ); + let d1 = with_uid( + make_slice_full( + &["127.0.0.1"], + port_d1 as i32, + true, + "ns", + "decode-1", + decode_labels, + ), + "u-d1", + ); + let d2 = with_uid( + make_slice_full( + &["127.0.0.1"], + port_d2 as i32, + true, + "ns", + "decode-2", + decode_labels, + ), + "u-d2", + ); + + // Manager pipeline: DiscoveryEvent channel → run_with_introspector. + let registry = Arc::new(WorkerRegistry::default()); + let (dtx, drx) = mpsc::channel::(16); + let introspector = Arc::new(WorkerIntrospector::new(Duration::from_millis(500))); + let manager_handle = tokio::spawn(run_with_introspector( + drx, + registry.clone(), + None, + None, + None, + introspector, + )); + + // Drive process_events with the four slice Apply events, then + // drop dtx so the manager loop exits cleanly once it has drained. + let events = vec![ + Ok(watcher::Event::Apply(p1)), + Ok(watcher::Event::Apply(p2)), + Ok(watcher::Event::Apply(d1)), + Ok(watcher::Event::Apply(d2)), + ]; + let producer = tokio::spawn(async move { + let stream = futures::stream::iter(events); + process_events(stream, dtx, pd_mode()).await; + }); + + // Poll the registry until all four workers are present with + // their resolved mode + bootstrap_port — order isn't deterministic + // because each worker's `/server_info` round-trip happens in a + // separate manager task. + let expected_p1_id = WorkerId(format!("ns/prefill-1/127.0.0.1:{port_p1}")); + let expected_p2_id = WorkerId(format!("ns/prefill-2/127.0.0.1:{port_p2}")); + let expected_d1_id = WorkerId(format!("ns/decode-1/127.0.0.1:{port_d1}")); + let expected_d2_id = WorkerId(format!("ns/decode-2/127.0.0.1:{port_d2}")); + + let settled = tokio::time::timeout(Duration::from_secs(3), async { + loop { + let p1 = registry.get(&expected_p1_id); + let p2 = registry.get(&expected_p2_id); + let d1 = registry.get(&expected_d1_id); + let d2 = registry.get(&expected_d2_id); + if let (Some(p1), Some(p2), Some(d1), Some(d2)) = (p1, p2, d1, d2) { + if p1.mode() == WorkerMode::Prefill + && p2.mode() == WorkerMode::Prefill + && d1.mode() == WorkerMode::Decode + && d2.mode() == WorkerMode::Decode + && p1.bootstrap_port() == Some(8998) + && p2.bootstrap_port() == Some(8999) + && d1.bootstrap_port().is_none() + && d2.bootstrap_port().is_none() + { + return true; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + settled.is_ok(), + "registry did not converge to (2 prefill + 2 decode) with per-pod \ + bootstrap_port within 3s. current state: \ + p1={:?}, p2={:?}, d1={:?}, d2={:?}", + registry + .get(&expected_p1_id) + .map(|w| (w.mode(), w.bootstrap_port())), + registry + .get(&expected_p2_id) + .map(|w| (w.mode(), w.bootstrap_port())), + registry + .get(&expected_d1_id) + .map(|w| (w.mode(), w.bootstrap_port())), + registry + .get(&expected_d2_id) + .map(|w| (w.mode(), w.bootstrap_port())), + ); + + // Producer should exit when the iter stream ends; manager exits + // when the producer drops dtx. Both should finish quickly. + let _ = tokio::time::timeout(Duration::from_secs(1), producer).await; + let _ = tokio::time::timeout(Duration::from_secs(1), manager_handle).await; + } + + /// Helper: build a slice where every endpoint carries a synthetic + /// `target_ref.uid`. The endpoint at position `i` gets `uids[i]`. + fn make_slice_with_uids(addrs: &[&str], port: i32, uids: &[&str]) -> EndpointSlice { + use k8s_openapi::api::core::v1::ObjectReference; + assert_eq!(addrs.len(), uids.len()); + let endpoints = addrs + .iter() + .zip(uids.iter()) + .map(|(addr, uid)| Endpoint { + addresses: vec![(*addr).to_string()], + conditions: Some(EndpointConditions { + ready: Some(true), + ..Default::default() + }), + target_ref: Some(ObjectReference { + uid: Some((*uid).to_string()), + ..Default::default() + }), + ..Default::default() + }) + .collect(); + EndpointSlice { + metadata: ObjectMeta { + name: Some("svc".into()), + namespace: Some("ns".into()), + ..Default::default() + }, + address_type: "IPv4".into(), + endpoints, + ports: Some(vec![EndpointPort { + port: Some(port), + ..Default::default() + }]), + } + } + + /// Pod is replaced (same IP, different UID) — router must see this as + /// a Removed+Added cycle so the new pod gets fresh CB/active_load + /// state. Without UID-keyed WorkerIds, two consecutive + /// `process_events` snapshots would dedup by `addr:port` and the + /// new pod would inherit the dead pod's state. + #[tokio::test] + async fn pod_replace_with_same_ip_emits_remove_then_add() { + let s_old = with_uid( + make_slice_with_uids(&["10.0.0.1"], 30000, &["uid-old"]), + "u-1", + ); + let s_new = with_uid( + make_slice_with_uids(&["10.0.0.1"], 30000, &["uid-new"]), + "u-1", + ); + let (tx, mut rx) = mpsc::channel(16); + process_events( + futures::stream::iter(vec![ + Ok(watcher::Event::Apply(s_old)), + Ok(watcher::Event::Apply(s_new)), + ]), + tx, + plain_mode(), + ) + .await; + let mut events = Vec::new(); + while let Ok(e) = rx.try_recv() { + events.push(e); + } + // Expect: Added(uid-old) → Removed(uid-old) + Added(uid-new). + // The order of Removed/Added within the second apply depends on + // emit_diff's iteration; assert by counting each variant. + assert_eq!(events.len(), 3, "got {events:?}"); + let added: Vec<&WorkerSpec> = events + .iter() + .filter_map(|e| match e { + DiscoveryEvent::Added(spec) => Some(spec), + _ => None, + }) + .collect(); + let removed: Vec<&WorkerId> = events + .iter() + .filter_map(|e| match e { + DiscoveryEvent::Removed { id } => Some(id), + _ => None, + }) + .collect(); + assert_eq!(added.len(), 2, "two Added (one per UID): {events:?}"); + assert_eq!(removed.len(), 1, "one Removed (for uid-old): {events:?}"); + assert_eq!( + added[0].id.0, "ns/uid-old", + "first Added is the original pod", + ); + assert_eq!( + added[1].id.0, "ns/uid-new", + "second Added is the replacement pod with a fresh UID", + ); + assert_eq!( + removed[0].0, "ns/uid-old", + "Removed targets the original pod's UID, not the IP-keyed id", + ); + // Same URL across both, confirming the IP didn't change. + assert_eq!(added[0].url, added[1].url); + } +} diff --git a/experimental/sgl-router/src/discovery/mod.rs b/experimental/sgl-router/src/discovery/mod.rs new file mode 100644 index 000000000000..f9544f836d41 --- /dev/null +++ b/experimental/sgl-router/src/discovery/mod.rs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +pub mod k8s; +pub mod static_urls; +pub mod types; +pub use types::*; + +use crate::config::{Config, DiscoveryBackend}; +use anyhow::Result; +use tokio::sync::mpsc; + +/// Channel capacity for discovery → registry events. Bounded to 128 — +/// pod-add/remove is infrequent, but a bound prevents unbounded memory +/// growth under any pathological burst. +pub const DISCOVERY_CHANNEL_CAP: usize = 128; + +/// Spawn the configured discovery backend. +/// +/// Returns the consumer end of the event channel and a [`tokio::task::JoinHandle`] +/// for the producer task. The static_urls backend's task exits once the +/// initial fan-out completes; the k8s backend's task runs for the lifetime +/// of the watch. +pub async fn spawn_discovery( + cfg: &Config, +) -> Result<(mpsc::Receiver, tokio::task::JoinHandle<()>)> { + let (tx, rx) = mpsc::channel(DISCOVERY_CHANNEL_CAP); + let handle = match &cfg.discovery.backend { + DiscoveryBackend::StaticUrls(s) => static_urls::spawn(s.clone(), tx).await?, + DiscoveryBackend::K8s(k) => k8s::spawn(k.clone(), tx).await?, + }; + Ok((rx, handle)) +} diff --git a/experimental/sgl-router/src/discovery/static_urls.rs b/experimental/sgl-router/src/discovery/static_urls.rs new file mode 100644 index 000000000000..a548d392250c --- /dev/null +++ b/experimental/sgl-router/src/discovery/static_urls.rs @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Static-URL discovery backend. +//! +//! Takes a fixed list of worker URLs and fans one [`DiscoveryEvent::Added`] +//! per entry. After the initial fan-out the task exits — there is no +//! hot-reload; topology changes require a restart. +//! +//! Each emitted [`WorkerSpec`] uses the URL itself as the `WorkerId` and +//! seeds `mode = Plain` with empty `model_ids` and `bootstrap_port = None`. +//! The worker manager fills those in from each worker's `/server_info` +//! response (see [`crate::workers::introspect`]) and overrides the seeded +//! mode/bootstrap when the worker self-discloses a PD role — so prefill, +//! decode, and plain workers can all appear in the same `urls` list and +//! end up classified correctly. +//! +//! Requires modern SGLang that exposes `disaggregation_mode` in +//! `/server_info`. Workers on older SGLang versions that predate that +//! field stay seeded as `Plain` because the manager has no signal to +//! override with — operators running PD with such a worker should use +//! the K8s backend (which can still classify via pod labels). + +use crate::config::StaticUrlsDiscoveryConfig; +use crate::discovery::{DiscoveryEvent, WorkerId, WorkerMode, WorkerSpec}; +use anyhow::Result; +use tokio::sync::mpsc; + +/// Spawn the static-URLs producer task and return its `JoinHandle`. +/// +/// Returns `Result` for parity with [`crate::discovery::k8s::spawn`] (which +/// can fail to construct a `kube::Client`); this backend itself is +/// infallible. +pub async fn spawn( + cfg: StaticUrlsDiscoveryConfig, + tx: mpsc::Sender, +) -> Result> { + let handle = tokio::spawn(async move { + for url in cfg.urls { + let spec = WorkerSpec { + id: WorkerId(url.clone()), + url, + mode: WorkerMode::Plain, + model_ids: Vec::new(), + bootstrap_port: None, + }; + if tx.send(DiscoveryEvent::Added(spec)).await.is_err() { + tracing::info!( + "static_urls discovery: event channel closed during fan-out; exiting" + ); + return; + } + } + tracing::debug!( + "static_urls discovery: initial fan-out complete; parking until channel closes" + ); + // After fan-out the static backend has no further work — but + // `server::supervisor::supervise_critical_tasks` treats *any* + // discovery exit as fatal and flips `/readyz` to 503. Park here + // until the consumer drops the receiver. `tx.closed()` resolves + // the moment every `Receiver` has been dropped; the supervisor's + // normal-shutdown path aborts this task before that. So + // reaching the `info!` below means either (a) we lost the abort + // race during a clean shutdown, or (b) the worker manager exited + // unexpectedly — in case (b) the supervisor will catch the + // subsequent discovery-task exit and `error!` + mark unready, + // and this breadcrumb gives operator triage a starting point. + tx.closed().await; + tracing::info!( + "static_urls discovery: event channel closed by receiver \ + (worker manager dropped its end, or shutdown abort raced); exiting" + ); + }); + Ok(handle) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Task exits cleanly when the consumer drops the receiver mid-fanout. + /// Without this early exit, the producer would block forever on the + /// closed channel and shutdown would have to abort it. Kept in-source + /// (rather than as a component test) because it inspects the + /// `send().is_err()` branch, which is an implementation detail of + /// this module — fan-out and event-shape assertions live in + /// `tests/component/discovery/static_urls.rs`. + #[tokio::test] + async fn exits_when_receiver_dropped() { + let cfg = StaticUrlsDiscoveryConfig { + urls: (0..10).map(|i| format!("http://w{i}:30000")).collect(), + }; + let (tx, rx) = mpsc::channel(1); + drop(rx); + let h = spawn(cfg, tx).await.unwrap(); + // No panic, no hang — task exits on the first send error. + h.await.unwrap(); + } + + /// After fan-out the task must STAY ALIVE so the critical-task + /// supervisor (`server::supervisor::supervise_critical_tasks`) + /// doesn't treat the exit as a failure and flip `/readyz` to 503. + /// The static_urls backend has no hot-reload, so the only reasons + /// it should ever exit are (a) the consumer dropped the receiver, + /// or (b) the supervisor aborted it on shutdown. A "natural" exit + /// after fan-out used to be the third path, and was wrongly + /// interpreted as a panic by the supervisor — pinned here so a + /// regression to "exit after fan-out" can't sneak back in. + #[tokio::test] + async fn stays_alive_after_fanout_until_receiver_dropped() { + use std::time::Duration; + + let cfg = StaticUrlsDiscoveryConfig { + urls: vec!["http://w0:30000".into(), "http://w1:30000".into()], + }; + let (tx, mut rx) = mpsc::channel(8); + let h = spawn(cfg, tx).await.unwrap(); + + // Drain the fan-out so the task is past the for-loop. + for _ in 0..2 { + let _ = rx.recv().await.expect("fan-out event"); + } + + // Now give the task a long-by-test-standards moment to exit + // post-fanout. Pre-fix this would have completed in under a + // millisecond; post-fix it must time out. + let mut handle = h; + let exited = tokio::time::timeout(Duration::from_millis(200), &mut handle).await; + let still_running = exited.is_err(); + if !still_running { + panic!( + "static_urls task exited after fan-out (joined as {exited:?}); \ + this trips `supervise_critical_tasks` → mark_unready and the pod \ + becomes /readyz 503. The task must park until the receiver is dropped." + ); + } + // Clean shutdown: dropping the receiver closes the channel, which + // the post-fix task uses as its "time to exit" signal. Pin both + // halves of the contract — parks while the receiver is alive AND + // exits cleanly once it's dropped — so a future refactor that + // parks the task on the wrong signal (e.g., a sleep, a token that + // never fires) is caught here rather than silently lingering. + drop(rx); + let joined = tokio::time::timeout(Duration::from_secs(2), handle) + .await + .expect("task must exit promptly after the receiver is dropped"); + joined.expect("task panicked during clean shutdown"); + } +} diff --git a/experimental/sgl-router/src/discovery/types.rs b/experimental/sgl-router/src/discovery/types.rs new file mode 100644 index 000000000000..16ef4f9b1c91 --- /dev/null +++ b/experimental/sgl-router/src/discovery/types.rs @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize}; + +/// Opaque worker identifier. Wraps a string so callsites can't confuse it +/// with other string types (e.g. `ModelId`). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct WorkerId(pub String); + +impl std::fmt::Display for WorkerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Opaque model identifier. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct ModelId(pub String); + +impl std::fmt::Display for ModelId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Prefill/Decode/Plain role of a worker. +/// +/// Serialises as `"plain"`, `"prefill"`, `"decode"` (snake_case). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkerMode { + Plain, + Prefill, + Decode, +} + +/// Immutable worker description emitted by a discovery backend. +/// +/// Backends emit [`DiscoveryEvent::Added`] carrying a `WorkerSpec` when a +/// new worker becomes available, and [`DiscoveryEvent::Removed`] when it +/// leaves. +/// +/// `bootstrap_port` is the SGLang disagg bootstrap server port for +/// prefill workers (set via `--disaggregation-bootstrap-port` at worker +/// startup). Resolved from each worker's `/server_info` response (see +/// [`crate::workers::introspect`]); discovery backends seed it as +/// `None`. `None` for decode and plain workers — they don't own a +/// bootstrap server. The router copies the selected prefill worker's +/// `bootstrap_host`/`bootstrap_port` plus a random `bootstrap_room` +/// u64 onto every PD-disagg request body so the prefill engine can +/// match incoming KV-transfer requests from the decode peer. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkerSpec { + pub id: WorkerId, + pub url: String, + pub mode: WorkerMode, + pub model_ids: Vec, + #[serde(default)] + pub bootstrap_port: Option, +} + +/// Event produced by a discovery backend and consumed by `WorkerManager`. +/// +/// Tagged with `"event"` for JSON clarity: +/// ```json +/// {"event":"added","id":"w1","url":"http://…","mode":"plain","model_ids":["m"]} +/// {"event":"removed","id":"w1"} +/// {"event":"mode_changed","id":"w1","mode":"decode"} +/// ``` +/// +/// The `Added` variant wraps the full [`WorkerSpec`]; the others carry only +/// what changed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +pub enum DiscoveryEvent { + Added(WorkerSpec), + Removed { + id: WorkerId, + }, + /// Used by the k8s backend when only the PD label flips (rare). + ModeChanged { + id: WorkerId, + mode: WorkerMode, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn worker_spec_serde_round_trip() { + let w = WorkerSpec { + id: WorkerId("w1".into()), + url: "http://10.0.0.1:30000".into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("qwen".into())], + bootstrap_port: None, + }; + let s = serde_json::to_string(&w).unwrap(); + let d: WorkerSpec = serde_json::from_str(&s).unwrap(); + assert_eq!(w, d); + } + + #[test] + fn worker_spec_with_bootstrap_port_round_trip() { + let w = WorkerSpec { + id: WorkerId("p1".into()), + url: "http://10.0.0.1:30000".into(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("qwen".into())], + bootstrap_port: Some(8997), + }; + let s = serde_json::to_string(&w).unwrap(); + assert!(s.contains("\"bootstrap_port\":8997")); + let d: WorkerSpec = serde_json::from_str(&s).unwrap(); + assert_eq!(w, d); + } + + #[test] + fn worker_spec_deserializes_with_missing_bootstrap_port() { + // Older configs / hand-written JSON without the field should + // still parse — bootstrap_port defaults to None for non-PD + // deployments. + let json = r#"{"id":"w","url":"http://x","mode":"plain","model_ids":["m"]}"#; + let w: WorkerSpec = serde_json::from_str(json).unwrap(); + assert_eq!(w.bootstrap_port, None); + } + + #[test] + fn worker_mode_serializes_snake_case() { + assert_eq!( + serde_json::to_string(&WorkerMode::Plain).unwrap(), + "\"plain\"" + ); + assert_eq!( + serde_json::to_string(&WorkerMode::Prefill).unwrap(), + "\"prefill\"" + ); + assert_eq!( + serde_json::to_string(&WorkerMode::Decode).unwrap(), + "\"decode\"" + ); + } + + #[test] + fn discovery_event_round_trip() { + let e = DiscoveryEvent::Added(WorkerSpec { + id: WorkerId("w1".into()), + url: "http://x:30000".into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("m1".into())], + bootstrap_port: None, + }); + let s = serde_json::to_string(&e).unwrap(); + let d: DiscoveryEvent = serde_json::from_str(&s).unwrap(); + assert_eq!(e, d); + } +} diff --git a/experimental/sgl-router/src/health/circuit_breaker.rs b/experimental/sgl-router/src/health/circuit_breaker.rs new file mode 100644 index 000000000000..35779f70ed5c --- /dev/null +++ b/experimental/sgl-router/src/health/circuit_breaker.rs @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +// NOTE: `opened_at` uses `tokio::time::Instant` rather than `std::time::Instant` +// so that `#[tokio::test(start_paused = true)]` + `tokio::time::advance` can +// move the clock forward in tests. `std::time::Instant` is not paused by +// tokio's mock clock, so `elapsed()` would always return near-zero inside a +// paused-time test, preventing the Open → HalfOpen transition from being +// exercised deterministically. + +use std::num::NonZeroU32; +use std::sync::Mutex; +use std::time::Duration; +use tokio::time::Instant; + +#[derive(Debug, Clone)] +pub struct CircuitBreakerConfig { + pub threshold: NonZeroU32, + pub cool_down: Duration, +} + +impl Default for CircuitBreakerConfig { + fn default() -> Self { + Self { + threshold: NonZeroU32::new(3).expect("3 is non-zero"), + cool_down: Duration::from_secs(30), + } + } +} + +#[derive(Debug, Clone, Copy)] +enum State { + Closed, + Open { opened_at: Instant }, + HalfOpen { probe_in_flight: bool }, +} + +#[derive(Debug)] +struct Inner { + state: State, + consecutive_failures: u32, +} + +#[derive(Debug)] +pub struct CircuitBreaker { + inner: Mutex, + config: CircuitBreakerConfig, +} + +impl CircuitBreaker { + pub fn new() -> Self { + Self::with_config(CircuitBreakerConfig::default()) + } + + pub fn with_config(config: CircuitBreakerConfig) -> Self { + Self { + inner: Mutex::new(Inner { + state: State::Closed, + consecutive_failures: 0, + }), + config, + } + } + + /// Non-mutating predicate: would [`allow`] return `true` if called + /// right now? + /// + /// Used by enumeration / filter paths (e.g. + /// [`crate::workers::registry::WorkerRegistry::healthy_workers_for`]) + /// that need to inspect breaker readiness without claiming a half-open + /// probe slot. Calling `allow()` for filtering would leak probe slots + /// to unselected candidates and starve dispatch: the policy would + /// enumerate a worker as "healthy", then the proxy's `allow()` at + /// dispatch time would see `probe_in_flight=true` and reject. + /// + /// Semantics: + /// - `Closed` → `true` + /// - `Open` past `cool_down` → `true` (a probe slot is available) + /// - `Open` within `cool_down` → `false` + /// - `HalfOpen { probe_in_flight: true }` → `false` + /// - `HalfOpen { probe_in_flight: false }` → `true` + pub fn would_allow(&self) -> bool { + let g = self.inner.lock().unwrap(); + match g.state { + State::Closed => true, + State::Open { opened_at } => opened_at.elapsed() >= self.config.cool_down, + State::HalfOpen { probe_in_flight } => !probe_in_flight, + } + } + + /// True if a request may proceed. Mutates state when transitioning + /// from Open → HalfOpen. + pub fn allow(&self) -> bool { + let mut g = self.inner.lock().unwrap(); + match g.state { + State::Closed => true, + State::Open { opened_at } => { + if opened_at.elapsed() >= self.config.cool_down { + g.state = State::HalfOpen { + probe_in_flight: true, + }; + true + } else { + false + } + } + State::HalfOpen { probe_in_flight } => { + if probe_in_flight { + false + } else { + g.state = State::HalfOpen { + probe_in_flight: true, + }; + true + } + } + } + } + + pub fn record_success(&self) { + let mut g = self.inner.lock().unwrap(); + g.consecutive_failures = 0; + g.state = State::Closed; + } + + pub fn record_failure(&self) { + let mut g = self.inner.lock().unwrap(); + match g.state { + State::Closed | State::HalfOpen { .. } => { + g.consecutive_failures += 1; + if g.consecutive_failures >= self.config.threshold.get() { + g.state = State::Open { + opened_at: Instant::now(), + }; + } + } + State::Open { .. } => { + // Already open: ticking consecutive_failures or refreshing opened_at + // would pin us Open during a failure storm. The cool_down is + // measured from first-open; failures during Open are ignored. + } + } + } +} + +impl Default for CircuitBreaker { + fn default() -> Self { + Self::new() + } +} diff --git a/experimental/sgl-router/src/health/mod.rs b/experimental/sgl-router/src/health/mod.rs new file mode 100644 index 000000000000..6e38db32f3c2 --- /dev/null +++ b/experimental/sgl-router/src/health/mod.rs @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +pub mod circuit_breaker; diff --git a/experimental/sgl-router/src/lib.rs b/experimental/sgl-router/src/lib.rs new file mode 100644 index 000000000000..469e1fddc4ef --- /dev/null +++ b/experimental/sgl-router/src/lib.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! sgl-router: slim KV-aware OpenAI-compatible router for SGLang workers. +//! +//! See `~/.claude/projects/-Users-kangyan-zhou-sglang-workspace-sglang/specs/2026-05-14-sgl-router-slim-design.md` +//! for the design roadmap. + +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +pub mod config; +pub mod discovery; +pub mod health; +pub mod policies; +pub mod proxy; +pub mod server; +pub mod tokenizer; +pub mod workers; diff --git a/experimental/sgl-router/src/main.rs b/experimental/sgl-router/src/main.rs new file mode 100644 index 000000000000..00e1b4e7baa8 --- /dev/null +++ b/experimental/sgl-router/src/main.rs @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{Context, Result}; +use clap::Parser; +use sgl_router::config::LogFormat; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::signal::unix::{signal, Signal, SignalKind}; + +#[derive(Parser, Debug)] +#[command(name = "sgl-router", version)] +struct Cli { + #[arg(long, env = "SGL_ROUTER_CONFIG")] + config: PathBuf, +} + +/// Install the global tracing subscriber. +/// +/// Idempotent: a second call returns `Ok` without panicking. When +/// `try_init` errors, some other code has already installed a subscriber, +/// so the `tracing::debug!` below is delivered through THAT subscriber — +/// no recursive init. +/// +/// `format` selects the output shape: `Json` emits one JSON record per +/// line (target for production / k8s log aggregators), `Text` is the +/// human-readable default. The `RUST_LOG` environment variable always +/// wins over `default_level`. +fn init_tracing(default_level: &str, format: LogFormat) -> Result<()> { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(default_level)); + let install_result = match format { + LogFormat::Json => tracing_subscriber::fmt() + .with_env_filter(filter) + .with_target(true) + .json() + .try_init(), + LogFormat::Text => tracing_subscriber::fmt() + .with_env_filter(filter) + .with_target(true) + .try_init(), + }; + if let Err(e) = install_result { + // A second install attempt; the existing subscriber is fine. + // Surface the attempted default level so an operator can see + // what we tried. + tracing::debug!( + default_level = %default_level, + ?format, + error = %e, + "tracing subscriber already installed; continuing" + ); + } + Ok(()) +} + +/// Install a minimal text-format subscriber BEFORE config parsing so a +/// config-load error has somewhere to surface. The real subscriber +/// (driven by `Config.observability`) is installed after; the second +/// `try_init` is a no-op because a subscriber is already present. +/// The bootstrap subscriber respects `RUST_LOG` so an operator can +/// debug startup with `RUST_LOG=debug` even when the config file is +/// missing or malformed. +fn install_bootstrap_subscriber() { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .with_target(true) + .try_init(); +} + +/// Install SIGTERM and SIGINT handlers up front so a failure here surfaces +/// before `axum::serve` starts. If installation fails (rare: container +/// without signal capability, seccomp policy), we return an error and the +/// process exits cleanly rather than running deaf to k8s termination. +fn install_signal_handlers() -> Result<(Signal, Signal)> { + let sigterm = signal(SignalKind::terminate()).context("install SIGTERM handler")?; + let sigint = signal(SignalKind::interrupt()).context("install SIGINT handler")?; + Ok((sigterm, sigint)) +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + // Bootstrap subscriber so a Config::from_path error has structured + // output. The configured-format subscriber installs after this and + // becomes a no-op via try_init's idempotency. + install_bootstrap_subscriber(); + let cfg = sgl_router::config::Config::from_path(&cli.config) + .with_context(|| format!("load config from {}", cli.config.display()))?; + + init_tracing(&cfg.observability.log_level, cfg.observability.log_format)?; + + tracing::info!( + "sgl-router {} starting on {}:{}", + env!("CARGO_PKG_VERSION"), + cfg.server.host, + cfg.server.port + ); + + let tokenizers = Arc::new( + sgl_router::tokenizer::TokenizerRegistry::load_from_config(&cfg) + .context("load tokenizers")?, + ); + + let registry = Arc::new(sgl_router::workers::WorkerRegistry::default()); + + // Build the KV-event index up front so the cache-aware-zmq policy can + // share its `HashTree` handle + `BlockSizeOracle`. When no model uses + // `cache_aware_zmq`, the index is still constructed (cheap) but no + // subscribers are ever added. + let block_size_oracle = sgl_router::policies::kv_events::BlockSizeOracle::new(); + let kv_index = sgl_router::policies::kv_events::KvEventIndex::new_with_http_and_oracle( + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(2)) + .build() + .expect("default http client builds"), + Arc::clone(&block_size_oracle), + ); + let policies = Arc::new( + sgl_router::policies::factory::build_registry( + &cfg, + kv_index.tree(), + Arc::clone(&tokenizers), + Arc::clone(&block_size_oracle), + ) + .context("build policy registry")?, + ); + + // Shared ActiveLoadRegistry + janitor task. The janitor reaps + // request entries whose lifetime exceeded `stale_request_timeout`, + // so a leaked guard (proxy task panic, etc.) does not inflate a + // worker's load forever. The registry is built BEFORE the manager + // is spawned so the manager can call `forget_worker` on + // `DiscoveryEvent::Removed`. + let stale_timeout = std::time::Duration::from_secs(cfg.active_load.stale_request_timeout_secs); + let active_load = sgl_router::policies::active_load::ActiveLoadRegistry::new( + Arc::new(sgl_router::policies::active_load::SystemTimeClock), + stale_timeout, + ); + // Sweep cadence is 1/10 of the configured timeout, clamped to + // [1 s, 60 s]. A short timeout (test setting) needs frequent + // sweeps to fire within the test's window; a long timeout + // (production) doesn't need sub-minute checks. + let sweep_interval = std::time::Duration::from_secs( + (cfg.active_load.stale_request_timeout_secs / 10).clamp(1, 60), + ); + let janitor_handle = + sgl_router::policies::active_load::spawn_janitor(Arc::clone(&active_load), sweep_interval); + + // Spawn discovery + manager tasks. + let (event_rx, discovery_handle) = sgl_router::discovery::spawn_discovery(&cfg) + .await + .context("spawn discovery")?; + let kv_index_opt: Option> = + Some(Arc::clone(&kv_index)); + let manager_handle = tokio::spawn(sgl_router::workers::manager::run_with_config( + event_rx, + registry.clone(), + Some(Arc::new(cfg.clone())), + kv_index_opt, + Some(Arc::clone(&active_load)), + )); + + let proxy = Arc::new( + sgl_router::proxy::Proxy::new(std::time::Duration::from_secs( + cfg.proxy.request_timeout_secs, + )) + .context("build proxy client")?, + ); + + let ctx = Arc::new( + sgl_router::server::app_context::AppContext::with_active_load( + cfg.clone(), + tokenizers, + proxy, + registry, + policies, + active_load, + ), + ); + ctx.mark_ready(); + + let app = sgl_router::server::app::build_router(ctx.clone()); + + let bind = format!("{}:{}", cfg.server.host, cfg.server.port); + let listener = tokio::net::TcpListener::bind(&bind) + .await + .with_context(|| format!("bind {bind}"))?; + tracing::info!("listening on {bind}"); + + let (sigterm, sigint) = install_signal_handlers()?; + + let serve = axum::serve(listener, app).with_graceful_shutdown(shutdown_signal(sigterm, sigint)); + let server_result = serve.await.context("axum serve"); + + // Best-effort: cancel discovery + manager + janitor on shutdown. + // The janitor handle's drop signals cancellation; we additionally + // await `shutdown` so the task joins cleanly before the process + // exits — useful for tracing tail logs. + discovery_handle.abort(); + manager_handle.abort(); + janitor_handle.shutdown().await; + server_result +} + +async fn shutdown_signal(mut sigterm: Signal, mut sigint: Signal) { + tokio::select! { + _ = sigterm.recv() => tracing::info!("got SIGTERM, shutting down"), + _ = sigint.recv() => tracing::info!("got SIGINT, shutting down"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn install_signal_handlers_returns_both() { + // Pins the contract that handler installation works on a standard + // tokio runtime. If this fails on a sandboxed runner, the real + // service would also fail to install — which is the point. + assert!(install_signal_handlers().is_ok()); + } + + #[test] + fn init_tracing_is_idempotent() { + let _ = init_tracing("info", LogFormat::Text); + let _ = init_tracing("info", LogFormat::Text); + } + + #[test] + fn init_tracing_accepts_json_format() { + // Doesn't matter whether we win or lose the race against another + // subscriber install — the function must return Ok either way. + assert!(init_tracing("info", LogFormat::Json).is_ok()); + } +} diff --git a/experimental/sgl-router/src/policies/active_load.rs b/experimental/sgl-router/src/policies/active_load.rs new file mode 100644 index 000000000000..f87c9782bc9d --- /dev/null +++ b/experimental/sgl-router/src/policies/active_load.rs @@ -0,0 +1,945 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Per-worker active-load tracking with RAII guards and a stale-request +//! janitor. +//! +//! The cache-aware-zmq policy ([`super::cache_aware_zmq`]) needs to combine +//! the hash tree's overlap score with a per-worker load signal. The +//! per-worker `Worker::active_requests` counter tracks one axis — number of +//! in-flight HTTP requests — and is already drop-safe through +//! [`crate::workers::LoadGuard`]. +//! +//! This module adds two things on top of that: +//! +//! 1. **Per-request bookkeeping** keyed on a `RequestId` so a background +//! janitor can sweep requests that outlive the configured +//! `stale_request_timeout` and decrement the counters they were holding. +//! Without this, a request whose `LoadGuard` is leaked (proxy task +//! panics before the future drops, server hits a panic-catching +//! middleware, etc.) would inflate a worker's load forever. +//! 2. **Two-axis tracking** so PD-disaggregation can score prefill (token +//! count) separately from decode (block count). The two counters share +//! the same registry shape; we expose them as a single +//! [`ActiveLoadGuard`] holding both so the proxy's hot path mints one +//! guard per request rather than two. +//! +//! # Drop semantics +//! +//! Guards decrement on drop AND remove themselves from the request tracker +//! so the janitor never double-decrements. The implementation uses +//! `Option` inside the guard: the janitor's `expire_now` +//! path takes the handle (rendering subsequent drop a no-op for that +//! request), while normal RAII drop also takes the handle (rendering +//! subsequent janitor sweep a no-op). Either path may run first — the +//! other becomes a no-op. Rust's affine type system makes a literal +//! double-drop of the same guard value unreachable. +//! +//! # Clock injection +//! +//! [`ActiveLoadRegistry::new`] is generic over the clock so tests can drive +//! the janitor deterministically. Production wires a `SystemTimeClock`; +//! tests use a `MockClock`. The `Instant`-based timestamp on registration +//! is sufficient for the timeout comparison (monotonic), so the clock +//! abstraction is just two methods: `now()` and an associated `Instant` +//! type whose `duration_since(other)` returns the wall-clock delta. + +use crate::discovery::WorkerId; +use crate::server::metrics::{ActiveLoadKind, MetricsRegistry}; +use dashmap::DashMap; +use parking_lot::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +/// Unique identifier for an in-flight request. Minted by +/// [`ActiveLoadRegistry::register`] and carried inside [`ActiveLoadGuard`] +/// so the janitor can address one request at a time. +#[derive(Clone, Eq, Hash, PartialEq, Debug)] +pub struct RequestId(pub Uuid); + +impl RequestId { + pub fn new_v4() -> Self { + Self(Uuid::new_v4()) + } +} + +impl std::fmt::Display for RequestId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Per-worker counters: one for prefill (token) load, one for decode (block) +/// load. The two axes are tracked separately so cache-aware-zmq can score +/// prefill candidates by token load and decode candidates by block load +/// without each axis spamming through the other's counter. +/// +/// Production tracks **active requests** as the unit (count of in-flight +/// requests pinning the worker), not raw token / block counts — until the +/// proxy wires real prompt-token / completion-block accounting through to +/// `register`. The two axes will become meaningful once the proxy starts +/// passing `prompt_tokens` and `output_blocks` to it. +#[derive(Debug, Default)] +struct WorkerCounters { + prefill_load: AtomicUsize, + decode_load: AtomicUsize, +} + +/// Per-request bookkeeping the janitor consults to find expired requests. +/// +/// `cancel` is a [`CancellationToken`] the janitor fires when the entry +/// is swept. The chat handler holds a clone (via +/// [`ActiveLoadGuard::cancel_token`]) and aborts its upstream fetch +/// with `ApiError::StaleRequestExpired` when the token resolves — +/// surfacing the stale-request expiry as a 504 to the client instead +/// of leaving the handler hung on a long-lived upstream. +/// +/// `counters` is the **exact** `WorkerCounters` instance that was +/// incremented at register time. Holding the `Arc` directly (instead +/// of re-looking-up `workers.get(&worker)` at sweep time) pins the +/// decrement to the same instance — so a worker that is +/// `forget_worker`-removed and re-added under the same `WorkerId` +/// does not underflow the new (zero-initialized) counters slot. +#[derive(Debug)] +struct RequestEntry { + worker: WorkerId, + /// Worker URL captured at register time. The metrics gauge + /// (`sgl_router_active_load`) is keyed by URL, not by `WorkerId`, so + /// drop / sweep paths need the URL to emit the decremented gauge + /// value. Stored on the entry (not looked up via the worker + /// registry) so a `forget_worker` between register and drop still + /// produces a coherent metric trace. + worker_url: String, + counters: Arc, + prefill_load: usize, + decode_load: usize, + registered_at: Instant, + cancel: CancellationToken, +} + +/// Clock abstraction so tests can drive the janitor deterministically. +/// +/// We only need `now()`; ordering is via `Instant::duration_since` which +/// already exists on the std type. Production implementers return the +/// monotonic system instant; tests return whatever value `MockClock` is set +/// to via `set_now`. +pub trait Clock: Send + Sync + std::fmt::Debug { + fn now(&self) -> Instant; +} + +/// Monotonic system clock used in production. +#[derive(Debug, Default, Clone)] +pub struct SystemTimeClock; + +impl Clock for SystemTimeClock { + fn now(&self) -> Instant { + Instant::now() + } +} + +/// Test-only clock that returns a caller-controlled instant. +/// +/// Wrapped in `parking_lot::Mutex` because tests cross await points; the +/// type is `Send + Sync` so it can be stored behind `Arc`. +#[derive(Debug)] +pub struct MockClock { + now: Mutex, +} + +impl MockClock { + pub fn new(start: Instant) -> Self { + Self { + now: Mutex::new(start), + } + } + + /// Advance the clock by `delta`. Returns the new `now`. + pub fn advance(&self, delta: Duration) -> Instant { + let mut guard = self.now.lock(); + *guard += delta; + *guard + } +} + +impl Clock for MockClock { + fn now(&self) -> Instant { + *self.now.lock() + } +} + +/// Registry of in-flight requests + per-worker active-load counters. +/// +/// Constructed once per `AppContext`; the cache-aware-zmq policy reads +/// per-worker `prefill_load` / `decode_load` from here when scoring +/// candidates, and the proxy holds an [`ActiveLoadGuard`] per request so +/// counters decrement on drop. A background task periodically calls +/// [`Self::sweep_stale`] to evict requests that outlived +/// `stale_request_timeout`. +#[derive(Debug)] +pub struct ActiveLoadRegistry { + workers: DashMap>, + requests: DashMap, + clock: Arc, + stale_request_timeout: Duration, + /// Optional Prometheus metrics sink. When attached via + /// [`Self::attach_metrics`] (typically from `AppContext`), every + /// `register` / drop / `sweep_stale` emits the live per-worker + /// `sgl_router_active_load` gauge for both axes. Late binding via + /// `Mutex>` keeps construction order flexible: the + /// registry can be created before the metrics registry exists. + metrics: Mutex>>, +} + +impl ActiveLoadRegistry { + /// Construct an [`ActiveLoadRegistry`] wrapped in an [`Arc`]. + /// + /// The registry is always shared (proxy + janitor + selector all hold + /// the same instance), so the public constructor mints the `Arc` + /// directly to remove an easy footgun where callers forget to wrap + /// it. Tests that need the inner type for direct field access also + /// receive `Arc`. + pub fn new(clock: Arc, stale_request_timeout: Duration) -> Arc { + Arc::new(Self { + workers: DashMap::new(), + requests: DashMap::new(), + clock, + stale_request_timeout, + metrics: Mutex::new(None), + }) + } + + /// Attach (or replace) the [`MetricsRegistry`] this registry pushes + /// gauge updates into. Idempotent; safe to call multiple times. + /// Production wires this from `AppContext` after the metrics registry + /// is constructed; tests skip it unless they assert on the gauge. + pub fn attach_metrics(&self, metrics: Arc) { + *self.metrics.lock() = Some(metrics); + } + + /// Snapshot the current per-worker load and push it to the metrics + /// gauge (if any). Called from the register / drop / sweep paths + /// after the counter mutation completes. Reading `counters.load()` + /// here (rather than computing from the delta) keeps the gauge + /// eventually-consistent with the canonical counter even under + /// concurrent register + drop interleavings. + fn publish_gauge(&self, counters: &WorkerCounters, worker_url: &str) { + let Some(metrics) = self.metrics.lock().clone() else { + return; + }; + metrics.set_active_load( + worker_url, + ActiveLoadKind::PrefillTokens, + counters.prefill_load.load(Ordering::Relaxed) as i64, + ); + metrics.set_active_load( + worker_url, + ActiveLoadKind::DecodeBlocks, + counters.decode_load.load(Ordering::Relaxed) as i64, + ); + } + + /// Default-config registry: monotonic system clock + 10-minute stale + /// timeout. Convenience constructor for production callers; tests use + /// [`Self::new`] with a `MockClock`. Mirrors + /// `default_stale_request_timeout_secs` in `config::types`. + /// + /// 10 minutes is comfortable above 99p generation tail latency + /// (including long-queue throughput-focused workloads) while + /// bounding leak-induced load inflation. + pub fn with_defaults() -> Arc { + Self::new( + Arc::new(SystemTimeClock) as Arc, + Duration::from_secs(10 * 60), + ) + } + + /// Register a new in-flight request and return a guard that holds the + /// active-load counters up. The guard's drop / explicit complete path + /// decrements the counters and removes the request entry. + /// + /// `worker_url` is captured on the entry so drop / sweep can emit a + /// coherent gauge update via the attached [`MetricsRegistry`] (if + /// any). Callers in the request path pass `&worker.url`; tests pass a + /// stable placeholder. + pub fn register( + self: &Arc, + worker: WorkerId, + worker_url: impl Into, + prefill_load: usize, + decode_load: usize, + ) -> ActiveLoadGuard { + let worker_url = worker_url.into(); + let request_id = RequestId::new_v4(); + let counters = self + .workers + .entry(worker.clone()) + .or_insert_with(|| Arc::new(WorkerCounters::default())) + .value() + .clone(); + counters + .prefill_load + .fetch_add(prefill_load, Ordering::Relaxed); + counters + .decode_load + .fetch_add(decode_load, Ordering::Relaxed); + self.publish_gauge(&counters, &worker_url); + let cancel = CancellationToken::new(); + self.requests.insert( + request_id.clone(), + RequestEntry { + worker: worker.clone(), + worker_url, + counters, + prefill_load, + decode_load, + registered_at: self.clock.now(), + cancel: cancel.clone(), + }, + ); + ActiveLoadGuard { + registry: Some(Arc::clone(self)), + request_id: Some(request_id), + worker, + cancel, + } + } + + /// Drop a worker's per-worker counters entry. Called from + /// [`crate::workers::manager`] on `DiscoveryEvent::Removed` so the + /// `WorkerCounters` slot for a now-gone worker does not leak. + /// + /// Guards still alive for that worker remain valid; their drop tries + /// `workers.get(&entry.worker)` which returns `None`, and the + /// per-request `requests` entry is still removed cleanly. A subsequent + /// `register` for the same `WorkerId` reinitializes the slot to 0 — + /// the in-flight guards' loads are NOT re-added, by design (the + /// worker is gone, those loads no longer mean anything). + pub fn forget_worker(&self, id: &WorkerId) { + self.workers.remove(id); + } + + /// Returns `true` if the registry currently has a per-worker + /// counters entry for `id`. Cheap; intended for tests + diagnostics. + pub fn is_known(&self, id: &WorkerId) -> bool { + self.workers.contains_key(id) + } + + /// Current prefill load (sum across in-flight requests) for a worker. + pub fn prefill_load(&self, worker: &WorkerId) -> usize { + self.workers + .get(worker) + .map(|c| c.prefill_load.load(Ordering::Relaxed)) + .unwrap_or(0) + } + + /// Current decode load (sum across in-flight requests) for a worker. + pub fn decode_load(&self, worker: &WorkerId) -> usize { + self.workers + .get(worker) + .map(|c| c.decode_load.load(Ordering::Relaxed)) + .unwrap_or(0) + } + + /// Number of in-flight requests tracked (cheap; useful for tests + + /// metrics). + pub fn inflight_count(&self) -> usize { + self.requests.len() + } + + /// Sweep entries whose `registered_at + stale_request_timeout` is in + /// the past. Returns the number of entries expired. + /// + /// Decrements both axes' worker counters for each expired entry. Safe + /// to call concurrently with `register` and with guard drops — each + /// `remove` operation is atomic and the per-worker counters are + /// `AtomicUsize` so partial visibility cannot under-decrement. + pub fn sweep_stale(&self) -> usize { + let now = self.clock.now(); + let mut expired_ids: Vec = Vec::new(); + for entry in self.requests.iter() { + if now.duration_since(entry.value().registered_at) >= self.stale_request_timeout { + expired_ids.push(entry.key().clone()); + } + } + let mut count = 0; + for id in expired_ids { + // Use `remove`: if the guard's drop concurrently removed the + // entry between our scan and this point, the second remove + // returns `None` and we skip (no double-decrement). + if let Some((_, entry)) = self.requests.remove(&id) { + // Decrement the **captured** counters Arc — the same + // instance the register call incremented. This stays + // correct across `forget_worker` + re-register cycles: + // even if `self.workers[&entry.worker]` now points at a + // brand-new `WorkerCounters`, our decrement targets the + // original one (still alive via this Arc clone). + entry + .counters + .prefill_load + .fetch_sub(entry.prefill_load, Ordering::Relaxed); + entry + .counters + .decode_load + .fetch_sub(entry.decode_load, Ordering::Relaxed); + self.publish_gauge(&entry.counters, &entry.worker_url); + // Wake the chat handler awaiting this request so it can + // return `ApiError::StaleRequestExpired` to the client. + // Cancellation is idempotent; if the handler already + // finished and dropped the guard, the token is already + // dropped and `cancel()` is a no-op for everyone. + entry.cancel.cancel(); + count += 1; + tracing::warn!( + request_id = %id, + worker = %entry.worker, + prefill_load = entry.prefill_load, + decode_load = entry.decode_load, + "stale request swept by active-load janitor", + ); + } + } + count + } +} + +/// Spawn a background janitor task that periodically calls +/// [`ActiveLoadRegistry::sweep_stale`]. +/// +/// Returns a [`JanitorHandle`] that owns the join handle and a cancellation +/// token. Dropping the handle cancels the task; calling +/// [`JanitorHandle::shutdown`] cancels and awaits the join. +/// +/// `interval` is the wall-clock cadence of the sweep. A sensible default +/// is half the configured `stale_request_timeout` so an expired entry is +/// reaped within 1.5× the timeout in the worst case. Pass a fresh +/// `Arc` (cloned from the shared one held in +/// `AppContext`). +pub fn spawn_janitor(registry: Arc, interval: Duration) -> JanitorHandle { + let cancel = CancellationToken::new(); + let cancel_for_task = cancel.clone(); + let join = tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = cancel_for_task.cancelled() => { + tracing::debug!("active-load janitor: shutdown requested"); + return; + } + _ = ticker.tick() => { + let n = registry.sweep_stale(); + if n > 0 { + tracing::info!( + swept = n, + "active-load janitor: removed stale requests", + ); + } + } + } + } + }); + JanitorHandle { + cancel, + join: Some(join), + } +} + +/// Owner handle for the background janitor task. Dropping the handle +/// cancels the task; calling [`Self::shutdown`] cancels AND awaits join, +/// giving callers a clean shutdown path. +#[must_use = "JanitorHandle owns the background task; dropping it cancels the janitor"] +pub struct JanitorHandle { + cancel: CancellationToken, + join: Option>, +} + +impl JanitorHandle { + pub async fn shutdown(mut self) { + self.cancel.cancel(); + if let Some(j) = self.join.take() { + // 2 s ceiling guards against a runtime-teardown hang; the + // janitor exits within one tick of `cancelled()`. + let _ = tokio::time::timeout(Duration::from_secs(2), j).await; + } + } +} + +impl Drop for JanitorHandle { + fn drop(&mut self) { + self.cancel.cancel(); + } +} + +/// RAII guard returned by [`ActiveLoadRegistry::register`]. +/// +/// `#[must_use]`: a statement-form `registry.register(...)` would drop the +/// guard on the same line and decrement the counter before the request +/// actually executed, defeating the purpose. The compile-time warning +/// catches that misuse. +#[must_use = "ActiveLoadGuard must be held for the request's lifetime; dropping it immediately decrements counters"] +#[derive(Debug)] +pub struct ActiveLoadGuard { + registry: Option>, + /// `None` after the janitor expired this request — drop becomes a + /// no-op in that case. The guard keeps only the `RequestId`; the + /// per-axis amounts (and the captured `Arc`) live + /// in the registry's `RequestEntry` so drop and the janitor + /// consult the same source of truth. + request_id: Option, + worker: WorkerId, + /// Cancellation token mirrored from `RequestEntry::cancel`. The + /// chat handler awaits `cancel.cancelled()` in a `tokio::select!` + /// branch so the janitor can interrupt an upstream fetch and force + /// the handler to return `ApiError::StaleRequestExpired` (HTTP 504). + cancel: CancellationToken, +} + +impl ActiveLoadGuard { + /// Read-only accessor (mainly for tests + diagnostic logging). + pub fn worker(&self) -> &WorkerId { + &self.worker + } + + /// Borrow the cancellation token. The chat handler clones it for + /// the `tokio::select!` branch (`token.cancelled().await`) so the + /// guard itself can still move into the SSE pump task (or stay in + /// the buffered-response scope) without losing the wake-up channel. + pub fn cancel_token(&self) -> &CancellationToken { + &self.cancel + } +} + +impl Drop for ActiveLoadGuard { + fn drop(&mut self) { + // If the janitor already expired this request (or `expire_now` was + // called explicitly), `request_id` is `None` and we skip — the + // janitor already decremented the counters. + let (Some(registry), Some(id)) = (self.registry.take(), self.request_id.take()) else { + return; + }; + // `remove` returns `Some` exactly once; if the janitor races us + // and wins, we skip the decrement here. Decrement the **same** + // counters Arc the register call incremented (see + // `ActiveLoadGuard::counters`) — pinning the decrement to a + // specific WorkerCounters instance keeps the math correct + // across `forget_worker` + re-register cycles. + if let Some((_, entry)) = registry.requests.remove(&id) { + entry + .counters + .prefill_load + .fetch_sub(entry.prefill_load, Ordering::Relaxed); + entry + .counters + .decode_load + .fetch_sub(entry.decode_load, Ordering::Relaxed); + registry.publish_gauge(&entry.counters, &entry.worker_url); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn registry_with_mock_clock(timeout: Duration) -> (Arc, Arc) { + let clock = Arc::new(MockClock::new(Instant::now())); + let registry = ActiveLoadRegistry::new(Arc::clone(&clock) as Arc, timeout); + (registry, clock) + } + + #[test] + fn single_worker_increment_decrement_round_trip() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + let g = registry.register(w.clone(), "test://100-5", 100, 5); + assert_eq!(registry.prefill_load(&w), 100); + assert_eq!(registry.decode_load(&w), 5); + assert_eq!(registry.inflight_count(), 1); + drop(g); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + assert_eq!(registry.inflight_count(), 0); + } + + #[test] + fn two_concurrent_guards_increment_to_2_then_drop_to_0() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + let g1 = registry.register(w.clone(), "test://10-1", 10, 1); + let g2 = registry.register(w.clone(), "test://20-2", 20, 2); + assert_eq!(registry.prefill_load(&w), 30); + assert_eq!(registry.decode_load(&w), 3); + assert_eq!(registry.inflight_count(), 2); + drop(g1); + assert_eq!(registry.prefill_load(&w), 20); + assert_eq!(registry.decode_load(&w), 2); + drop(g2); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + } + + #[test] + fn guard_decrements_on_implicit_drop_via_scope_exit() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + { + let _g = registry.register(w.clone(), "test://7-1", 7, 1); + assert_eq!(registry.prefill_load(&w), 7); + } + assert_eq!(registry.prefill_load(&w), 0); + } + + #[test] + fn distinct_workers_are_isolated() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w0 = WorkerId("w0".into()); + let w1 = WorkerId("w1".into()); + let _g0 = registry.register(w0.clone(), "test://5-0", 5, 0); + let _g1 = registry.register(w1.clone(), "test://11-0", 11, 0); + assert_eq!(registry.prefill_load(&w0), 5); + assert_eq!(registry.prefill_load(&w1), 11); + } + + /// Gap closer #2: double-drop safety. + /// + /// Rust's affine type system makes a literal double-drop of the same + /// `ActiveLoadGuard` value impossible — the compiler rejects + /// `drop(g); drop(g);`. The interesting property is that the + /// registry's own bookkeeping never under-decrements, even if the + /// janitor and a guard's drop race. We assert that by simulating the + /// race: the janitor wins (entry removed via `sweep_stale`), then the + /// guard's drop runs — must be a no-op. + #[test] + fn janitor_then_guard_drop_does_not_underflow() { + let (registry, clock) = registry_with_mock_clock(Duration::from_secs(1)); + let w = WorkerId("w0".into()); + let g = registry.register(w.clone(), "test://50-5", 50, 5); + clock.advance(Duration::from_secs(2)); + let n = registry.sweep_stale(); + assert_eq!(n, 1); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + // Janitor already removed entry; guard's drop must not under-flow. + drop(g); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + assert_eq!(registry.inflight_count(), 0); + } + + /// Gap closer #4: stale-request janitor expiry zeroes counters. + #[test] + fn janitor_expires_stale_requests() { + let (registry, clock) = registry_with_mock_clock(Duration::from_secs(5)); + let w = WorkerId("w0".into()); + let _g = registry.register(w.clone(), "test://100-4", 100, 4); + assert_eq!(registry.prefill_load(&w), 100); + // Just below the threshold — no expiry. + clock.advance(Duration::from_secs(4)); + assert_eq!(registry.sweep_stale(), 0); + assert_eq!(registry.prefill_load(&w), 100); + // Past the threshold — expires. + clock.advance(Duration::from_secs(2)); + assert_eq!(registry.sweep_stale(), 1); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + } + + #[test] + fn janitor_is_idempotent_on_double_run() { + let (registry, clock) = registry_with_mock_clock(Duration::from_secs(1)); + let w = WorkerId("w0".into()); + let _g = registry.register(w.clone(), "test://7-0", 7, 0); + clock.advance(Duration::from_secs(2)); + assert_eq!(registry.sweep_stale(), 1); + // Second run finds nothing to do. + assert_eq!(registry.sweep_stale(), 0); + assert_eq!(registry.prefill_load(&w), 0); + } + + #[test] + fn janitor_leaves_fresh_requests_alone() { + let (registry, clock) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + let _g = registry.register(w.clone(), "test://50-0", 50, 0); + clock.advance(Duration::from_secs(1)); + assert_eq!(registry.sweep_stale(), 0); + assert_eq!(registry.prefill_load(&w), 50); + } + + /// Spawned janitor sweeps stale entries on its periodic tick. Uses + /// real (short) sleeps so that the tokio interval timer fires; the + /// registry's clock is the real `SystemTimeClock` so both views of + /// "now" advance together. 200 ms total wait is comfortably above + /// the 30 ms timeout we configure. + #[tokio::test] + async fn spawn_janitor_sweeps_stale_entries() { + let clock: Arc = Arc::new(SystemTimeClock); + let registry = ActiveLoadRegistry::new(clock, Duration::from_millis(30)); + let w = WorkerId("w0".into()); + let _g = registry.register(w.clone(), "test://50-2", 50, 2); + assert_eq!(registry.inflight_count(), 1); + + let handle = spawn_janitor(Arc::clone(®istry), Duration::from_millis(20)); + // Wait long enough for at least one sweep to find the entry + // past the 30 ms timeout. 200 ms allows ~9 ticks of slack. + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(registry.inflight_count(), 0, "janitor should have swept"); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + handle.shutdown().await; + } + + /// Shutdown is idempotent — calling `shutdown` once must cleanly + /// terminate the janitor without hanging. + #[tokio::test] + async fn spawn_janitor_shutdown_is_clean() { + let clock: Arc = Arc::new(SystemTimeClock); + let registry = ActiveLoadRegistry::new(clock, Duration::from_secs(60)); + let handle = spawn_janitor(Arc::clone(®istry), Duration::from_millis(100)); + // Verify shutdown completes within a generous bound. + let r = tokio::time::timeout(Duration::from_secs(2), handle.shutdown()).await; + assert!(r.is_ok(), "janitor shutdown timed out"); + } + + /// Task B: `forget_worker` drops the per-worker counters entry so a + /// disappeared worker does not leak a `WorkerCounters` slot. + /// Existing guards still drop cleanly (no underflow / panic). + #[test] + fn forget_worker_drops_counters_entry() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + let g = registry.register(w.clone(), "test://7-2", 7, 2); + assert!(registry.is_known(&w), "worker is known after register"); + assert_eq!(registry.prefill_load(&w), 7); + + registry.forget_worker(&w); + assert!( + !registry.is_known(&w), + "worker counters entry must be removed after forget_worker", + ); + // Per-worker counters are gone, so the load query reads 0. + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + + // The guard still has a live request entry pointing at the + // now-forgotten worker. Drop must NOT panic; the registry's + // worker map being empty for this id is treated as the + // "already-cleaned-up" terminal state. + drop(g); + assert_eq!( + registry.inflight_count(), + 0, + "guard's drop must still tear down the request entry", + ); + } + + /// Task B: forgetting an unknown worker is a no-op (idempotent). + /// The manager calls `forget_worker` unconditionally on `Removed`, + /// so a double-Removed event or a Removed for a never-seen worker + /// must not panic. + #[test] + fn forget_unknown_worker_is_noop() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + registry.forget_worker(&WorkerId("never-registered".into())); + // No assertion beyond "did not panic"; the body of the test + // exercises the contract. + } + + /// Task B regression: an in-flight guard must NOT underflow the + /// counters of a freshly re-registered worker that reuses its + /// predecessor's `WorkerId`. This is the exact scenario `forget_ + /// worker` is supposed to make safe — and it requires Drop to + /// decrement the **captured** `Arc`, not the + /// current `workers[worker]` lookup. + #[test] + fn forget_then_reregister_does_not_underflow_new_counters() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + let old_guard = registry.register(w.clone(), "test://7-2", 7, 2); + registry.forget_worker(&w); + // Re-register under the same id → brand-new WorkerCounters + // slot. Fresh load of 0 (we mint a no-op guard to materialize + // the slot without bumping any counters). + let _new_guard = registry.register(w.clone(), "test://0-0", 0, 0); + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + + // Dropping the OLD guard must subtract from the OLD counters + // (which are now orphaned but kept alive via the Arc captured + // in the RequestEntry). The NEW counters' values are unaffected. + drop(old_guard); + assert_eq!( + registry.prefill_load(&w), + 0, + "new worker's prefill_load must NOT underflow when old guard drops", + ); + assert_eq!( + registry.decode_load(&w), + 0, + "new worker's decode_load must NOT underflow when old guard drops", + ); + } + + /// Task D: janitor expiry fires the guard's cancellation token so + /// the in-flight handler can return `StaleRequestExpired`. + #[tokio::test] + async fn janitor_expiry_fires_guard_cancel_token() { + let (registry, clock) = registry_with_mock_clock(Duration::from_secs(1)); + let w = WorkerId("w0".into()); + let g = registry.register(w.clone(), "test://50-5", 50, 5); + let cancel = g.cancel_token().clone(); + assert!( + !cancel.is_cancelled(), + "fresh guard's cancel token must not be cancelled", + ); + + clock.advance(Duration::from_secs(2)); + assert_eq!(registry.sweep_stale(), 1); + + // The sweep must have fired the token. We don't await + // `cancelled()` because the test is single-threaded and the + // token resolves synchronously after `cancel.cancel()`. + assert!( + cancel.is_cancelled(), + "stale sweep must cancel the guard's token", + ); + // Drop the guard last so the test exits cleanly. + drop(g); + } + + /// Task D: normal completion (guard drop) does NOT cancel the token. + /// The chat handler's `select!` branch is meant to fire only on a + /// janitor expiry — successful completion drops the guard without + /// touching the token, so the request returns 200 OK. + #[test] + fn guard_drop_does_not_cancel_token() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + let g = registry.register(w, "test://1-1", 1, 1); + let cancel = g.cancel_token().clone(); + drop(g); + assert!( + !cancel.is_cancelled(), + "normal guard drop must NOT cancel the token (sweep-only signal)", + ); + } + + /// When a [`MetricsRegistry`] is attached, the per-worker active-load + /// gauge mirrors the live counter on register / drop / sweep. + /// Regression: prior code exposed [`MetricsRegistry::set_active_load`] + /// but nothing in the request hot path ever called it, leaving + /// `sgl_router_active_load` permanently at 0 in production. + #[test] + fn metrics_gauge_tracks_active_load_on_register_and_drop() { + use crate::server::metrics::MetricsRegistry; + + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let metrics = MetricsRegistry::new(); + registry.attach_metrics(Arc::clone(&metrics)); + let w = WorkerId("w0".into()); + let url = "http://w0:30000"; + + // Before any register, the gauge isn't surfaced (no entry yet). + let rendered = metrics.render(); + assert!( + !rendered.contains("sgl_router_active_load{worker_url=\"http://w0:30000\""), + "no gauge entry expected before first register; got:\n{rendered}" + ); + + // Register a request with prefill_load=100, decode_load=5. + let g = registry.register(w.clone(), url, 100, 5); + let rendered = metrics.render(); + assert!( + rendered.contains( + "sgl_router_active_load{worker_url=\"http://w0:30000\",kind=\"prefill_tokens\"} 100" + ), + "expected prefill_tokens=100 gauge, got:\n{rendered}" + ); + assert!( + rendered.contains( + "sgl_router_active_load{worker_url=\"http://w0:30000\",kind=\"decode_blocks\"} 5" + ), + "expected decode_blocks=5 gauge, got:\n{rendered}" + ); + + // Drop the guard → gauge returns to 0. + drop(g); + let rendered = metrics.render(); + assert!( + rendered.contains( + "sgl_router_active_load{worker_url=\"http://w0:30000\",kind=\"prefill_tokens\"} 0" + ), + "expected prefill_tokens=0 after drop, got:\n{rendered}" + ); + assert!( + rendered.contains( + "sgl_router_active_load{worker_url=\"http://w0:30000\",kind=\"decode_blocks\"} 0" + ), + "expected decode_blocks=0 after drop, got:\n{rendered}" + ); + } + + #[test] + fn metrics_gauge_tracks_active_load_on_sweep_stale() { + use crate::server::metrics::MetricsRegistry; + + let (registry, clock) = registry_with_mock_clock(Duration::from_millis(50)); + let metrics = MetricsRegistry::new(); + registry.attach_metrics(Arc::clone(&metrics)); + let w = WorkerId("w1".into()); + let url = "http://w1:30000"; + + // `register` returns a guard but we don't drop it — janitor sweeps it. + let _g = registry.register(w, url, 200, 10); + let rendered = metrics.render(); + assert!( + rendered.contains( + "sgl_router_active_load{worker_url=\"http://w1:30000\",kind=\"prefill_tokens\"} 200" + ), + "register emits gauge; got:\n{rendered}" + ); + + // Advance past timeout and sweep. + clock.advance(Duration::from_millis(60)); + let swept = registry.sweep_stale(); + assert_eq!(swept, 1, "exactly one entry should have expired"); + let rendered = metrics.render(); + assert!( + rendered.contains( + "sgl_router_active_load{worker_url=\"http://w1:30000\",kind=\"prefill_tokens\"} 0" + ), + "sweep emits decremented gauge; got:\n{rendered}" + ); + } + + /// Concurrent stress: many guards on the same worker should leave the + /// counter back at zero once all guards drop. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_register_drop_returns_to_zero() { + let (registry, _) = registry_with_mock_clock(Duration::from_secs(60)); + let w = WorkerId("w0".into()); + let mut set = tokio::task::JoinSet::new(); + for _ in 0..100 { + let r = Arc::clone(®istry); + let wid = w.clone(); + set.spawn(async move { + for _ in 0..100 { + let _g = r.register(wid.clone(), "test://1-1", 1, 1); + // Yield occasionally so tasks interleave. + tokio::task::yield_now().await; + } + }); + } + while set.join_next().await.is_some() {} + assert_eq!(registry.prefill_load(&w), 0); + assert_eq!(registry.decode_load(&w), 0); + assert_eq!(registry.inflight_count(), 0); + } +} diff --git a/experimental/sgl-router/src/policies/cache_aware_zmq.rs b/experimental/sgl-router/src/policies/cache_aware_zmq.rs new file mode 100644 index 000000000000..804a415e15a8 --- /dev/null +++ b/experimental/sgl-router/src/policies/cache_aware_zmq.rs @@ -0,0 +1,675 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Cache-aware-ZMQ selection policy. +//! +//! Combines the KV-event-fed [`HashTree`] with active-load scoring and +//! tokenizer-driven block-hash lookup to pick the worker most likely to +//! already hold the request's prefix in its KV cache. +//! +//! # Selection algorithm +//! +//! Given `workers` (already filtered to healthy + matching pool by the +//! caller) and a `SelectionContext` carrying the JSON request body: +//! +//! 1. **Load-imbalance fast-path.** If `max_load - min_load > +//! balance_abs_threshold` AND `max_load > min_load * +//! balance_rel_threshold`, skip the cache lookup and pick the +//! lowest-load worker. This prevents one hot worker from dominating +//! cache-aware selection while every other worker idles. +//! 2. **Tokenize.** Pull the prompt text out of the JSON body (`messages` or +//! `prompt` field), run it through the per-model tokenizer. On any +//! failure (no body, no tokenizer, encode error, empty tokens), fall +//! through to step 4 (min-load fallback). +//! 3. **Hash + match.** Compute block hashes via +//! [`super::kv_events::compute_block_hashes`], query the shared hash tree +//! for the longest matching prefix. If `match_rate > cache_threshold`, +//! pick the lowest-load worker whose `url` appears in the match result. +//! Otherwise, fall through. +//! 4. **Min-load fallback.** Pick the lowest-load worker by +//! `Worker::active_load()`. +//! +//! The implementation never returns `None` for a non-empty `workers` slice; +//! a misconfigured tree or tokenizer degrades to round-robin-with-load +//! tiebreak, not a routing failure. + +use crate::config::CacheAwareConfig; + +use crate::discovery::ModelId; +use crate::policies::kv_events::{compute_block_hashes, BlockSizeOracle, HashTree}; +use crate::policies::{Policy, SelectionContext}; +use crate::tokenizer::{adapter, TokenizerRegistry}; +use crate::workers::Worker; +use std::sync::Arc; + +/// Selection policy that scores candidates by tree-overlap with the +/// request's prefix and falls back to load-based picking when the tree +/// doesn't have useful signal. +pub struct CacheAwareZmqPolicy { + config: CacheAwareConfig, + /// Per-process KV-event hash tree, fed by the indexer. Cheap to + /// clone an `Arc`; we never write to the tree from here. + tree: Arc, + /// Tokenizer registry — selection reads `model_id` from the context + /// and looks up the per-model tokenizer. + tokenizers: Arc, + /// Worker-sourced block size, shared with the `KvEventIndex` that + /// seeds it on worker registration. Read once per request; if + /// `None` (no worker has reported a `page_size` yet) the policy + /// degrades to min-load — the router cannot hash a prompt without + /// a block size that matches what the worker publishes. + block_size_oracle: Arc, +} + +impl std::fmt::Debug for CacheAwareZmqPolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CacheAwareZmqPolicy") + .field("config", &self.config) + .field("tree_nodes", &self.tree.node_count()) + .finish() + } +} + +impl CacheAwareZmqPolicy { + pub fn new( + config: CacheAwareConfig, + tree: Arc, + tokenizers: Arc, + block_size_oracle: Arc, + ) -> Self { + Self { + config, + tree, + tokenizers, + block_size_oracle, + } + } + + /// Lowest-load worker — ties broken by stable iteration order (which + /// is the order the registry returned, i.e. dashmap-undefined). For + /// production traffic the ties are rare; tests pin the load skew. + fn pick_min_load(workers: &[Arc]) -> Option> { + workers + .iter() + .min_by_key(|w| w.active_load()) + .map(Arc::clone) + } + + /// Detect load imbalance. Returns `true` when the spread between max + /// and min load is large enough that cache-aware routing would dump + /// even more on the hot worker. + fn is_imbalanced(&self, workers: &[Arc]) -> bool { + let (min_load, max_load) = workers.iter().fold((usize::MAX, 0usize), |(mn, mx), w| { + let l = w.active_load(); + (mn.min(l), mx.max(l)) + }); + let min_load = if min_load == usize::MAX { 0 } else { min_load }; + let abs_diff = max_load.saturating_sub(min_load); + let rel_threshold = (min_load as f32 * self.config.balance_rel_threshold) as usize; + abs_diff > self.config.balance_abs_threshold && max_load > rel_threshold + } + + /// Extract a prompt-text candidate from a JSON request body. Returns + /// `None` if the body isn't valid JSON or doesn't contain a routable + /// text field; the caller falls back to non-cache-aware routing. + /// + /// Supported shapes (in priority order): + /// 1. `"prompt": "..."` — `/v1/completions`-style. + /// 2. `"prompt": ["...", "..."]` — `/v1/completions` array form; + /// concatenated with `"\n"`. + /// 3. `"messages": [{"content": "..."}]` — `/v1/chat/completions` + /// with string content; concatenated with `"\n"`. + /// 4. `"messages": [{"content": [{"text": "..."}]}]` — chat with + /// multimodal content blocks; text-only blocks concatenated. + /// 5. `"text": "..."` — SGLang `/generate` native form. + /// + /// Anything else yields `None`. + fn extract_prompt_text(body: &[u8]) -> Option { + let v: serde_json::Value = serde_json::from_slice(body).ok()?; + if let Some(s) = v.get("prompt").and_then(|p| p.as_str()) { + return Some(s.to_string()); + } + if let Some(arr) = v.get("prompt").and_then(|p| p.as_array()) { + let parts: Vec<&str> = arr.iter().filter_map(|x| x.as_str()).collect(); + if !parts.is_empty() { + return Some(parts.join("\n")); + } + } + if let Some(msgs) = v.get("messages").and_then(|m| m.as_array()) { + let mut buf = String::new(); + for m in msgs { + match m.get("content") { + Some(serde_json::Value::String(s)) => { + if !buf.is_empty() { + buf.push('\n'); + } + buf.push_str(s); + } + Some(serde_json::Value::Array(parts)) => { + for part in parts { + if let Some(t) = part.get("text").and_then(|t| t.as_str()) { + if !buf.is_empty() { + buf.push('\n'); + } + buf.push_str(t); + } + } + } + _ => {} + } + } + if !buf.is_empty() { + return Some(buf); + } + } + if let Some(s) = v.get("text").and_then(|t| t.as_str()) { + return Some(s.to_string()); + } + None + } + + /// Tokenize `text` for `model_id`. Returns `None` if no tokenizer is + /// loaded (the model_id may be misconfigured) or if encoding fails. + /// Errors log at debug — they degrade routing but are not fatal. + fn tokenize(&self, model_id: &ModelId, text: &str) -> Option> { + let tokenizer = self.tokenizers.get(&model_id.0)?; + match adapter::encode(&tokenizer, text) { + Ok(ids) if !ids.is_empty() => Some(ids), + Ok(_) => None, + Err(e) => { + tracing::debug!( + model = %model_id, + error = %e, + "cache-aware-zmq: tokenize failed; falling back to min-load", + ); + None + } + } + } +} + +impl Policy for CacheAwareZmqPolicy { + fn select(&self, workers: &[Arc], ctx: &SelectionContext<'_>) -> Option> { + if workers.is_empty() { + return None; + } + + // 1. Load-imbalance fast-path: even the best cache hit gets + // dropped in favour of evening out load. + if self.is_imbalanced(workers) { + return Self::pick_min_load(workers); + } + + // 2. Extract the prompt text. + let body = match ctx.request_body() { + Some(b) if !b.is_empty() => b, + _ => return Self::pick_min_load(workers), + }; + let Some(text) = Self::extract_prompt_text(body) else { + return Self::pick_min_load(workers); + }; + + // 3. Tokenize + hash + match. + let Some(tokens) = self.tokenize(ctx.model(), &text) else { + return Self::pick_min_load(workers); + }; + // Source block_size from the worker — the router can only hash + // prompts at the block size the workers publish at. If no worker + // has registered yet (oracle empty), cache-aware routing has no + // ground truth to score against; fall back to min-load. + let Some(block_size) = self.block_size_oracle.get() else { + return Self::pick_min_load(workers); + }; + let block_hashes = compute_block_hashes(&tokens, block_size as usize); + if block_hashes.is_empty() { + return Self::pick_min_load(workers); + } + let matched = self.tree.match_prefix(None, &block_hashes); + let match_rate = matched.matched_blocks as f32 / block_hashes.len() as f32; + tracing::debug!( + model = %ctx.model(), + n_blocks = block_hashes.len(), + matched_blocks = matched.matched_blocks, + match_rate, + cache_threshold = self.config.cache_threshold, + "cache-aware-zmq match_prefix", + ); + if match_rate <= self.config.cache_threshold || matched.workers.is_empty() { + return Self::pick_min_load(workers); + } + // Among workers in the matched set, pick the lowest-load one. + let matched_urls: std::collections::HashSet<&str> = + matched.workers.iter().map(|kw| kw.url.as_str()).collect(); + let best_matched: Option> = workers + .iter() + .filter(|w| matched_urls.contains(w.url.as_str())) + .min_by_key(|w| w.active_load()) + .map(Arc::clone); + best_matched.or_else(|| Self::pick_min_load(workers)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::CacheAwareConfig; + use crate::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; + use crate::policies::kv_events::tree::KvWorkerId; + use crate::policies::kv_events::HashTree; + + fn cfg_default() -> CacheAwareConfig { + CacheAwareConfig { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + } + } + + /// Helper: build a `BlockSizeOracle` already primed to the test's + /// canonical block size (4). Mirrors what `KvEventIndex::add_worker` + /// would do when the first real worker registers. + fn oracle_for_tests(block_size: u32) -> Arc { + let o = BlockSizeOracle::new(); + o.try_set(block_size) + .expect("fresh oracle accepts first set"); + o + } + + fn worker(url: &str, model_id: &str) -> Arc { + Arc::new(Worker::new(WorkerSpec { + id: WorkerId(url.into()), + url: url.into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId(model_id.into())], + bootstrap_port: None, + })) + } + + fn tokenizer_registry_with_tiny() -> Arc { + let cfg = crate::config::Config { + server: crate::config::ServerConfig { + host: "0".into(), + port: 0, + }, + observability: Default::default(), + models: vec![crate::config::ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: crate::config::PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: crate::config::DiscoveryConfig { + backend: crate::config::DiscoveryBackend::StaticUrls( + crate::config::StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }, + ), + }, + proxy: crate::config::ProxyConfig::default(), + active_load: crate::config::ActiveLoadConfig::default(), + }; + Arc::new(TokenizerRegistry::load_from_config(&cfg).expect("load tiny tokenizer")) + } + + /// Empty workers list returns None (parity with other policies). + #[test] + fn empty_workers_returns_none() { + let tree = Arc::new(HashTree::new()); + let policy = CacheAwareZmqPolicy::new( + cfg_default(), + tree, + tokenizer_registry_with_tiny(), + oracle_for_tests(4), + ); + let model = ModelId("tiny".into()); + let ctx = SelectionContext::new(&model, Some(b"{\"prompt\":\"hi\"}")); + assert!(policy.select(&[], &ctx).is_none()); + } + + /// Empty tree: no overlap signal anywhere, fall through to min-load. + #[test] + fn empty_tree_falls_back_to_min_load() { + let tree = Arc::new(HashTree::new()); + let policy = CacheAwareZmqPolicy::new( + cfg_default(), + tree, + tokenizer_registry_with_tiny(), + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + // Bump w0's load so min-load picks w1 deterministically. + let _g = w0.load_guard(); + let _g2 = w0.load_guard(); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = br#"{"prompt":"hello world"}"#; + let ctx = SelectionContext::new(&model, Some(body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000"); + } + + /// Tree contains w0's prefix; cache-aware selection picks w0 even + /// though w1 has lower load (the load skew is below the imbalance + /// threshold, so cache wins). + #[test] + fn non_empty_tree_highest_overlap_wins() { + let tree = Arc::new(HashTree::new()); + // Insert w0's tokens into the tree. The tiny tokenizer's hash + // chain for our input is whatever `compute_block_hashes` returns; + // we mimic the policy's hashing path so the test stays + // deterministic against tokenizer changes. + let registry = tokenizer_registry_with_tiny(); + let text = "hello world hello world hello world"; // longer → more blocks + let tok = registry.get("tiny").unwrap(); + let ids = adapter::encode(&tok, text).unwrap(); + let block_size = 4u32; + let hashes = compute_block_hashes(&ids, block_size as usize); + assert!( + !hashes.is_empty(), + "tiny tokenizer must produce at least one full block", + ); + tree.insert(&KvWorkerId::new("http://w0:30000".into(), 0), None, &hashes); + + let policy = CacheAwareZmqPolicy::new( + CacheAwareConfig { + cache_threshold: 0.0, // any match counts + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }, + tree, + registry, + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = serde_json::to_vec(&serde_json::json!({"prompt": text})).unwrap(); + let ctx = SelectionContext::new(&model, Some(&body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w0:30000"); + } + + /// Two workers both hold the prefix; the lower-load one wins. + #[test] + fn tie_break_by_lowest_active_load() { + let tree = Arc::new(HashTree::new()); + let registry = tokenizer_registry_with_tiny(); + let text = "hello world hello world hello world"; + let tok = registry.get("tiny").unwrap(); + let ids = adapter::encode(&tok, text).unwrap(); + let block_size = 4u32; + let hashes = compute_block_hashes(&ids, block_size as usize); + assert!(!hashes.is_empty()); + // Both workers hold the prefix. + tree.insert(&KvWorkerId::new("http://w0:30000".into(), 0), None, &hashes); + tree.insert(&KvWorkerId::new("http://w1:30000".into(), 0), None, &hashes); + + let policy = CacheAwareZmqPolicy::new( + CacheAwareConfig { + cache_threshold: 0.0, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }, + tree, + registry, + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + // Bump w0 to load=1; w1 is at 0 — tiebreak picks w1. + let _g = w0.load_guard(); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = serde_json::to_vec(&serde_json::json!({"prompt": text})).unwrap(); + let ctx = SelectionContext::new(&model, Some(&body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000"); + } + + /// w0 holds the prefix but is heavily overloaded → imbalance branch + /// skips cache-aware and picks w1. + #[test] + fn imbalanced_pool_skips_cache_check() { + let tree = Arc::new(HashTree::new()); + let registry = tokenizer_registry_with_tiny(); + let text = "hello world hello world hello world"; + let tok = registry.get("tiny").unwrap(); + let ids = adapter::encode(&tok, text).unwrap(); + let block_size = 4u32; + let hashes = compute_block_hashes(&ids, block_size as usize); + tree.insert(&KvWorkerId::new("http://w0:30000".into(), 0), None, &hashes); + + let policy = CacheAwareZmqPolicy::new( + CacheAwareConfig { + cache_threshold: 0.0, // would normally always match + balance_abs_threshold: 5, + balance_rel_threshold: 2.0, + }, + tree, + registry, + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + // Bump w0 well above the imbalance threshold. + let mut guards = Vec::new(); + for _ in 0..20 { + guards.push(w0.load_guard()); + } + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = serde_json::to_vec(&serde_json::json!({"prompt": text})).unwrap(); + let ctx = SelectionContext::new(&model, Some(&body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000", "imbalance must dominate"); + } + + /// Tokenizer is missing for the requested model → fall back to + /// min-load (no panic, no error). + #[test] + fn missing_tokenizer_falls_back_to_min_load() { + let tree = Arc::new(HashTree::new()); + let empty_registry = Arc::new(TokenizerRegistry::default()); + let policy = + CacheAwareZmqPolicy::new(cfg_default(), tree, empty_registry, oracle_for_tests(4)); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + let _g = w0.load_guard(); + let _g2 = w0.load_guard(); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = br#"{"prompt":"hello"}"#; + let ctx = SelectionContext::new(&model, Some(body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000"); + } + + /// Missing body → fall back to min-load. + #[test] + fn missing_request_body_falls_back_to_min_load() { + let tree = Arc::new(HashTree::new()); + let policy = CacheAwareZmqPolicy::new( + cfg_default(), + tree, + tokenizer_registry_with_tiny(), + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + let _g = w0.load_guard(); + let _g2 = w0.load_guard(); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let ctx = SelectionContext::new(&model, None); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000"); + } + + /// Body present but no recognizable prompt field → fall back. + #[test] + fn body_without_prompt_field_falls_back_to_min_load() { + let tree = Arc::new(HashTree::new()); + let policy = CacheAwareZmqPolicy::new( + cfg_default(), + tree, + tokenizer_registry_with_tiny(), + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + let _g = w0.load_guard(); + let _g2 = w0.load_guard(); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = br#"{"frobnicate":42}"#; + let ctx = SelectionContext::new(&model, Some(body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000"); + } + + /// Body has a non-text shape that yields zero tokens → fall back. + /// (Tokenizer always returns ≥0 ids; an empty string yields the + /// empty vec, then `compute_block_hashes` returns empty too.) + #[test] + fn empty_text_falls_back_to_min_load() { + let tree = Arc::new(HashTree::new()); + let policy = CacheAwareZmqPolicy::new( + cfg_default(), + tree, + tokenizer_registry_with_tiny(), + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + let _g = w0.load_guard(); + let _g2 = w0.load_guard(); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = br#"{"prompt":""}"#; + let ctx = SelectionContext::new(&model, Some(body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000"); + } + + /// Match rate below the threshold → fall back. Threshold = 0.99 + /// means the tree must match every single block; we insert an + /// UNRELATED chain so the rate is 0. + #[test] + fn low_match_rate_falls_back_to_min_load() { + let tree = Arc::new(HashTree::new()); + // Tree contains a chain unrelated to the test's request. + tree.insert( + &KvWorkerId::new("http://w0:30000".into(), 0), + None, + &[999, 998, 997], + ); + + let policy = CacheAwareZmqPolicy::new( + CacheAwareConfig { + cache_threshold: 0.99, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }, + tree, + tokenizer_registry_with_tiny(), + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + let _g = w0.load_guard(); + let _g2 = w0.load_guard(); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = br#"{"prompt":"hello world hello world hello world"}"#; + let ctx = SelectionContext::new(&model, Some(body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w1:30000"); + } + + /// Chat completions shape with `messages[*].content` string. + #[test] + fn extract_prompt_chat_string_content() { + let body = br#"{"model":"x","messages":[{"role":"user","content":"hello"}]}"#; + let s = CacheAwareZmqPolicy::extract_prompt_text(body).unwrap(); + assert_eq!(s, "hello"); + } + + /// Chat completions shape with multimodal content blocks (text parts). + #[test] + fn extract_prompt_chat_block_content() { + let body = br#"{"messages":[{"role":"user","content":[{"type":"text","text":"hi"},{"type":"image_url","image_url":"x"}]}]}"#; + let s = CacheAwareZmqPolicy::extract_prompt_text(body).unwrap(); + assert_eq!(s, "hi"); + } + + /// `/v1/completions` array form is joined with newlines. + #[test] + fn extract_prompt_completions_array() { + let body = br#"{"prompt":["a","b","c"]}"#; + let s = CacheAwareZmqPolicy::extract_prompt_text(body).unwrap(); + assert_eq!(s, "a\nb\nc"); + } + + /// SGLang native `text` field. + #[test] + fn extract_prompt_sglang_text_field() { + let body = br#"{"text":"abc"}"#; + let s = CacheAwareZmqPolicy::extract_prompt_text(body).unwrap(); + assert_eq!(s, "abc"); + } + + /// Unknown shape → None. + #[test] + fn extract_prompt_unknown_shape_returns_none() { + let body = br#"{"frobnicate":42}"#; + assert!(CacheAwareZmqPolicy::extract_prompt_text(body).is_none()); + } + + /// Lifecycle: removing a worker from the tree via `clear_worker` + /// makes subsequent matches miss; the policy then falls back to + /// min-load. + #[test] + fn lifecycle_clear_worker_removes_overlap() { + let tree = Arc::new(HashTree::new()); + let registry = tokenizer_registry_with_tiny(); + let text = "hello world hello world hello world"; + let tok = registry.get("tiny").unwrap(); + let ids = adapter::encode(&tok, text).unwrap(); + let block_size = 4u32; + let hashes = compute_block_hashes(&ids, block_size as usize); + let kw0 = KvWorkerId::new("http://w0:30000".into(), 0); + tree.insert(&kw0, None, &hashes); + + let policy = CacheAwareZmqPolicy::new( + CacheAwareConfig { + cache_threshold: 0.0, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }, + tree.clone(), + registry, + oracle_for_tests(4), + ); + let w0 = worker("http://w0:30000", "tiny"); + let w1 = worker("http://w1:30000", "tiny"); + let workers = vec![Arc::clone(&w0), Arc::clone(&w1)]; + let model = ModelId("tiny".into()); + let body = serde_json::to_vec(&serde_json::json!({"prompt": text})).unwrap(); + + // Before clear: w0 wins. + let ctx = SelectionContext::new(&model, Some(&body)); + let chosen = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen.url, "http://w0:30000"); + + // After clear: tree no longer attributes the prefix to w0. + tree.clear_worker(&kw0); + // Bump w0's load so min-load fallback distinguishes from w1. + let _g = w0.load_guard(); + let _g2 = w0.load_guard(); + let chosen2 = policy.select(&workers, &ctx).expect("must pick"); + assert_eq!(chosen2.url, "http://w1:30000"); + } +} diff --git a/experimental/sgl-router/src/policies/factory.rs b/experimental/sgl-router/src/policies/factory.rs new file mode 100644 index 000000000000..cd0a34faf8c9 --- /dev/null +++ b/experimental/sgl-router/src/policies/factory.rs @@ -0,0 +1,186 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::config::{Config, ModelConfig, PolicyKind}; +use crate::discovery::ModelId; +use crate::policies::{ + cache_aware_zmq::CacheAwareZmqPolicy, + kv_events::{BlockSizeOracle, HashTree}, + power_of_two::PowerOfTwoChoicesPolicy, + random::RandomPolicy, + round_robin::RoundRobinPolicy, + Policy, PolicyRegistry, +}; +use crate::tokenizer::TokenizerRegistry; +use anyhow::Result; +use std::sync::Arc; + +/// Construct a policy for a single model from its [`ModelConfig`] and the +/// process-shared `HashTree` + `TokenizerRegistry` + `BlockSizeOracle`. +/// +/// The tree, tokenizer registry, and oracle are only consulted by the +/// cache-aware-zmq variant; other policies ignore them. Callers building +/// all policies for the same process pass the same instances to every +/// model. +pub fn build_policy( + model: &ModelConfig, + tree: Arc, + tokenizers: Arc, + block_size_oracle: Arc, +) -> Arc { + match model.policy { + PolicyKind::RoundRobin => Arc::new(RoundRobinPolicy::new()), + PolicyKind::Random => Arc::new(RandomPolicy::new()), + PolicyKind::PowerOfTwo => Arc::new(PowerOfTwoChoicesPolicy::new()), + PolicyKind::CacheAwareZmq => { + let cache_cfg = model.cache_aware.unwrap_or_default(); + Arc::new(CacheAwareZmqPolicy::new( + cache_cfg, + tree, + tokenizers, + block_size_oracle, + )) + } + } +} + +/// Compatibility shim used by tests + non-cache-aware code paths. Builds +/// a policy without wiring the cache-aware dependencies; rejects +/// `CacheAwareZmq` to keep the call sites that don't have a `HashTree` / +/// `TokenizerRegistry` to hand from accidentally compiling. +#[cfg(test)] +pub fn build_policy_kind_only(kind: PolicyKind) -> Arc { + match kind { + PolicyKind::RoundRobin => Arc::new(RoundRobinPolicy::new()), + PolicyKind::Random => Arc::new(RandomPolicy::new()), + PolicyKind::PowerOfTwo => Arc::new(PowerOfTwoChoicesPolicy::new()), + PolicyKind::CacheAwareZmq => { + // Provide an empty tree + empty tokenizer registry + fresh + // oracle so the test policy is constructible. Production + // callers go through `build_policy` with the real + // process-shared instances. + Arc::new(CacheAwareZmqPolicy::new( + crate::config::CacheAwareConfig::default(), + Arc::new(HashTree::new()), + Arc::new(TokenizerRegistry::default()), + BlockSizeOracle::new(), + )) + } + } +} + +pub fn build_registry( + cfg: &Config, + tree: Arc, + tokenizers: Arc, + block_size_oracle: Arc, +) -> Result { + let reg = PolicyRegistry::default(); + for m in &cfg.models { + reg.insert( + ModelId(m.id.clone()), + build_policy( + m, + Arc::clone(&tree), + Arc::clone(&tokenizers), + Arc::clone(&block_size_oracle), + ), + ); + } + Ok(reg) +} + +/// Convenience for tests + non-cache-aware callers: builds a registry with +/// a fresh, empty `HashTree` and an empty `TokenizerRegistry`. The +/// cache-aware-zmq policy will then degrade to min-load (no tokenizer + +/// no worker-published block size → fallback) — which is exactly what +/// the legacy tests assume. +/// +/// Production callers go through [`build_registry`] with the real +/// process-shared instances. +pub fn build_registry_with_defaults(cfg: &Config) -> Result { + build_registry( + cfg, + Arc::new(HashTree::new()), + Arc::new(TokenizerRegistry::default()), + BlockSizeOracle::new(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ModelConfig, ProxyConfig, + ServerConfig, StaticUrlsDiscoveryConfig, + }; + + use crate::config::PolicyKind; + + fn cfg_with_models(policies: &[(&str, PolicyKind)]) -> Config { + Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: Default::default(), + models: policies + .iter() + .map(|(id, p)| ModelConfig { + id: (*id).into(), + tokenizer_path: "/tmp/x".into(), + policy: *p, + circuit_breaker: None, + cache_aware: None, + }) + .collect(), + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + } + } + + #[test] + fn build_policy_kind_only_covers_all_variants() { + // Trivially total — the match is exhaustive over `PolicyKind`. + let _ = build_policy_kind_only(PolicyKind::RoundRobin); + let _ = build_policy_kind_only(PolicyKind::Random); + let _ = build_policy_kind_only(PolicyKind::PowerOfTwo); + let _ = build_policy_kind_only(PolicyKind::CacheAwareZmq); + } + + #[test] + fn registry_assigns_per_model() { + let cfg = cfg_with_models(&[ + ("qwen", PolicyKind::RoundRobin), + ("deepseek", PolicyKind::Random), + ]); + let tree = Arc::new(HashTree::new()); + let tokenizers = Arc::new(TokenizerRegistry::default()); + let reg = build_registry(&cfg, tree, tokenizers, BlockSizeOracle::new()).unwrap(); + assert!(reg.get(&ModelId("qwen".into())).is_some()); + assert!(reg.get(&ModelId("deepseek".into())).is_some()); + assert!(reg.get(&ModelId("missing".into())).is_none()); + } + + #[test] + fn cache_aware_zmq_builds_via_factory() { + let cfg = cfg_with_models(&[("modelA", PolicyKind::CacheAwareZmq)]); + let tree = Arc::new(HashTree::new()); + let tokenizers = Arc::new(TokenizerRegistry::default()); + let reg = build_registry(&cfg, tree, tokenizers, BlockSizeOracle::new()).unwrap(); + let p = reg.get(&ModelId("modelA".into())).unwrap(); + // Down-cast probe via Debug — cheaper than carrying a type-tag + // on the trait. Pinning the debug repr is fine because the field + // name is part of the file's public test surface. + let dbg = format!("{p:?}"); + assert!( + dbg.contains("CacheAwareZmqPolicy"), + "expected CacheAwareZmqPolicy debug repr, got: {dbg}", + ); + } +} diff --git a/experimental/sgl-router/src/policies/kv_events/block_size_oracle.rs b/experimental/sgl-router/src/policies/kv_events/block_size_oracle.rs new file mode 100644 index 000000000000..a2ec7604b55d --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/block_size_oracle.rs @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Process-shared per-(cache-aware-zmq) `block_size`, sourced from the +//! workers themselves. +//! +//! # Why an oracle instead of a config field? +//! +//! `compute_block_hashes` must hash with the **same** block size the +//! worker uses to publish KV-cache events; otherwise every cache-aware +//! lookup misses silently. The worker advertises its `page_size` via +//! `/server_info` (parsed into [`crate::policies::kv_events::EventConfig::block_size`]). +//! Earlier versions of sgl-router carried a static `block_size` field on +//! `CacheAwareConfig`; nothing reconciled it with the worker-reported +//! value, so a mismatch silently destroyed cache-hit routing. +//! +//! Dynamo's design treats `kv_cache_block_size` as a property of the +//! `ModelDeploymentCard` populated by the worker registrar (see +//! `~/dynamo/components/src/dynamo/sglang/register.py`); a mismatch +//! across workers for the same model is rejected loudly +//! (`lib/kv-router/src/standalone_indexer/registry.rs::bail!`). The +//! oracle here is the sgl-router analog — first worker establishes the +//! value, mismatches are refused. +//! +//! # Single oracle vs per-model +//! +//! For now the oracle is process-wide. Realistic deployments use one +//! `page_size` across the cluster, so a single value suffices and +//! mismatches across models indicate misconfiguration the operator +//! should see. A per-model oracle would require threading `ModelId` +//! through `KvEventIndex::add_worker`; that refactor can land later +//! without changing the oracle's public surface. + +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; + +/// First-wins, idempotent block-size publisher. +/// +/// Internally an `AtomicU32` where 0 means "not yet known". Use +/// [`Self::try_set`] to publish a worker-reported value and +/// [`Self::get`] to read at routing time. +#[derive(Debug, Default)] +pub struct BlockSizeOracle { + value: AtomicU32, +} + +/// Returned by [`BlockSizeOracle::try_set`] when the candidate disagrees +/// with the already-established value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlockSizeMismatch { + pub established: u32, + pub candidate: u32, +} + +impl BlockSizeOracle { + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// Returns the established block size, or `None` if no worker has + /// reported one yet. Routing-time consumers (`CacheAwareZmqPolicy`) + /// fall back to min-load when this is `None`, because they cannot + /// hash a prompt without a block size. + pub fn get(&self) -> Option { + let v = self.value.load(Ordering::Relaxed); + if v == 0 { + None + } else { + Some(v) + } + } + + /// Publish a candidate block size. Returns the established value on + /// success (idempotent: same candidate as already set is `Ok`); + /// returns `Err(BlockSizeMismatch)` when the candidate disagrees. + /// + /// `candidate == 0` is rejected because 0 is reserved as the "not + /// yet known" sentinel. + pub fn try_set(&self, candidate: u32) -> Result { + if candidate == 0 { + return Err(BlockSizeMismatch { + established: self.value.load(Ordering::Relaxed), + candidate, + }); + } + match self + .value + .compare_exchange(0, candidate, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => Ok(candidate), + Err(existing) if existing == candidate => Ok(existing), + Err(existing) => Err(BlockSizeMismatch { + established: existing, + candidate, + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fresh_oracle_returns_none() { + let oracle = BlockSizeOracle::new(); + assert_eq!(oracle.get(), None); + } + + #[test] + fn first_set_establishes_the_value() { + let oracle = BlockSizeOracle::new(); + assert_eq!(oracle.try_set(64), Ok(64)); + assert_eq!(oracle.get(), Some(64)); + } + + #[test] + fn matching_set_is_idempotent() { + let oracle = BlockSizeOracle::new(); + assert_eq!(oracle.try_set(64), Ok(64)); + assert_eq!(oracle.try_set(64), Ok(64)); + assert_eq!(oracle.try_set(64), Ok(64)); + assert_eq!(oracle.get(), Some(64)); + } + + #[test] + fn mismatching_set_fails_without_changing_state() { + let oracle = BlockSizeOracle::new(); + oracle.try_set(64).unwrap(); + assert_eq!( + oracle.try_set(128), + Err(BlockSizeMismatch { + established: 64, + candidate: 128 + }) + ); + assert_eq!( + oracle.get(), + Some(64), + "mismatched candidate must not overwrite established value" + ); + } + + #[test] + fn zero_candidate_is_rejected() { + let oracle = BlockSizeOracle::new(); + assert!(oracle.try_set(0).is_err()); + assert_eq!(oracle.get(), None); + } +} diff --git a/experimental/sgl-router/src/policies/kv_events/discovery.rs b/experimental/sgl-router/src/policies/kv_events/discovery.rs new file mode 100644 index 000000000000..0afb038d2fda --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/discovery.rs @@ -0,0 +1,404 @@ +//! Per-worker KV-event publisher discovery. +//! +//! Calls the worker's `/server_info` endpoint (extended on the SGLang +//! Python side) to learn where to connect its ZMQ KV-event publisher. +//! Returns an [`EventConfig`] on success or `Ok(None)` when the worker +//! is reachable but explicitly does not run an event publisher (older +//! SGLang, `kv-events-config` unset, `null` publisher, etc.). +//! +//! # Failure semantics +//! +//! - Network errors and 5xx responses are **transient** and retried +//! inside [`fetch_event_config`] up to [`FETCH_MAX_ATTEMPTS`] with +//! exponential backoff. If every attempt fails, the call returns +//! `Err(_)` so the caller can distinguish "definitely not publishing" +//! (`Ok(None)`) from "we couldn't tell" (`Err`). +//! - 4xx responses are non-retriable (the worker answered +//! authoritatively) and surface as `Err`. +//! - Caller behaviour: [`super::index::KvEventIndex::add_worker`] logs +//! the error and skips subscription, but the worker remains in the +//! broader router registry. Future re-discovery may retry. + +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use serde::Deserialize; +use tracing::{debug, warn}; +use url::Url; + +/// Per-worker KV-event publisher configuration, resolved to something the +/// gateway can directly use to open ZMQ SUB sockets. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EventConfig { + /// The host the gateway should connect to. If the worker reports a + /// wildcard bind host (`*`, `0.0.0.0`, `::`) this is replaced by the + /// host parsed out of the worker URL; otherwise the explicit + /// `endpoint_host` is kept verbatim. + pub host: String, + /// Base port for rank 0. Per-rank port = `port_base + dp_rank`. + pub port_base: u16, + /// ZMQ topic prefix the gateway should SUBSCRIBE to. + pub topic: String, + /// Worker-reported `page_size`. Callers MUST compare against their + /// own configured `block_size`; a mismatch produces silent + /// miscompute since [`super::hash::compute_block_hashes`] is keyed + /// on the caller's value, not on this one. + pub block_size: u32, + /// Number of attention-DP ranks publishing. The gateway opens this + /// many SUB connections (one per rank), skipping any rank whose + /// `port_base + dp_rank` overflows `u16`. + pub dp_size: u32, +} + +/// Default timeout for the `/server_info` introspection request. The +/// worker is on the same network as the gateway in production; 2 seconds +/// is generous and still bounds gateway-startup latency. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(2); + +/// Bounded retry for transient `/server_info` failures. A worker that just +/// booted may need a few hundred ms before its HTTP server accepts +/// requests; retry absorbs the race without permanently disabling +/// cache-aware routing for that worker. +const FETCH_MAX_ATTEMPTS: u32 = 3; +const FETCH_BACKOFF_BASE: Duration = Duration::from_millis(100); + +/// Fetch the worker's KV-event publisher config via `/server_info`. +/// +/// Returns: +/// - `Ok(Some(cfg))` when the worker exposed a usable `kv_events` block. +/// - `Ok(None)` when the worker is **reachable** but explicitly does not +/// expose one (older SGLang, `kv-events-config` unset, `null` +/// publisher, etc.). Cache-aware routing is disabled for that worker. +/// - `Err(_)` when `worker_url` cannot be parsed, OR when every transient +/// attempt failed (network error or 5xx). Caller decides whether to +/// retry; the worker is still added to the registry but cache-aware +/// routing is disabled until a future re-discovery. +pub async fn fetch_event_config( + worker_url: &str, + client: &reqwest::Client, +) -> Result> { + let parsed = + Url::parse(worker_url).map_err(|e| anyhow!("invalid worker_url {worker_url}: {e}"))?; + let worker_host = parsed + .host_str() + .ok_or_else(|| anyhow!("worker_url {worker_url} has no host"))? + .to_owned(); + + let server_info_url = format!("{}/server_info", worker_url.trim_end_matches('/')); + + let body = fetch_with_retry(&server_info_url, worker_url, client).await?; + + let block = match body.kv_events { + Some(b) => b, + None => { + debug!( + worker_url = worker_url, + "kv-events discovery: /server_info has no kv_events block; worker is not publishing" + ); + return Ok(None); + } + }; + + // Wildcard bind hosts mean "any interface" on the worker side — the + // gateway has to connect to a routable address, which it learns from + // the worker URL. + let host = if matches!( + block.endpoint_host.as_str(), + "*" | "0.0.0.0" | "::" | "[::]" + ) { + worker_host + } else { + block.endpoint_host + }; + + Ok(Some(EventConfig { + host, + port_base: block.endpoint_port_base, + topic: block.topic, + block_size: block.block_size, + dp_size: block.dp_size, + })) +} + +/// Issue the `/server_info` request with bounded retry on transient errors +/// (network failures, 5xx). 4xx responses and JSON-parse errors are +/// non-retriable: the worker answered, just not with what we expect. +async fn fetch_with_retry( + server_info_url: &str, + worker_url: &str, + client: &reqwest::Client, +) -> Result { + let mut last_err: Option = None; + let mut delay = FETCH_BACKOFF_BASE; + for attempt in 1..=FETCH_MAX_ATTEMPTS { + match client + .get(server_info_url) + .timeout(DEFAULT_TIMEOUT) + .send() + .await + { + Err(e) => { + last_err = Some(format!("network error: {e}")); + warn!( + worker_url = worker_url, + attempt, + error = %e, + "kv-events discovery: /server_info request failed; will retry" + ); + } + Ok(resp) if resp.status().is_server_error() => { + last_err = Some(format!("server error: {}", resp.status())); + warn!( + worker_url = worker_url, + attempt, + status = resp.status().as_u16(), + "kv-events discovery: /server_info returned 5xx; will retry" + ); + } + Ok(resp) if !resp.status().is_success() => { + // 4xx — worker answered authoritatively, retrying won't help. + return Err(anyhow!( + "/server_info returned {} (non-retriable)", + resp.status() + )); + } + Ok(resp) => { + return resp + .json::() + .await + .map_err(|e| anyhow!("/server_info JSON parse failed: {e}")); + } + } + if attempt < FETCH_MAX_ATTEMPTS { + tokio::time::sleep(delay).await; + delay *= 2; + } + } + Err(anyhow!( + "/server_info failed after {} attempts: {}", + FETCH_MAX_ATTEMPTS, + last_err.unwrap_or_else(|| "unknown".into()), + )) +} + +#[derive(Deserialize)] +struct ServerInfoResponse { + #[serde(default)] + kv_events: Option, +} + +#[derive(Deserialize)] +struct KvEventsBlock { + // `publisher` is captured for forward-compatibility but unused: the + // only publisher implementation supported on the gateway side is + // ZMQ. Keeping the field optional means a future SGLang that adds a + // non-ZMQ publisher string won't fail this deserialize; the + // resulting subscriber will still try to open a ZMQ connection on + // `endpoint_host:endpoint_port_base` and fail visibly there. + #[allow(dead_code)] + #[serde(default)] + publisher: Option, + endpoint_host: String, + endpoint_port_base: u16, + #[serde(default)] + topic: String, + block_size: u32, + dp_size: u32, +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{routing::get, Json, Router}; + use serde_json::{json, Value}; + use std::sync::Arc; + use tokio::net::TcpListener; + use tokio::sync::oneshot; + + /// Spin up a tiny axum server that returns `body` on GET /server_info. + /// Returns the base URL (`http://127.0.0.1:`) and a shutdown handle. + async fn spawn_fake_worker(body: Arc) -> (String, oneshot::Sender<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let body_clone = body.clone(); + let app = Router::new().route( + "/server_info", + get(move || { + let body = body_clone.clone(); + async move { Json((*body).clone()) } + }), + ); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = rx.await; + }) + .await; + }); + (format!("http://127.0.0.1:{port}"), tx) + } + + fn client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .unwrap() + } + + /// Happy path: worker advertises a ZMQ publisher; gateway substitutes + /// `*` with the worker host. + #[tokio::test] + async fn fetch_returns_event_config_when_block_present() { + let body = Arc::new(json!({ + "kv_events": { + "publisher": "zmq", + "endpoint_host": "*", + "endpoint_port_base": 5557, + "topic": "kv", + "block_size": 64, + "dp_size": 2, + } + })); + let (url, _shutdown) = spawn_fake_worker(body).await; + let got = fetch_event_config(&url, &client()).await.unwrap(); + assert_eq!( + got, + Some(EventConfig { + host: "127.0.0.1".to_string(), + port_base: 5557, + topic: "kv".to_string(), + block_size: 64, + dp_size: 2, + }) + ); + } + + /// Worker reports a specific bind host (not wildcard): gateway must + /// honour it instead of overwriting from the URL. + #[tokio::test] + async fn fetch_keeps_explicit_bind_host() { + let body = Arc::new(json!({ + "kv_events": { + "publisher": "zmq", + "endpoint_host": "10.1.2.3", + "endpoint_port_base": 6000, + "topic": "", + "block_size": 128, + "dp_size": 1, + } + })); + let (url, _shutdown) = spawn_fake_worker(body).await; + let got = fetch_event_config(&url, &client()).await.unwrap(); + assert_eq!(got.unwrap().host, "10.1.2.3"); + } + + /// Worker reachable but the `kv_events` field is null / missing: + /// caller should fall back to its static config. + #[tokio::test] + async fn fetch_returns_none_when_block_null() { + let body = Arc::new(json!({ "kv_events": null })); + let (url, _shutdown) = spawn_fake_worker(body).await; + let got = fetch_event_config(&url, &client()).await.unwrap(); + assert!(got.is_none()); + } + + /// Worker is reachable but its `/server_info` response doesn't even + /// have a `kv_events` field (older SGLang). + #[tokio::test] + async fn fetch_returns_none_when_field_absent() { + let body = Arc::new(json!({ "other_stuff": 1 })); + let (url, _shutdown) = spawn_fake_worker(body).await; + let got = fetch_event_config(&url, &client()).await.unwrap(); + assert!(got.is_none()); + } + + /// Connection-refused: no server at the URL. The retry loop exhausts + /// every attempt and propagates `Err`. The caller (KvEventIndex) logs + /// + skips the subscriber so a single flaky worker doesn't poison + /// startup, but the failure remains distinguishable from "worker + /// reachable but not publishing" (`Ok(None)`) so future re-discovery + /// can retry. + #[tokio::test] + async fn fetch_returns_err_on_connection_failure() { + let url = "http://127.0.0.1:1"; // port 1 is reserved / refused + let got = fetch_event_config(url, &client_fast_retry()).await; + assert!(got.is_err(), "expected Err on permanent connect refused"); + } + + /// HTTP client with a short timeout so the connection-failure tests don't + /// pay the full 2s × FETCH_MAX_ATTEMPTS budget. + fn client_fast_retry() -> reqwest::Client { + reqwest::Client::builder() + .timeout(Duration::from_millis(100)) + .build() + .unwrap() + } + + /// Invalid worker URL is the one case we propagate as Err — there's + /// nothing to fall back to and the operator config is broken. + #[tokio::test] + async fn fetch_returns_err_on_invalid_url() { + let got = fetch_event_config("not a url", &client()).await; + assert!(got.is_err()); + } + + /// Multi-DP publisher contract: a worker reporting `dp_size = 8` + /// produces an `EventConfig` with `dp_size = 8` and the base port + /// preserved. The subscriber is responsible for opening 8 SUB + /// sockets at `port_base + 0..8`; discovery just carries the values. + #[tokio::test] + async fn fetch_handles_multi_dp_publisher_dp_size_eight() { + let body = Arc::new(json!({ + "kv_events": { + "publisher": "zmq", + "endpoint_host": "*", + "endpoint_port_base": 5557, + "topic": "kv", + "block_size": 64, + "dp_size": 8, + } + })); + let (url, _shutdown) = spawn_fake_worker(body).await; + let got = fetch_event_config(&url, &client()).await.unwrap().unwrap(); + assert_eq!(got.dp_size, 8); + assert_eq!(got.port_base, 5557); + // Verify the implicit port range fits in u16. + let max_port = u32::from(got.port_base) + got.dp_size - 1; + assert!( + max_port <= u32::from(u16::MAX), + "max per-rank port {max_port} must fit in u16", + ); + } + + /// Documents the discovery-layer contract for ports near the u16 ceiling: + /// discovery does NOT validate `port_base + dp_size` overflow. The + /// subscriber MUST defend against `port_base + dp_rank > u16::MAX` + /// when opening sockets. Pinning this so that a future addition of + /// validation at the discovery layer is a deliberate design change, + /// not an accident. + #[tokio::test] + async fn fetch_accepts_high_port_base_near_u16_max() { + let body = Arc::new(json!({ + "kv_events": { + "publisher": "zmq", + "endpoint_host": "*", + // u16::MAX = 65535. With dp_size = 4, ranks 2 and 3 would + // overflow. Discovery still returns the EventConfig as-is. + "endpoint_port_base": 65533, + "topic": "kv", + "block_size": 64, + "dp_size": 4, + } + })); + let (url, _shutdown) = spawn_fake_worker(body).await; + let got = fetch_event_config(&url, &client()).await.unwrap().unwrap(); + assert_eq!(got.port_base, 65533); + assert_eq!(got.dp_size, 4); + let last_rank = u32::from(got.port_base) + got.dp_size - 1; + assert!( + last_rank > u32::from(u16::MAX), + "test fixture must put the last rank's port past u16::MAX so subscriber-level overflow handling is exercised by its own tests", + ); + } +} diff --git a/experimental/sgl-router/src/policies/kv_events/hash.rs b/experimental/sgl-router/src/policies/kv_events/hash.rs new file mode 100644 index 000000000000..900a49171c3a --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/hash.rs @@ -0,0 +1,259 @@ +//! Block-hash compute matching SGLang's `radix_cache` worker. +//! +//! This is the gateway-side mirror of SGLang's per-page SHA256 chaining used +//! to derive `BlockStored.block_hashes` on workers (Python: +//! `python/sglang/srt/mem_cache/radix_cache.py::hash_page` and +//! `python/sglang/srt/mem_cache/utils.py::hash_str_to_int64`). +//! +//! ### Algorithm +//! +//! For each page (chunk of `block_size` tokens, last page possibly short): +//! 1. Initialize a SHA256 hasher. +//! 2. If a prior page exists, feed the prior page's **full 32-byte SHA256 +//! digest** (raw bytes, not the truncated i64) into the hasher. +//! 3. Feed each token in the page as 4 little-endian unsigned bytes. +//! 4. Take the 32-byte digest as the new "prior" for the next page. +//! 5. Truncate the digest to a signed i64 by reading the first 16 hex chars +//! (top 64 bits) and reinterpreting as signed. +//! +//! ### Why no `parent_hash: Option` argument +//! +//! SGLang's worker chains on the **full 32-byte digest** of the parent block, +//! not on the i64 truncation. An `Option` is lossy — you cannot +//! reconstruct 32 bytes of SHA256 from 64 bits — so accepting one as a +//! "starting point" would silently produce hashes that disagree with the +//! worker. +//! +//! In the gateway we only need to compute hashes for an entire request from +//! scratch (i.e. starting with no parent). That matches the Python emission +//! path where the first page of a freshly-stored node may have a parent block +//! hash for the radix tree key, but the **page-hash computation itself** +//! starts from the parent's **full hex digest** (`node.parent.hash_value[-1]`). +//! For request-side hashing on the routing path, there is no parent, so we +//! expose the "from-scratch" entry point only. +//! +//! ### Bigram mode +//! +//! Not supported in v1. SGLang's bigram mode interleaves overlapping +//! `(t_i, t_{i+1})` pairs into the hash. The gateway does not need this +//! today; if/when it does, add a separate `compute_block_hashes_bigram` rather +//! than complicating the non-bigram fast path. + +use sha2::{Digest, Sha256}; + +/// Compute per-block i64 hashes for a token sequence, matching SGLang's +/// worker emission for a chain that starts with no parent block. +/// +/// The returned `Vec` has `ceil(token_ids.len() / block_size)` entries, +/// each being the i64 truncation (top 64 bits, signed) of the per-page +/// SHA256 digest as defined in [`Self`-module docs](self). +/// +/// # Panics +/// +/// Panics if `block_size == 0`. Callers are expected to validate this once +/// up-front against the worker-published `block_size`; an invalid value is +/// a programmer/config bug, not a runtime input we should swallow. +pub fn compute_block_hashes(token_ids: &[u32], block_size: usize) -> Vec { + assert!(block_size > 0, "block_size must be positive"); + if token_ids.is_empty() { + return Vec::new(); + } + + let n = token_ids.len(); + let num_blocks = n.div_ceil(block_size); + let mut out = Vec::with_capacity(num_blocks); + let mut prior: Option<[u8; 32]> = None; + + let mut start = 0; + while start < n { + let end = (start + block_size).min(n); + let digest = chain_block(prior.as_ref(), &token_ids[start..end]); + out.push(sha256_to_i64(&digest)); + prior = Some(digest); + start = end; + } + + out +} + +/// Hash a single page, optionally chained to a parent block's full 32-byte +/// SHA256 digest. Returns the new 32-byte digest. +#[inline] +fn chain_block(parent_digest: Option<&[u8; 32]>, block_tokens: &[u32]) -> [u8; 32] { + let mut hasher = Sha256::new(); + if let Some(parent) = parent_digest { + hasher.update(parent); + } + for t in block_tokens { + hasher.update(t.to_le_bytes()); + } + hasher.finalize().into() +} + +/// Convert a full 32-byte SHA256 digest to the signed i64 truncation that +/// SGLang publishes on the wire (top 64 bits, big-endian, reinterpreted as +/// signed). +/// +/// Mirrors Python's `hash_str_to_int64`: +/// ```text +/// uint64_val = int(hash_str[:16], 16) +/// return uint64_val - 2**64 if uint64_val >= 2**63 else uint64_val +/// ``` +/// which is equivalent to `i64::from_be_bytes(digest[..8])`. +#[inline] +pub fn sha256_to_i64(digest: &[u8; 32]) -> i64 { + let mut top = [0u8; 8]; + top.copy_from_slice(&digest[..8]); + i64::from_be_bytes(top) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Helper for tests: derive the expected i64 from a list of tokens + /// chained against an optional parent digest. This mirrors `chain_block` + /// but is duplicated here so a regression in the production helper + /// cannot also hide itself in the test oracle. + fn oracle_block_digest(parent: Option<&[u8; 32]>, tokens: &[u32]) -> [u8; 32] { + let mut h = Sha256::new(); + if let Some(p) = parent { + h.update(p); + } + for t in tokens { + h.update(t.to_le_bytes()); + } + h.finalize().into() + } + + fn oracle_i64(digest: &[u8; 32]) -> i64 { + let mut top = [0u8; 8]; + top.copy_from_slice(&digest[..8]); + i64::from_be_bytes(top) + } + + #[test] + fn empty_input_returns_empty_vec() { + assert!(compute_block_hashes(&[], 4).is_empty()); + assert!(compute_block_hashes(&[], 1).is_empty()); + } + + #[test] + #[should_panic(expected = "block_size must be positive")] + fn zero_block_size_panics() { + let _ = compute_block_hashes(&[1, 2, 3], 0); + } + + #[test] + fn single_full_block() { + // Independent oracle: SHA256 of the LE bytes of [1,2,3,4], take top 8 bytes. + let expected_digest = oracle_block_digest(None, &[1, 2, 3, 4]); + let expected_i64 = oracle_i64(&expected_digest); + + let got = compute_block_hashes(&[1, 2, 3, 4], 4); + assert_eq!(got, vec![expected_i64]); + } + + #[test] + fn partial_last_block_chains_against_first_block_digest() { + // 5 tokens, block_size 4 → block 0 = [1,2,3,4], block 1 = [5] chained + // against block 0's full 32-byte digest. + let d0 = oracle_block_digest(None, &[1, 2, 3, 4]); + let d1 = oracle_block_digest(Some(&d0), &[5]); + let expected = vec![oracle_i64(&d0), oracle_i64(&d1)]; + + let got = compute_block_hashes(&[1, 2, 3, 4, 5], 4); + assert_eq!(got, expected); + } + + #[test] + fn multi_block_chain() { + // 8 tokens, block_size 2 → 4 blocks, each chained against the + // previous block's full 32-byte digest. + let tokens: [u32; 8] = [10, 20, 30, 40, 50, 60, 70, 80]; + let d0 = oracle_block_digest(None, &tokens[0..2]); + let d1 = oracle_block_digest(Some(&d0), &tokens[2..4]); + let d2 = oracle_block_digest(Some(&d1), &tokens[4..6]); + let d3 = oracle_block_digest(Some(&d2), &tokens[6..8]); + let expected = vec![ + oracle_i64(&d0), + oracle_i64(&d1), + oracle_i64(&d2), + oracle_i64(&d3), + ]; + + let got = compute_block_hashes(&tokens, 2); + assert_eq!(got, expected); + } + + #[test] + fn sha256_to_i64_handles_top_bit_set() { + // sha256("") = e3b0c44298fc1c14 9afbf4c8996fb924 27ae41e4649b934c a495991b7852b855 + // Top 8 bytes = e3b0c44298fc1c14 → uint64 0xe3b0c44298fc1c14 + // Top bit set → signed value = uint64 - 2**64 = -2039914840885289964 + let digest: [u8; 32] = Sha256::digest(b"").into(); + assert_eq!(sha256_to_i64(&digest), -2039914840885289964_i64); + } + + /// Cross-language goldens: values produced by a Python script that + /// mirrors `radix_cache.hash_page` (non-bigram path) and + /// `mem_cache.utils.hash_str_to_int64`. These are the contract with the + /// SGLang worker and lock down algorithmic equivalence regardless of + /// changes to the Rust-internal helpers. + /// + /// Reproducer (saved temporarily to `/tmp/sglang_hash_oracle.py` during + /// development; not committed): + /// ```python + /// import hashlib + /// def hash_page(prior, toks): + /// h = hashlib.sha256() + /// if prior: + /// h.update(bytes.fromhex(prior)) + /// for t in toks: + /// h.update(int(t).to_bytes(4, "little", signed=False)) + /// return h.hexdigest() + /// def hash_str_to_int64(s): + /// v = int(s[:16], 16) + /// return v - 2**64 if v >= 2**63 else v + /// def chain(tokens, bs): + /// out, prior = [], None + /// for i in range(0, len(tokens), bs): + /// hx = hash_page(prior, tokens[i:i+bs]) + /// out.append(hash_str_to_int64(hx)); prior = hx + /// return out + /// ``` + #[test] + fn cross_language_golden_single_block() { + // Python: chain([1,2,3,4], 4) -> [-3488128144981237669] + let got = compute_block_hashes(&[1, 2, 3, 4], 4); + assert_eq!(got, vec![-3488128144981237669_i64]); + } + + #[test] + fn cross_language_golden_partial_last_block() { + // Python: chain([1,2,3,4,5], 4) + // -> [-3488128144981237669, -3787494577174227566] + let got = compute_block_hashes(&[1, 2, 3, 4, 5], 4); + assert_eq!( + got, + vec![-3488128144981237669_i64, -3787494577174227566_i64] + ); + } + + #[test] + fn cross_language_golden_multi_block() { + // Python: chain([10,20,30,40,50,60,70,80], 2) + // -> [978178666101069530, -895308556211281782, + // -8033692805846017938, 835415944263129316] + let got = compute_block_hashes(&[10, 20, 30, 40, 50, 60, 70, 80], 2); + assert_eq!( + got, + vec![ + 978178666101069530_i64, + -895308556211281782_i64, + -8033692805846017938_i64, + 835415944263129316_i64, + ] + ); + } +} diff --git a/experimental/sgl-router/src/policies/kv_events/index.rs b/experimental/sgl-router/src/policies/kv_events/index.rs new file mode 100644 index 000000000000..c29fa5b71425 --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/index.rs @@ -0,0 +1,682 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Lifecycle bundle for the KV-event index. +//! +//! Couples the three submodules that are independent in their own right but +//! always operate together in production: +//! +//! - [`HashTree`] — the cache-aware routing index keyed by SGLang block hash. +//! - [`KvEventSubscriberRegistry`] — one ZMQ SUB connection per `(worker_url, +//! dp_rank)`. +//! - A pump task that drains [`WorkerEvent`]s from the subscriber and applies +//! them to the tree. +//! +//! `add_worker` / `remove_worker` are driven from the worker manager on every +//! `DiscoveryEvent::Added` / `DiscoveryEvent::Removed`. +//! +//! # Race avoidance +//! +//! The pump runs independently of the lifecycle calls, so an event can sit in +//! the mpsc buffer while `remove_worker` is in progress. To prevent stale +//! events from re-inserting tree state for a worker that was just torn down, +//! [`KvEventIndex`] maintains a `live_workers` set; entries are removed +//! **before** the subscriber tasks are joined, and the pump filters every +//! event through this set before mutating the tree. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; + +use parking_lot::Mutex; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, warn}; + +use super::block_size_oracle::BlockSizeOracle; +use super::discovery::{fetch_event_config, EventConfig}; +use super::subscriber::{KvEventSubscriberRegistry, WorkerEvent}; +use super::tree::{HashTree, KvWorkerId}; +use super::wire::KvCacheEvent; + +/// Channel buffer between the subscriber registry and the pump task. +/// +/// Bounded so a misbehaving publisher cannot exhaust memory. Realistic +/// per-worker event rates are < 1 kHz; a 1024-deep buffer absorbs a +/// half-second burst at 2 kHz before back-pressuring the SUB sockets. +const EVENT_CHANNEL_BUFFER: usize = 1024; + +/// Per-worker bookkeeping kept inside [`KvEventIndex`] so `remove_worker` +/// knows which DP ranks were actually subscribed (not the advertised +/// `dp_size`, which may overflow `u16` and skip ranks). +#[derive(Debug, Clone)] +struct WorkerEntry { + /// DP ranks that were successfully spawned for this worker. Used by + /// `remove_worker` to know which `(url, dp_rank)` cursors and tree + /// states to clear. + dp_ranks: Vec, +} + +/// Bundle of `HashTree` + `KvEventSubscriberRegistry` + pump task. +/// +/// Construct one instance per router process and hand it to the worker +/// manager as `Option>` — `None` disables the cache-aware +/// routing path entirely. +pub struct KvEventIndex { + tree: Arc, + subscribers: Arc, + pump: Mutex>>, + pump_cancel: CancellationToken, + workers: Mutex>, + http: reqwest::Client, + /// Set of currently-attached `(worker_url, dp_rank)` pairs. The pump + /// drops any event whose `worker` is not in this set, so a batch + /// queued by a subscriber that was torn down by `remove_worker` does + /// not re-pollute the tree after `clear_worker` ran. + live_workers: Arc>>, + /// Per-`(worker_url, dp_rank)` last-applied sequence number. The + /// subscriber forwards every batch with no de-dup; this map filters + /// any batch whose `seq` is not strictly greater than the previously + /// applied one. Cleared on `remove_worker` because a re-added worker + /// may legitimately have a fresh publisher whose sequence numbers + /// restart from 1. + cursors: Arc>>, + /// Worker-sourced `page_size` shared with the cache-aware-zmq policy. + /// `add_worker` calls `try_set(cfg.block_size)` so the first worker + /// establishes the value; subsequent workers that disagree are + /// rejected (logged + not subscribed). The policy reads it at routing + /// time to size its `compute_block_hashes` call. + block_size_oracle: Arc, +} + +impl KvEventIndex { + /// Build an empty index and spawn the pump task. + pub fn new() -> Arc { + Self::new_with_http( + reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .expect("default http client builds"), + ) + } + + /// Constructor used by tests so they can supply a custom timeout. + pub fn new_with_http(http: reqwest::Client) -> Arc { + Self::new_with_http_and_oracle(http, BlockSizeOracle::new()) + } + + /// Constructor that lets the caller supply a pre-shared + /// [`BlockSizeOracle`]. Production wires this from `AppContext` so + /// the same oracle the index seeds is the one the cache-aware-zmq + /// policy reads at routing time. Tests use this to pre-populate the + /// oracle and exercise the mismatch-rejection path. + pub fn new_with_http_and_oracle( + http: reqwest::Client, + block_size_oracle: Arc, + ) -> Arc { + let tree = Arc::new(HashTree::new()); + let (tx, rx) = mpsc::channel::(EVENT_CHANNEL_BUFFER); + let subscribers = Arc::new(KvEventSubscriberRegistry::new(tx)); + let cursors: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let live_workers: Arc>> = Arc::new(Mutex::new(HashSet::new())); + let pump_cancel = CancellationToken::new(); + let pump = tokio::spawn(pump_loop( + tree.clone(), + cursors.clone(), + live_workers.clone(), + pump_cancel.clone(), + rx, + )); + Arc::new(Self { + tree, + subscribers, + pump: Mutex::new(Some(pump)), + pump_cancel, + workers: Mutex::new(HashMap::new()), + http, + live_workers, + cursors, + block_size_oracle, + }) + } + + /// Shared accessor for the per-process block-size oracle. The + /// `CacheAwareZmqPolicy` (via [`crate::policies::factory`]) holds the + /// same `Arc` so the value the index seeds is the value the policy + /// hashes against. + pub fn block_size_oracle(&self) -> Arc { + Arc::clone(&self.block_size_oracle) + } + + /// Clone the underlying tree handle for cache-aware selection and + /// metrics. The pump is the sole writer; callers should treat the + /// returned handle as read-only. + pub fn tree(&self) -> Arc { + self.tree.clone() + } + + /// Register a worker. If `preresolved` is `Some`, the caller has + /// already fetched `/server_info` (worker manager path) and we skip + /// the internal HTTP round-trip; otherwise (standalone callers, + /// e.g. integration tests) we fall back to `fetch_event_config`. + /// + /// Opens one ZMQ SUB per advertised DP rank. If the worker is not + /// publishing KV events (older SGLang, opt-out config), this is a + /// logged no-op — the worker still routes via the non-cache-aware + /// policies. + pub async fn add_worker(&self, worker_url: &str, preresolved: Option) { + let cfg: EventConfig = match preresolved { + Some(c) => c, + None => match fetch_event_config(worker_url, &self.http).await { + Ok(Some(c)) => c, + Ok(None) => { + info!( + worker_url = %worker_url, + "kv-events: worker is not publishing; cache-aware routing disabled for this worker", + ); + return; + } + Err(e) => { + warn!( + worker_url = %worker_url, + error = %e, + "kv-events: /server_info introspection failed; skipping subscriber", + ); + return; + } + }, + }; + // Reconcile this worker's `page_size` with the oracle BEFORE + // any subscriber state is created. The first worker establishes + // the value; later workers must agree. A mismatch means the + // router and at least one engine would compute different block + // hashes for the same prompt, silently destroying cache-aware + // routing quality — reject loudly instead. + if let Err(err) = self.block_size_oracle.try_set(cfg.block_size) { + warn!( + worker_url = %worker_url, + established_block_size = err.established, + worker_block_size = err.candidate, + "kv-events: worker page_size disagrees with established block_size; \ + skipping worker — cache-aware routing requires every worker to publish \ + at the same block size", + ); + return; + } + info!( + worker_url = %worker_url, + dp_size = cfg.dp_size, + port_base = cfg.port_base, + block_size = cfg.block_size, + "kv-events: subscribing", + ); + // Compute the DP ranks that will actually be subscribed (skip + // ranks whose port overflows u16; the subscriber will warn on + // each skipped rank). + let port_base_u32 = u32::from(cfg.port_base); + let dp_ranks: Vec = (0..cfg.dp_size) + .filter(|rank| (port_base_u32 + rank) <= u32::from(u16::MAX)) + .collect(); + if dp_ranks.is_empty() { + warn!( + worker_url = %worker_url, + port_base = cfg.port_base, + dp_size = cfg.dp_size, + "kv-events: every advertised rank's port overflows u16; skipping worker", + ); + return; + } + // Mark every rank live BEFORE the subscriber starts so any event + // it queues is accepted by the pump. + { + let mut live = self.live_workers.lock(); + for &rank in &dp_ranks { + live.insert(KvWorkerId { + url: worker_url.to_string(), + dp_rank: rank, + }); + } + } + self.workers.lock().insert( + worker_url.to_string(), + WorkerEntry { + dp_ranks: dp_ranks.clone(), + }, + ); + self.subscribers.add_worker(worker_url, &cfg).await; + } + + /// Tear down a worker's subscribers and clear it from the tree. + /// Idempotent: a remove for a worker that was never added is a no-op. + /// + /// The live-worker entries are dropped **before** the subscriber join, + /// so any event still buffered in the mpsc by the time the pump + /// reaches it is dropped instead of re-inserted into the tree. + pub async fn remove_worker(&self, worker_url: &str) { + let Some(entry) = self.workers.lock().remove(worker_url) else { + return; + }; + let ids: Vec = entry + .dp_ranks + .iter() + .map(|&dp_rank| KvWorkerId { + url: worker_url.to_string(), + dp_rank, + }) + .collect(); + // 1. Mark every rank dead. Any pump-queued events arriving after + // this point will be filtered. + { + let mut live = self.live_workers.lock(); + for id in &ids { + live.remove(id); + } + } + // 2. Cancel and join the per-rank subscriber tasks. No further + // events for these ranks will be queued after this returns. + self.subscribers.remove_worker(worker_url).await; + // 3. Drop each rank's tree state and cursor. Any event already in + // the mpsc buffer at this point will be filtered by the + // live-set check inside the pump. + let mut cursors = self.cursors.lock(); + for id in &ids { + self.tree.clear_worker(id); + cursors.remove(id); + } + } + + /// Number of worker URLs the index is currently subscribed to. The + /// count includes workers whose `/server_info` resolved but excludes + /// any whose discovery returned `Ok(None)` (worker reachable but not + /// publishing) or `Err` (transient discovery failure). Exposed for + /// tests + future metrics; not part of the routing hot path. + pub fn known_worker_count(&self) -> usize { + self.workers.lock().len() + } + + /// Shut down the pump task. Cancels the subscriber registry first so no + /// further events are queued, then cancels the pump so any buffered + /// events are discarded and the task exits promptly. + pub async fn shutdown(&self) { + self.subscribers.shutdown().await; + self.pump_cancel.cancel(); + let handle = self.pump.lock().take(); + if let Some(h) = handle { + // 2s ceiling guards against a pathological tokio runtime + // teardown; under normal operation the pump exits within one + // poll of `pump_cancel.cancelled()`. + match tokio::time::timeout(Duration::from_secs(2), h).await { + Ok(Ok(())) => {} + Ok(Err(e)) => warn!(error = %e, "kv-events pump task did not join cleanly"), + Err(_) => warn!("kv-events pump task did not stop within 2s"), + } + } + } +} + +/// Drain `WorkerEvent`s and apply each batch to the tree. Out-of-order +/// (seq ≤ last_applied) and stale (worker not in `live_workers`) batches +/// are skipped. `PublisherReset` events clear the cursor so a publisher +/// restarting from seq=1 (after sending END_SEQ) is not filtered. +async fn pump_loop( + tree: Arc, + cursors: Arc>>, + live_workers: Arc>>, + cancel: CancellationToken, + mut rx: mpsc::Receiver, +) { + loop { + let ev = tokio::select! { + biased; + _ = cancel.cancelled() => { + info!("kv-events pump: shutdown requested; exiting"); + return; + } + recv = rx.recv() => match recv { + Some(ev) => ev, + None => { + warn!("kv-events pump: receiver closed unexpectedly; exiting"); + return; + } + } + }; + + // Filter events from workers that are no longer attached. This is + // load-bearing: `remove_worker` clears the live set BEFORE joining + // the subscriber task, so any event still buffered when the pump + // reaches it would otherwise re-pollute the tree. + let worker = ev.worker(); + if !live_workers.lock().contains(worker) { + debug!( + worker = ?worker, + "kv-events pump: dropping event from detached worker", + ); + continue; + } + + match ev { + WorkerEvent::PublisherReset { worker } => { + if cursors.lock().remove(&worker).is_some() { + info!( + worker = ?worker, + "kv-events pump: publisher reset; cursor cleared", + ); + } + } + WorkerEvent::Batch { worker, seq, batch } => { + let prev = cursors.lock().get(&worker).copied(); + if let Some(p) = prev { + if seq <= p { + debug!( + worker = ?worker, + seq, + last_applied = p, + "kv-events pump: out-of-order batch; skipping", + ); + continue; + } + } + for event in &batch.events { + match event { + KvCacheEvent::BlockStored(b) => { + tree.insert(&worker, b.parent_block_hash, &b.block_hashes); + } + KvCacheEvent::BlockRemoved(b) => { + tree.remove(&worker, &b.block_hashes); + } + KvCacheEvent::AllBlocksCleared => { + tree.clear_worker(&worker); + } + } + } + cursors.lock().insert(worker, seq); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::policies::kv_events::wire::{BlockRemoved, BlockStored, KvEventBatch}; + + fn worker_id(url: &str, rank: u32) -> KvWorkerId { + KvWorkerId { + url: url.into(), + dp_rank: rank, + } + } + + fn batch(events: Vec) -> KvEventBatch { + KvEventBatch { + ts: 0.0, + events, + attn_dp_rank: None, + } + } + + /// Bundle of plumbing returned by `spawn_pump` so individual tests + /// can destructure just the bits they need. + struct PumpHarness { + tree: Arc, + cursors: Arc>>, + #[allow(dead_code)] + live_set: Arc>>, + #[allow(dead_code)] + cancel: CancellationToken, + tx: mpsc::Sender, + pump: JoinHandle<()>, + } + + /// Build a tree + cursors + live-set wired through `pump_loop` with + /// the given workers pre-marked live. + fn spawn_pump(live: &[KvWorkerId]) -> PumpHarness { + let tree = Arc::new(HashTree::new()); + let cursors = Arc::new(Mutex::new(HashMap::new())); + let live_set: Arc>> = + Arc::new(Mutex::new(live.iter().cloned().collect())); + let cancel = CancellationToken::new(); + let (tx, rx) = mpsc::channel(4); + let pump = tokio::spawn(pump_loop( + tree.clone(), + cursors.clone(), + live_set.clone(), + cancel.clone(), + rx, + )); + PumpHarness { + tree, + cursors, + live_set, + cancel, + tx, + pump, + } + } + + /// Direct test of the pump loop's tree application — no sockets. + #[tokio::test] + async fn pump_applies_block_stored_to_tree() { + let id = worker_id("http://w1", 0); + let h = spawn_pump(std::slice::from_ref(&id)); + let (tree, tx, pump) = (h.tree, h.tx, h.pump); + + tx.send(WorkerEvent::Batch { + worker: id.clone(), + seq: 1, + batch: batch(vec![KvCacheEvent::BlockStored(BlockStored { + parent_block_hash: None, + block_hashes: vec![10, 20, 30], + token_ids: vec![], + block_size: 64, + lora_id: None, + medium: None, + })]), + }) + .await + .unwrap(); + drop(tx); + // Don't cancel — let rx.recv() return None naturally so any + // queued events drain first. (The pump's `biased` select would + // otherwise preempt unprocessed events on cancel.) + pump.await.unwrap(); + + let m = tree.match_prefix(None, &[10, 20, 30]); + assert_eq!(m.matched_blocks, 3); + assert!(m.workers.contains(&id), "tree must hold the worker"); + } + + /// Out-of-order seq is filtered: a batch with seq <= last_applied is + /// dropped silently and does not mutate the tree. + #[tokio::test] + async fn pump_filters_out_of_order_seq() { + let id = worker_id("http://w1", 0); + let h = spawn_pump(std::slice::from_ref(&id)); + let (tree, cursors, tx, pump) = (h.tree, h.cursors, h.tx, h.pump); + + // Apply seq=5 with block 10. + tx.send(WorkerEvent::Batch { + worker: id.clone(), + seq: 5, + batch: batch(vec![KvCacheEvent::BlockStored(BlockStored { + parent_block_hash: None, + block_hashes: vec![10], + token_ids: vec![], + block_size: 64, + lora_id: None, + medium: None, + })]), + }) + .await + .unwrap(); + // Then a duplicate-style seq=3 that tries to remove block 10. Must + // be dropped. + tx.send(WorkerEvent::Batch { + worker: id.clone(), + seq: 3, + batch: batch(vec![KvCacheEvent::BlockRemoved(BlockRemoved { + block_hashes: vec![10], + medium: None, + })]), + }) + .await + .unwrap(); + drop(tx); + // Don't cancel — let rx.recv() return None naturally so any + // queued events drain first. (The pump's `biased` select would + // otherwise preempt unprocessed events on cancel.) + pump.await.unwrap(); + + let m = tree.match_prefix(None, &[10]); + assert_eq!( + m.matched_blocks, 1, + "out-of-order remove must not undo the prior insert", + ); + assert_eq!(cursors.lock().get(&id).copied(), Some(5)); + } + + /// AllBlocksCleared wipes the worker's tree state entirely. + #[tokio::test] + async fn pump_handles_all_blocks_cleared() { + let id = worker_id("http://w1", 0); + let h = spawn_pump(std::slice::from_ref(&id)); + let (tree, tx, pump) = (h.tree, h.tx, h.pump); + + tx.send(WorkerEvent::Batch { + worker: id.clone(), + seq: 1, + batch: batch(vec![KvCacheEvent::BlockStored(BlockStored { + parent_block_hash: None, + block_hashes: vec![1, 2], + token_ids: vec![], + block_size: 64, + lora_id: None, + medium: None, + })]), + }) + .await + .unwrap(); + tx.send(WorkerEvent::Batch { + worker: id.clone(), + seq: 2, + batch: batch(vec![KvCacheEvent::AllBlocksCleared]), + }) + .await + .unwrap(); + drop(tx); + // Don't cancel — let rx.recv() return None naturally so any + // queued events drain first. (The pump's `biased` select would + // otherwise preempt unprocessed events on cancel.) + pump.await.unwrap(); + + let m = tree.match_prefix(None, &[1, 2]); + assert_eq!( + m.matched_blocks, 0, + "AllBlocksCleared must purge the worker" + ); + } + + /// The pump drops events whose worker is not in `live_workers`. This + /// is the safety net against the remove-then-pump race: an event + /// queued before `remove_worker` clears the live set must not mutate + /// the tree. + #[tokio::test] + async fn pump_drops_events_from_detached_workers() { + let live_id = worker_id("http://live", 0); + let dead_id = worker_id("http://dead", 0); + let h = spawn_pump(std::slice::from_ref(&live_id)); + let (tree, tx, pump) = (h.tree, h.tx, h.pump); + + // Event from a worker that was never added (or was already + // removed). Must be dropped. + tx.send(WorkerEvent::Batch { + worker: dead_id.clone(), + seq: 1, + batch: batch(vec![KvCacheEvent::BlockStored(BlockStored { + parent_block_hash: None, + block_hashes: vec![42], + token_ids: vec![], + block_size: 64, + lora_id: None, + medium: None, + })]), + }) + .await + .unwrap(); + // Sanity: a live event still applies. + tx.send(WorkerEvent::Batch { + worker: live_id.clone(), + seq: 1, + batch: batch(vec![KvCacheEvent::BlockStored(BlockStored { + parent_block_hash: None, + block_hashes: vec![99], + token_ids: vec![], + block_size: 64, + lora_id: None, + medium: None, + })]), + }) + .await + .unwrap(); + drop(tx); + // Don't cancel — let rx.recv() return None naturally so any + // queued events drain first. (The pump's `biased` select would + // otherwise preempt unprocessed events on cancel.) + pump.await.unwrap(); + + assert_eq!(tree.match_prefix(None, &[42]).matched_blocks, 0); + assert_eq!(tree.match_prefix(None, &[99]).matched_blocks, 1); + } + + /// `add_worker` must reject a worker whose `EventConfig.block_size` + /// disagrees with the previously-established oracle value. The + /// router cannot hash prompts simultaneously at two block sizes; + /// silently accepting the mismatched worker would destroy + /// cache-aware routing quality for every request. + #[tokio::test] + async fn add_worker_rejects_block_size_mismatch() { + let index = KvEventIndex::new(); + // First worker establishes block_size=64 via the oracle. + index.block_size_oracle().try_set(64).unwrap(); + + let bad_cfg = EventConfig { + host: "127.0.0.1".into(), + port_base: 30100, + topic: String::new(), + block_size: 128, + dp_size: 1, + }; + index + .add_worker("http://127.0.0.1:30100", Some(bad_cfg)) + .await; + assert_eq!( + index.known_worker_count(), + 0, + "mismatched worker must not be registered" + ); + index.shutdown().await; + } + + #[tokio::test] + async fn add_worker_seeds_oracle_with_first_block_size() { + // Without any prior priming, the first worker through `add_worker` + // should publish its `EventConfig.block_size` into the oracle so + // subsequent matching workers reconcile and mismatched ones fail. + let index = KvEventIndex::new(); + assert_eq!(index.block_size_oracle().get(), None); + + // A dp_size=0 cfg short-circuits before the subscriber spawn but + // still runs through the block-size validation. + let cfg = EventConfig { + host: "127.0.0.1".into(), + port_base: 30200, + topic: String::new(), + block_size: 64, + dp_size: 0, + }; + index.add_worker("http://127.0.0.1:30200", Some(cfg)).await; + assert_eq!(index.block_size_oracle().get(), Some(64)); + index.shutdown().await; + } +} diff --git a/experimental/sgl-router/src/policies/kv_events/mod.rs b/experimental/sgl-router/src/policies/kv_events/mod.rs new file mode 100644 index 000000000000..4f76dd89a5c9 --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/mod.rs @@ -0,0 +1,33 @@ +//! ZMQ-based KV-cache event indexer for cache-aware routing. +//! +//! Decodes the msgpack wire format emitted by SGLang's `ZmqEventPublisher` +//! (see `python/sglang/srt/disaggregation/kv_events.py`) and maintains the +//! router-side index used for cache-aware request routing. +//! +//! # Submodules +//! +//! - [`wire`] — msgpack types and [`decode_event_batch`]; the contract +//! with the SGLang publisher. Pure decoding; no I/O. +//! - [`hash`] — block-hash compute mirroring SGLang `RadixKey.hash_page`. +//! - [`tree`] — hash-keyed radix tree consumed by the routing path. +//! - [`subscriber`] — per-worker ZMQ SUB tasks. +//! - [`discovery`] — `/server_info` parse → publisher endpoint. +//! - [`index`] — public façade bundling the tree + subscribers + pump. + +pub mod block_size_oracle; +pub mod discovery; +pub mod hash; +pub mod index; +pub mod subscriber; +pub mod tree; +pub mod wire; + +pub use block_size_oracle::BlockSizeOracle; +pub use discovery::{fetch_event_config, EventConfig}; +pub use hash::{compute_block_hashes, sha256_to_i64}; +pub use index::KvEventIndex; +pub use subscriber::{KvEventSubscriberRegistry, WorkerEvent}; +pub use tree::{HashTree, KvWorkerId, MatchResult}; +pub use wire::{ + decode_event_batch, BlockRemoved, BlockStored, DecodeError, KvCacheEvent, KvEventBatch, +}; diff --git a/experimental/sgl-router/src/policies/kv_events/subscriber.rs b/experimental/sgl-router/src/policies/kv_events/subscriber.rs new file mode 100644 index 000000000000..1c1de438e2d7 --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/subscriber.rs @@ -0,0 +1,1355 @@ +//! Per-worker, per-DP-rank ZMQ subscriber for SGLang's `ZmqEventPublisher`. +//! +//! This module owns the I/O plumbing between SGLang workers (which publish +//! KV-cache events on a PUB socket — see +//! `python/sglang/srt/disaggregation/kv_events.py`) and the in-memory hash +//! tree consumed by [`super::index::KvEventIndex`]. Each `(worker_url, +//! dp_rank)` pair gets its own SUB socket on its own tokio task, decodes +//! msgpack batches via [`super::wire`], and forwards [`WorkerEvent`]s to +//! a shared mpsc channel. +//! +//! # Wire format (3-frame multipart) +//! +//! Frames published by SGLang: +//! 1. `topic_bytes` — empty by default, present even when empty. +//! 2. `seq_bytes` — 8-byte big-endian signed `i64`. The publisher emits a +//! `-1` sentinel (`ZmqEventPublisher.END_SEQ`) on its replay DEALER +//! socket; we defensively recognise the same value on the PUB stream +//! and surface it as a [`WorkerEvent::PublisherReset`] so the +//! downstream pump can clear its cursor before a reconnecting publisher +//! restarts from seq=1. +//! 3. `payload` — msgpack-encoded [`KvEventBatch`]. +//! +//! # Endpoint construction +//! +//! Each call to [`KvEventSubscriberRegistry::add_worker`] takes an +//! [`EventConfig`] describing where the worker publishes: +//! `tcp://{cfg.host}:{cfg.port_base + dp_rank}` per rank in +//! `0..cfg.dp_size`. The host comes from the worker's `/server_info` +//! introspection in production (so wildcard bind hosts resolve to the +//! gateway-routable address) or from the worker URL as a fallback. +//! +//! # Reconnect +//! +//! `zeromq::SubSocket::connect` already spawns a background reconnection +//! task that re-sends our subscriptions on every reconnect, so we do not +//! need an outer reconnect loop. The initial `connect` + `subscribe` is +//! wrapped in a bounded exponential-backoff retry so a worker that just +//! booted (publisher not yet bound) doesn't permanently disable its +//! subscriber. Errors surfaced from `recv()` are logged and the task +//! continues; after [`RECV_ERROR_CEILING`] consecutive errors the task +//! exits with an `error!` log so the silent-stall failure mode is +//! detectable. +//! +//! # Ordering +//! +//! Events for one `(worker, dp_rank)` flow through one task and use one +//! mpsc sender — order is preserved per-worker. Order across DP ranks (or +//! across workers) is **not** preserved; downstream consumers must not +//! depend on it. +//! +//! # Backpressure +//! +//! The per-worker task `await`s `tx.send()` and will not consume new ZMQ +//! messages while the channel is full. ZMQ's HWM (configured by the +//! publisher) takes effect upstream — events are dropped at the publisher, +//! not buffered in the subscriber. Tune the `tx` channel buffer to absorb +//! expected event-batch bursts. Backpressure is per-worker: a slow consumer +//! for one worker stalls only that worker's events, not others. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::{mpsc, Mutex}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, trace, warn}; +use zeromq::{Socket, SocketRecv, SubSocket, ZmqMessage}; + +use super::discovery::EventConfig; +use super::tree::KvWorkerId; +use super::wire::{decode_event_batch, KvEventBatch}; + +/// Maximum number of consecutive `recv()` errors before the subscriber +/// gives up and exits its task. ZMQ's internal reconnect handles transient +/// network errors, so a stream of consecutive failures means the socket is +/// dead from our perspective; spinning forever masks the failure. +const RECV_ERROR_CEILING: u32 = 64; + +/// Bounded retry configuration for the initial connect + subscribe handshake. +/// A worker that just booted may need a few hundred ms before its PUB socket +/// accepts connections; this absorbs the race. +const CONNECT_MAX_ATTEMPTS: u32 = 5; +const CONNECT_BACKOFF_BASE: Duration = Duration::from_millis(50); +const CONNECT_BACKOFF_CAP: Duration = Duration::from_secs(2); + +/// Sentinel sequence number meaning "publisher is shutting down". Mirrors +/// `ZmqEventPublisher.END_SEQ = (-1).to_bytes(8, 'big', signed=True)`. +/// SGLang's authoritative emission is on the replay DEALER socket +/// (`_service_replay`); we accept the same sentinel on the PUB stream as +/// defense in depth so a future publisher that does broadcast a shutdown +/// signal is handled correctly. +const END_SEQ_SENTINEL: i64 = -1; + +/// Message forwarded from a per-worker subscriber task to the pump. +#[derive(Debug)] +pub enum WorkerEvent { + /// A normal decoded event batch. + Batch { + /// Identity of the SGLang worker (DP rank) that produced this batch. + worker: KvWorkerId, + /// 8-byte big-endian sequence number from the publisher's monotonic + /// counter. Useful for replay / gap detection downstream. + seq: i64, + /// Decoded batch payload. + batch: KvEventBatch, + }, + /// The publisher emitted its `END_SEQ` (-1) sentinel, signalling + /// shutdown. A re-connecting publisher will restart its sequence + /// counter from 1; the pump uses this to reset the cursor so those + /// fresh events are not filtered as out-of-order. + PublisherReset { worker: KvWorkerId }, +} + +impl WorkerEvent { + /// The worker that produced this event, regardless of variant. + pub fn worker(&self) -> &KvWorkerId { + match self { + Self::Batch { worker, .. } => worker, + Self::PublisherReset { worker } => worker, + } + } +} + +/// Internal handle for one running per-(worker, dp_rank) subscriber task. +struct SubscriberHandle { + cancel: CancellationToken, + join: JoinHandle<()>, +} + +/// Shared inner state for [`KvEventSubscriberRegistry`]. +struct Inner { + tx: mpsc::Sender, + /// Keyed by `(worker_url, dp_rank)`. Behind a `tokio::sync::Mutex` + /// because [`KvEventSubscriberRegistry::remove_worker`] and + /// [`KvEventSubscriberRegistry::shutdown`] await join handles while + /// holding the lock conceptually — we drop the lock before awaiting, + /// but using a tokio mutex avoids accidental blocking-mutex misuse if + /// the implementation evolves. + handles: Mutex>, +} + +/// Owns one ZMQ SUB connection per `(worker_url, dp_rank)`. Forwards +/// decoded batches to a tokio mpsc channel supplied at construction time. +pub struct KvEventSubscriberRegistry { + inner: Arc, +} + +impl KvEventSubscriberRegistry { + /// Build an empty registry. `tx` is where decoded events flow out; + /// the channel buffer capacity is the caller's choice. + pub fn new(tx: mpsc::Sender) -> Self { + Self { + inner: Arc::new(Inner { + tx, + handles: Mutex::new(HashMap::new()), + }), + } + } + + /// Open one SUB connection per `dp_rank` in `0..cfg.dp_size`, + /// connecting to `tcp://{cfg.host}:{cfg.port_base + dp_rank}`. Spawns + /// background tasks. Idempotent: a second `add_worker` for the same + /// `(worker_url, dp_rank)` pair is a no-op. + /// + /// `worker_url` is the HTTP URL the gateway uses for routing (e.g., + /// `"http://10.0.0.1:30000"`). It serves as the keying identity in the + /// registry but the actual ZMQ endpoint comes from `cfg` — the policy + /// layer is expected to have learned `cfg` from the worker's + /// `/server_info` introspection (or filled it from a global fallback). + /// + /// # Errors + /// + /// If `cfg.port_base + dp_rank` overflows `u16`, that rank is skipped + /// with a `warn!` log and the remaining ranks proceed. + pub async fn add_worker(&self, worker_url: &str, cfg: &EventConfig) { + let mut handles = self.inner.handles.lock().await; + for dp_rank in 0..cfg.dp_size { + let id = KvWorkerId { + url: worker_url.to_string(), + dp_rank, + }; + if handles.contains_key(&id) { + debug!( + worker_url = %worker_url, + dp_rank, + "subscriber already registered; skipping" + ); + continue; + } + let port = match u16::try_from(cfg.port_base as u32 + dp_rank) { + Ok(p) => p, + Err(_) => { + warn!( + worker_url = %worker_url, + dp_rank, + port_base = cfg.port_base, + "ZMQ event port overflows u16; skipping this rank" + ); + continue; + } + }; + let endpoint = format!("tcp://{}:{}", cfg.host, port); + let cancel = CancellationToken::new(); + let join = spawn_subscriber_task( + id.clone(), + endpoint, + cfg.topic.clone(), + self.inner.tx.clone(), + cancel.clone(), + ); + handles.insert(id, SubscriberHandle { cancel, join }); + } + } + + /// Cancel all subscribers for `worker_url` and await their shutdown. + pub async fn remove_worker(&self, worker_url: &str) { + let drained: Vec = { + let mut handles = self.inner.handles.lock().await; + // Pull out every entry whose URL matches; leave the others. + let to_drop: Vec = handles + .keys() + .filter(|k| k.url == worker_url) + .cloned() + .collect(); + to_drop + .into_iter() + .filter_map(|k| handles.remove(&k)) + .collect() + }; + for h in drained { + h.cancel.cancel(); + // A panicked task surfaces here; we log and continue so one + // poisoned subscriber cannot stall the registry. + if let Err(e) = h.join.await { + warn!( + worker_url = %worker_url, + error = %e, + "subscriber task did not join cleanly" + ); + } + } + } + + /// Sync cancellation: triggers every per-worker token without awaiting + /// the join handles. Use this when you cannot `.await` (e.g., from + /// `Drop`). After calling this, the subscriber tasks will exit on their + /// next yield point. Subscriptions and ZMQ sockets are released by + /// tokio task cleanup. + /// + /// If `try_lock` fails, the registry is mid-mutation elsewhere + /// (`shutdown`, `add_worker`, `remove_worker`); the cancel is + /// redundant in that case so we drop the call. + pub fn cancel_all(&self) { + if let Ok(handles) = self.inner.handles.try_lock() { + for h in handles.values() { + h.cancel.cancel(); + } + } + } + + /// Cancel everything and await shutdown. Caller is responsible for + /// draining any remaining events on the receiver side. + pub async fn shutdown(&self) { + let drained: Vec<(KvWorkerId, SubscriberHandle)> = { + let mut handles = self.inner.handles.lock().await; + handles.drain().collect() + }; + for (id, h) in drained { + h.cancel.cancel(); + if let Err(e) = h.join.await { + warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + error = %e, + "subscriber task did not join cleanly during shutdown" + ); + } + } + } +} + +/// Spawn the background task that owns one SUB socket and forwards +/// decoded batches. +fn spawn_subscriber_task( + id: KvWorkerId, + endpoint: String, + topic: String, + tx: mpsc::Sender, + cancel: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + run_subscriber(id, endpoint, topic, tx, cancel).await; + }) +} + +/// Inner subscriber loop. Returns when: +/// * the cancellation token fires, OR +/// * the downstream mpsc receiver is dropped, OR +/// * the initial connect/subscribe fails after [`CONNECT_MAX_ATTEMPTS`] +/// attempts with exponential backoff, OR +/// * `recv()` returns errors [`RECV_ERROR_CEILING`] times in a row +/// (escalated to `error!` so the silent stall is detectable). +async fn run_subscriber( + id: KvWorkerId, + endpoint: String, + topic: String, + tx: mpsc::Sender, + cancel: CancellationToken, +) { + debug!( + worker_url = %id.url, + dp_rank = id.dp_rank, + endpoint = %endpoint, + topic = %topic, + "starting kv-event subscriber" + ); + + let mut sub = match connect_with_backoff(&id, &endpoint, &topic, &cancel).await { + Some(s) => s, + None => return, + }; + + let mut errors_in_a_row = 0u32; + loop { + tokio::select! { + biased; + _ = cancel.cancelled() => { + debug!( + worker_url = %id.url, + dp_rank = id.dp_rank, + "subscriber cancelled" + ); + return; + } + res = sub.recv() => { + match res { + Ok(msg) => { + errors_in_a_row = 0; + if let Some(event) = decode_message(&id, msg) { + if tx.send(event).await.is_err() { + // The pump (or the entire index) is gone. + // This is unexpected mid-stream; warn so + // operators see it. + warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + "downstream mpsc receiver dropped; exiting" + ); + return; + } + } + } + Err(e) => { + errors_in_a_row += 1; + if errors_in_a_row >= RECV_ERROR_CEILING { + error!( + worker_url = %id.url, + dp_rank = id.dp_rank, + endpoint = %endpoint, + error = %e, + consecutive_errors = errors_in_a_row, + "SUB socket has produced {RECV_ERROR_CEILING} consecutive recv errors; giving up on this subscriber" + ); + return; + } + // SubSocket auto-reconnects internally; transient + // errors should resume once a new peer attaches. + warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + error = %e, + consecutive_errors = errors_in_a_row, + "recv error from SUB socket; continuing" + ); + tokio::task::yield_now().await; + } + } + } + } + } +} + +/// Open a `SubSocket`, connect to `endpoint`, and subscribe to the +/// supplied `topic` prefix (empty string = receive every message, +/// matching the prior all-topics behavior). +/// +/// Retries with exponential backoff up to [`CONNECT_MAX_ATTEMPTS`] times +/// so a worker that just booted (publisher not yet bound) doesn't +/// permanently disable its KV-event subscriber. +/// +/// Returns `None` if cancelled or if every attempt fails. All operations +/// are guarded by the cancellation token so shutdown is not delayed by +/// the backoff. +async fn connect_with_backoff( + id: &KvWorkerId, + endpoint: &str, + topic: &str, + cancel: &CancellationToken, +) -> Option { + let mut delay = CONNECT_BACKOFF_BASE; + for attempt in 1..=CONNECT_MAX_ATTEMPTS { + let mut sub = SubSocket::new(); + let connect_res = tokio::select! { + _ = cancel.cancelled() => { + debug!(worker_url = %id.url, dp_rank = id.dp_rank, "cancelled before connect"); + return None; + } + res = sub.connect(endpoint) => res, + }; + if let Err(e) = connect_res { + warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + endpoint = %endpoint, + attempt, + error = %e, + "kv-events: connect SUB socket failed; retrying" + ); + } else { + let subscribe_res = tokio::select! { + _ = cancel.cancelled() => { + debug!(worker_url = %id.url, dp_rank = id.dp_rank, "cancelled before subscribe"); + return None; + } + res = sub.subscribe(topic) => res, + }; + match subscribe_res { + Ok(()) => return Some(sub), + Err(e) => warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + endpoint = %endpoint, + attempt, + topic = %topic, + error = %e, + "kv-events: SUB subscribe failed; retrying" + ), + } + } + if attempt == CONNECT_MAX_ATTEMPTS { + break; + } + tokio::select! { + _ = cancel.cancelled() => { + debug!(worker_url = %id.url, dp_rank = id.dp_rank, "cancelled during connect backoff"); + return None; + } + _ = tokio::time::sleep(delay) => {} + } + delay = (delay * 2).min(CONNECT_BACKOFF_CAP); + } + error!( + worker_url = %id.url, + dp_rank = id.dp_rank, + endpoint = %endpoint, + attempts = CONNECT_MAX_ATTEMPTS, + "kv-events: gave up establishing SUB socket after {CONNECT_MAX_ATTEMPTS} attempts; this worker's cache-aware routing is disabled until next add_worker" + ); + None +} + +/// Validate, parse, and decode a single 3-frame multipart ZMQ message. +/// Returns `None` (with logging) for any non-event input (bad frame +/// count, sentinel sequence, or msgpack decode error). +fn decode_message(id: &KvWorkerId, msg: ZmqMessage) -> Option { + if msg.len() != 3 { + warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + frames = msg.len(), + "dropping ZMQ message with unexpected frame count (expected 3)" + ); + return None; + } + + // Frame 0 is the topic; we don't use it. Frame 1 is the BE i64 seq; + // frame 2 is the msgpack payload. The `len() == 3` guard above means + // these indices are always valid, but `?` cleanly bails out if a + // future change drops the guard. + let seq_frame = msg.get(1)?; + let payload = msg.get(2)?; + + // Decode the 8-byte BE seq. Frames smaller or larger than 8 bytes + // mean a malformed publisher; log and drop. + let seq_bytes: [u8; 8] = match seq_frame.as_ref().try_into() { + Ok(b) => b, + Err(_) => { + warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + seq_len = seq_frame.len(), + "dropping message with non-8-byte sequence frame" + ); + return None; + } + }; + let seq = i64::from_be_bytes(seq_bytes); + + if seq == END_SEQ_SENTINEL { + info!( + worker_url = %id.url, + dp_rank = id.dp_rank, + "publisher signalled shutdown (END_SEQ); forwarding cursor reset" + ); + return Some(WorkerEvent::PublisherReset { worker: id.clone() }); + } + + let batch = match decode_event_batch(payload.as_ref()) { + Ok(b) => b, + Err(e) => { + warn!( + worker_url = %id.url, + dp_rank = id.dp_rank, + seq, + error = %e, + "failed to decode KV event batch payload; dropping" + ); + return None; + } + }; + + trace!( + worker_url = %id.url, + dp_rank = id.dp_rank, + seq, + n_events = batch.events.len(), + "decoded KV event batch" + ); + + Some(WorkerEvent::Batch { + worker: id.clone(), + seq, + batch, + }) +} + +/// Pull the host out of a routing URL like `http://10.0.0.1:30000` or +/// `https://[::1]:30000`. Falls back to `None` for inputs the `url` crate +/// cannot parse. +/// +/// Test-only helper for fabricating [`EventConfig`]s from a worker URL. +#[cfg(test)] +fn extract_host(worker_url: &str) -> Option { + let parsed = url::Url::parse(worker_url).ok()?; + parsed.host_str().map(|s| s.to_string()) +} + +// --------------------------------------------------------------------------- +// Tests — bind real PUB sockets to ephemeral ports and confirm the +// subscriber wires data through correctly. All tests are localhost-only and +// use OS-assigned ports so they can run in parallel without conflict. +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + use std::time::Duration; + + use bytes::Bytes; + use tokio::time::timeout; + use zeromq::{Endpoint, PubSocket, Socket, SocketSend, ZmqMessage}; + + use crate::policies::kv_events::wire::KvCacheEvent; + + mod helpers { + use super::*; + use rmp::encode as mp; + + /// Bind a PUB socket to an OS-assigned localhost port and return + /// `(socket, port)`. + pub async fn make_pub_bound() -> (PubSocket, u16) { + let mut sock = PubSocket::new(); + let endpoint = sock + .bind("tcp://127.0.0.1:0") + .await + .expect("bind PUB socket"); + let port = match endpoint { + Endpoint::Tcp(_, p) => p, + other => panic!("unexpected endpoint: {other:?}"), + }; + (sock, port) + } + + /// Build a minimal [`EventConfig`] for test fixtures: take the host + /// from `worker_url` (matches the pre-discovery behavior) and fill + /// the rest with reasonable defaults. + pub fn cfg_for(worker_url: &str, port_base: u16, dp_size: u32) -> EventConfig { + EventConfig { + host: extract_host(worker_url).unwrap_or_else(|| "127.0.0.1".to_string()), + port_base, + topic: String::new(), + block_size: 64, + dp_size, + } + } + + /// Encode a minimal AllBlocksCleared batch with the given ts and + /// optional dp_rank, in the same array layout msgspec emits. + pub fn encode_all_blocks_cleared_batch(ts: f64, attn_dp_rank: Option) -> Vec { + let mut buf = Vec::new(); + // Outer batch array: [ts, [event], dp_rank?] + mp::write_array_len(&mut buf, 3).unwrap(); + mp::write_f64(&mut buf, ts).unwrap(); + // events array length 1 + mp::write_array_len(&mut buf, 1).unwrap(); + // event = ["AllBlocksCleared"] + mp::write_array_len(&mut buf, 1).unwrap(); + mp::write_str(&mut buf, "AllBlocksCleared").unwrap(); + match attn_dp_rank { + Some(v) => { + mp::write_uint(&mut buf, v as u64).unwrap(); + } + None => mp::write_nil(&mut buf).unwrap(), + } + buf + } + + /// Build a 3-frame multipart with topic="", the given seq (BE i64), + /// and the given payload bytes. + pub fn build_multipart(seq: i64, payload: Vec) -> ZmqMessage { + build_multipart_with_topic(b"", seq, payload) + } + + /// Build a 3-frame multipart with an explicit topic frame. + pub fn build_multipart_with_topic(topic: &[u8], seq: i64, payload: Vec) -> ZmqMessage { + let mut msg = ZmqMessage::from(Bytes::copy_from_slice(topic)); + msg.push_back(Bytes::copy_from_slice(&seq.to_be_bytes())); + msg.push_back(Bytes::from(payload)); + msg + } + + /// Wait briefly for the SubSocket to finish its handshake/subscribe. + /// 50ms is empirically enough on localhost without making tests + /// flaky. + pub async fn settle() { + tokio::time::sleep(Duration::from_millis(50)).await; + } + + /// Destructure a `WorkerEvent::Batch`, panicking on any other + /// variant. Keeps test assertions terse. + pub fn expect_batch(ev: WorkerEvent) -> (KvWorkerId, i64, KvEventBatch) { + match ev { + WorkerEvent::Batch { worker, seq, batch } => (worker, seq, batch), + WorkerEvent::PublisherReset { worker } => { + panic!("expected Batch, got PublisherReset for {worker:?}") + } + } + } + } + + /// Single subscriber: publish one batch, see one batch. + #[tokio::test] + async fn single_subscriber_receives_one_event() { + let (mut pub_sock, port) = helpers::make_pub_bound().await; + + let (tx, mut rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + + registry + .add_worker( + "http://127.0.0.1:30000", + &helpers::cfg_for("http://127.0.0.1:30000", port, 1), + ) + .await; + helpers::settle().await; + + let payload = helpers::encode_all_blocks_cleared_batch(1.0, Some(0)); + let msg = helpers::build_multipart(7, payload); + pub_sock.send(msg).await.expect("send"); + + let event = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("recv timed out") + .expect("channel closed"); + let (worker, seq, batch) = helpers::expect_batch(event); + + assert_eq!(seq, 7); + assert_eq!(worker.dp_rank, 0); + assert_eq!(worker.url, "http://127.0.0.1:30000"); + assert_eq!(batch.events.len(), 1); + assert!(matches!(batch.events[0], KvCacheEvent::AllBlocksCleared)); + + let shutdown_done = timeout(Duration::from_millis(500), registry.shutdown()).await; + assert!(shutdown_done.is_ok(), "shutdown should return promptly"); + } + + /// When the worker advertises a non-empty topic in + /// `EventConfig.topic`, the SUB socket must filter on that prefix: + /// only messages whose first frame *starts with* the topic bytes + /// reach our pump. ZMQ-level filtering is the only way the + /// configured topic affects routing — `decode_message` discards + /// frame 0 regardless — so a SUB socket that ignores `cfg.topic` + /// and subscribes to `""` lets every message on the endpoint + /// through, including events from unrelated publishers that + /// happen to share the host:port (e.g. a colocated worker + /// running a different model on the same machine). + /// + /// Scenario: subscribe to topic "match". Publish two messages on + /// the same PUB socket: topic=`match` first, then topic=`other`. + /// The matched message must be delivered AND the unmatched one + /// must not. We publish matched-first so the negative assertion + /// is the load-bearing check: a broken SUB filter that subscribes + /// to `""` (the pre-fix behavior) delivers BOTH messages in send + /// order, so the seq=22 assertion would still pass but the + /// stray-recv assertion would catch it. This removes a dependency + /// on PUB→SUB delivery ordering as the discriminator. + #[tokio::test] + async fn subscriber_filters_by_configured_topic() { + let (mut pub_sock, port) = helpers::make_pub_bound().await; + + let (tx, mut rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + + let worker_url = "http://127.0.0.1:30100"; + let mut cfg = helpers::cfg_for(worker_url, port, 1); + cfg.topic = "match".into(); + registry.add_worker(worker_url, &cfg).await; + helpers::settle().await; + + // Publish matched first, then `other`. A leaky `""` subscription + // delivers both in order; the topic filter must drop the second. + let payload_matched = helpers::encode_all_blocks_cleared_batch(1.0, Some(0)); + let payload_other = helpers::encode_all_blocks_cleared_batch(2.0, Some(0)); + pub_sock + .send(helpers::build_multipart_with_topic( + b"match", + 22, + payload_matched, + )) + .await + .unwrap(); + pub_sock + .send(helpers::build_multipart_with_topic( + b"other", + 11, + payload_other, + )) + .await + .unwrap(); + + let event = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out waiting for matched event") + .expect("channel closed"); + let (_, seq, _) = helpers::expect_batch(event); + assert_eq!(seq, 22, "matched message must arrive; got seq={seq}"); + + // The load-bearing assertion: no second message in 200ms. A + // SUB subscribed to `""` would have delivered the `other` + // message by now; the topic filter must drop it. + let stray = timeout(Duration::from_millis(200), rx.recv()).await; + assert!( + stray.is_err(), + "second message with topic=`other` must NOT pass the filter \ + (got {stray:?}); cfg.topic is being ignored at subscribe()", + ); + + registry.shutdown().await; + } + + /// DP rank fan-out: 3 PUB sockets, 3 distinct events, all delivered. + #[tokio::test] + async fn dp_rank_fan_out() { + let (mut pub0, p0) = helpers::make_pub_bound().await; + let (mut pub1, p1) = helpers::make_pub_bound().await; + let (mut pub2, p2) = helpers::make_pub_bound().await; + // We need contiguous ports for `base_port + dp_rank` to land on + // each PUB socket. OS-assigned ports won't be contiguous, so we + // bind one PUB socket per dp_rank with the same `worker_url` but + // call `add_worker` three times with `dp_size=1` and the right + // base_port for each. The registry does not require contiguous + // ports per call — but `add_worker` itself does, since it + // constructs `base_port + rank`. Workaround: use distinct + // `worker_url`s so each call's dp_rank=0 maps to its own port, + // and assert via the URL field. + let url0 = "http://127.0.0.1:30000"; + let url1 = "http://127.0.0.1:30001"; + let url2 = "http://127.0.0.1:30002"; + + let (tx, mut rx) = mpsc::channel::(16); + let registry = KvEventSubscriberRegistry::new(tx); + + registry + .add_worker(url0, &helpers::cfg_for(url0, p0, 1)) + .await; + registry + .add_worker(url1, &helpers::cfg_for(url1, p1, 1)) + .await; + registry + .add_worker(url2, &helpers::cfg_for(url2, p2, 1)) + .await; + helpers::settle().await; + + let payload0 = helpers::encode_all_blocks_cleared_batch(1.0, Some(0)); + let payload1 = helpers::encode_all_blocks_cleared_batch(2.0, Some(1)); + let payload2 = helpers::encode_all_blocks_cleared_batch(3.0, Some(2)); + + pub0.send(helpers::build_multipart(10, payload0)) + .await + .unwrap(); + pub1.send(helpers::build_multipart(20, payload1)) + .await + .unwrap(); + pub2.send(helpers::build_multipart(30, payload2)) + .await + .unwrap(); + + let mut seq_by_url: HashMap = HashMap::new(); + for _ in 0..3 { + let event = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out") + .expect("channel closed"); + let (worker, seq, _batch) = helpers::expect_batch(event); + seq_by_url.insert(worker.url, seq); + } + + assert_eq!(seq_by_url.len(), 3); + assert_eq!(seq_by_url[url0], 10); + assert_eq!(seq_by_url[url1], 20); + assert_eq!(seq_by_url[url2], 30); + + registry.shutdown().await; + } + + /// True per-DP fan-out behind a single worker URL: bind 3 PUB + /// sockets on contiguous ports and subscribe with `dp_size=3`. + #[tokio::test] + async fn dp_size_three_per_worker() { + // Pick a single base port and keep retrying until the next two + // ports are also free, so `base_port + 1` and `base_port + 2` + // really resolve to our PUB sockets. + let mut attempt = 0; + let (pub0, pub1, pub2, base_port) = loop { + attempt += 1; + assert!(attempt < 32, "could not find 3 contiguous free ports"); + + // Bind PUB at OS-assigned port to learn what's free, then try + // to bind the next two ports explicitly. + let mut p0 = PubSocket::new(); + let ep0 = p0.bind("tcp://127.0.0.1:0").await.unwrap(); + let base = match ep0 { + Endpoint::Tcp(_, p) => p, + _ => unreachable!(), + }; + + let mut p1 = PubSocket::new(); + let ep1 = p1.bind(&format!("tcp://127.0.0.1:{}", base + 1)).await; + if ep1.is_err() { + continue; + } + + let mut p2 = PubSocket::new(); + let ep2 = p2.bind(&format!("tcp://127.0.0.1:{}", base + 2)).await; + if ep2.is_err() { + continue; + } + break (p0, p1, p2, base); + }; + + let (tx, mut rx) = mpsc::channel::(16); + let registry = KvEventSubscriberRegistry::new(tx); + registry + .add_worker( + "http://127.0.0.1:30000", + &helpers::cfg_for("http://127.0.0.1:30000", base_port, 3), + ) + .await; + helpers::settle().await; + + let mut pub0 = pub0; + let mut pub1 = pub1; + let mut pub2 = pub2; + pub0.send(helpers::build_multipart( + 100, + helpers::encode_all_blocks_cleared_batch(1.0, Some(0)), + )) + .await + .unwrap(); + pub1.send(helpers::build_multipart( + 200, + helpers::encode_all_blocks_cleared_batch(2.0, Some(1)), + )) + .await + .unwrap(); + pub2.send(helpers::build_multipart( + 300, + helpers::encode_all_blocks_cleared_batch(3.0, Some(2)), + )) + .await + .unwrap(); + + let mut by_rank: HashMap = HashMap::new(); + for _ in 0..3 { + let event = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out") + .expect("channel closed"); + let (worker, seq, _batch) = helpers::expect_batch(event); + assert_eq!(worker.url, "http://127.0.0.1:30000"); + by_rank.insert(worker.dp_rank, seq); + } + assert_eq!(by_rank.get(&0), Some(&100)); + assert_eq!(by_rank.get(&1), Some(&200)); + assert_eq!(by_rank.get(&2), Some(&300)); + + registry.shutdown().await; + } + + /// 8-rank multi-publisher fan-out: a worker that publishes to 8 + /// contiguous ZMQ ports (one per DP rank) must produce 8 distinct + /// SUB connections and forward every rank's event. The 3-rank + /// test above pins basic fan-out; this one exercises the wider + /// fan-out shape that real multi-DP workers exhibit. + #[tokio::test] + async fn dp_size_eight_per_worker() { + const N: usize = 8; + let mut attempt = 0; + let mut publishers: Vec = Vec::new(); + let base_port: u16 = loop { + attempt += 1; + assert!(attempt < 64, "could not find 8 contiguous free ports"); + publishers.clear(); + + let mut p0 = PubSocket::new(); + let ep0 = p0.bind("tcp://127.0.0.1:0").await.unwrap(); + let base = match ep0 { + Endpoint::Tcp(_, p) => p, + _ => unreachable!(), + }; + // Ensure `base + N - 1` fits in u16 *and* we can bind every + // contiguous port. Retry on the rare overflow case at the + // high end of the ephemeral range. + if u32::from(base) + (N as u32) > u32::from(u16::MAX) { + continue; + } + publishers.push(p0); + let mut ok = true; + for offset in 1..N as u16 { + let mut p = PubSocket::new(); + let res = p.bind(&format!("tcp://127.0.0.1:{}", base + offset)).await; + if res.is_err() { + ok = false; + break; + } + publishers.push(p); + } + if ok { + break base; + } + }; + + let (tx, mut rx) = mpsc::channel::(64); + let registry = KvEventSubscriberRegistry::new(tx); + let worker_url = "http://127.0.0.1:30000"; + registry + .add_worker( + worker_url, + &helpers::cfg_for(worker_url, base_port, N as u32), + ) + .await; + helpers::settle().await; + + for (rank, pubsock) in publishers.iter_mut().enumerate() { + pubsock + .send(helpers::build_multipart( + 1000 + rank as i64, + helpers::encode_all_blocks_cleared_batch(rank as f64, Some(rank as u32)), + )) + .await + .unwrap(); + } + + let mut by_rank: HashMap = HashMap::new(); + for _ in 0..N { + let event = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out") + .expect("channel closed"); + let (worker, seq, _batch) = helpers::expect_batch(event); + assert_eq!(worker.url, worker_url); + by_rank.insert(worker.dp_rank, seq); + } + assert_eq!(by_rank.len(), N, "every rank must produce an event"); + for rank in 0..N as u32 { + assert_eq!( + by_rank.get(&rank), + Some(&(1000 + rank as i64)), + "rank {rank} missing or wrong seq", + ); + } + + registry.shutdown().await; + } + + /// Bad msgpack payload is logged and dropped; subsequent valid event + /// still arrives. + #[tokio::test] + async fn decoding_error_tolerated() { + let (mut pub_sock, port) = helpers::make_pub_bound().await; + let (tx, mut rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + registry + .add_worker( + "http://127.0.0.1", + &helpers::cfg_for("http://127.0.0.1", port, 1), + ) + .await; + helpers::settle().await; + + // Garbage payload (not msgpack). + pub_sock + .send(helpers::build_multipart(1, vec![0xff, 0xfe, 0xfd])) + .await + .unwrap(); + // Then a valid one. + let payload = helpers::encode_all_blocks_cleared_batch(0.0, None); + pub_sock + .send(helpers::build_multipart(2, payload)) + .await + .unwrap(); + + let event = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out") + .expect("channel closed"); + let (_worker, seq, batch) = helpers::expect_batch(event); + assert_eq!(seq, 2); + // We must NOT have received the bad message. + assert!(matches!(batch.events[0], KvCacheEvent::AllBlocksCleared)); + + registry.shutdown().await; + } + + /// 2-frame and 4-frame messages are dropped; valid 3-frame still works. + #[tokio::test] + async fn wrong_frame_count_tolerated() { + let (mut pub_sock, port) = helpers::make_pub_bound().await; + let (tx, mut rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + registry + .add_worker( + "http://127.0.0.1", + &helpers::cfg_for("http://127.0.0.1", port, 1), + ) + .await; + helpers::settle().await; + + // 2-frame: just topic + payload. + let mut bad2 = ZmqMessage::from(Bytes::new()); + bad2.push_back(Bytes::from_static(b"junk")); + pub_sock.send(bad2).await.unwrap(); + + // 4-frame: topic + seq + payload + extra. + let payload = helpers::encode_all_blocks_cleared_batch(0.0, None); + let mut bad4 = helpers::build_multipart(99, payload.clone()); + bad4.push_back(Bytes::from_static(b"extra")); + pub_sock.send(bad4).await.unwrap(); + + // Valid 3-frame. + pub_sock + .send(helpers::build_multipart(42, payload)) + .await + .unwrap(); + + let event = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out") + .expect("channel closed"); + let (_worker, seq, _batch) = helpers::expect_batch(event); + assert_eq!(seq, 42); + + registry.shutdown().await; + } + + /// END_SEQ sentinel (-1) is forwarded as a `PublisherReset` so the + /// downstream pump can clear its cursor; a subsequent valid event + /// still arrives as a normal `Batch`. + #[tokio::test] + async fn sequence_number_sentinel_propagates_as_reset() { + let (mut pub_sock, port) = helpers::make_pub_bound().await; + let (tx, mut rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + registry + .add_worker( + "http://127.0.0.1", + &helpers::cfg_for("http://127.0.0.1", port, 1), + ) + .await; + helpers::settle().await; + + pub_sock + .send(helpers::build_multipart(-1, b"ignored".to_vec())) + .await + .unwrap(); + let payload = helpers::encode_all_blocks_cleared_batch(0.0, None); + pub_sock + .send(helpers::build_multipart(5, payload)) + .await + .unwrap(); + + let first = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out") + .expect("channel closed"); + assert!( + matches!(first, WorkerEvent::PublisherReset { .. }), + "END_SEQ must surface as PublisherReset, got {first:?}", + ); + + let second = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("timed out") + .expect("channel closed"); + let (_worker, seq, _batch) = helpers::expect_batch(second); + assert_eq!(seq, 5); + + registry.shutdown().await; + } + + /// `remove_worker` cancels the task; further publishes are not + /// received. + #[tokio::test] + async fn remove_worker_cancels() { + let (mut pub_sock, port) = helpers::make_pub_bound().await; + let (tx, mut rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + registry + .add_worker( + "http://127.0.0.1:30000", + &helpers::cfg_for("http://127.0.0.1:30000", port, 1), + ) + .await; + helpers::settle().await; + + // First event arrives. + let payload = helpers::encode_all_blocks_cleared_batch(0.0, None); + pub_sock + .send(helpers::build_multipart(1, payload.clone())) + .await + .unwrap(); + let _ = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("first event timed out"); + + // Remove and verify the handle map empties. + registry.remove_worker("http://127.0.0.1:30000").await; + { + let handles = registry.inner.handles.lock().await; + assert!( + handles.is_empty(), + "handles map should be empty after remove" + ); + } + + // Publish more — receiver should see nothing. + pub_sock + .send(helpers::build_multipart(2, payload)) + .await + .unwrap(); + let res = timeout(Duration::from_millis(150), rx.recv()).await; + assert!( + res.is_err(), + "no event should arrive after remove_worker (got {:?})", + res.unwrap() + ); + } + + /// Calling `add_worker` twice for the same `(url, dp_rank)` pair + /// must not double-spawn. + #[tokio::test] + async fn add_worker_idempotent() { + let (_pub_sock, port) = helpers::make_pub_bound().await; + let (tx, _rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + + registry + .add_worker( + "http://127.0.0.1:30000", + &helpers::cfg_for("http://127.0.0.1:30000", port, 1), + ) + .await; + registry + .add_worker( + "http://127.0.0.1:30000", + &helpers::cfg_for("http://127.0.0.1:30000", port, 1), + ) + .await; + + { + let handles = registry.inner.handles.lock().await; + assert_eq!(handles.len(), 1, "expected 1 entry, got {}", handles.len()); + } + + registry.shutdown().await; + } + + /// `cancel_all` signals every per-worker token without awaiting; a + /// subsequent `shutdown` must still complete cleanly. This pins the + /// contract for any future `Drop` impl that needs a sync cancel path + /// (e.g. when the registry is dropped without an explicit `shutdown`). + #[tokio::test] + async fn cancel_all_then_shutdown_is_clean() { + let (_pub_sock, port) = helpers::make_pub_bound().await; + let (tx, _rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + + registry + .add_worker( + "http://127.0.0.1:30000", + &helpers::cfg_for("http://127.0.0.1:30000", port, 2), + ) + .await; + helpers::settle().await; + + // Sync cancel — must not block, must not panic. + registry.cancel_all(); + + // shutdown should still join cleanly even though the per-worker + // tokens were already fired by cancel_all. + let done = timeout(Duration::from_millis(500), registry.shutdown()).await; + assert!(done.is_ok(), "shutdown after cancel_all must not hang"); + } + + /// Direct unit test of [`extract_host`] — no socket required. + #[test] + fn extract_host_handles_common_urls() { + assert_eq!( + extract_host("http://10.0.0.1:30000").as_deref(), + Some("10.0.0.1") + ); + assert_eq!( + extract_host("https://my.host.example:443").as_deref(), + Some("my.host.example") + ); + // url crate strips brackets from IPv6 literals in host_str(). + assert_eq!(extract_host("http://[::1]:30000").as_deref(), Some("[::1]")); + assert!(extract_host("not a url").is_none()); + } + + /// Direct unit test of [`decode_message`] — exercises sentinel and + /// bad-frame paths without involving sockets. + #[test] + fn decode_message_unit() { + let id = KvWorkerId { + url: "http://x".to_string(), + dp_rank: 0, + }; + + // Wrong frame count. + let one_frame = ZmqMessage::from(Bytes::from_static(b"only")); + assert!(decode_message(&id, one_frame).is_none()); + + // Sentinel seq = -1 now surfaces as PublisherReset (not None) so + // the downstream pump can clear its cursor before a reconnecting + // publisher restarts from seq=1. + let sentinel = helpers::build_multipart(-1, b"ignored".to_vec()); + let reset = decode_message(&id, sentinel).expect("END_SEQ forwards"); + assert!(matches!(reset, WorkerEvent::PublisherReset { .. })); + + // Bad seq frame length. + let mut bad_seq = ZmqMessage::from(Bytes::new()); + bad_seq.push_back(Bytes::from_static(b"abc")); // 3 bytes, not 8 + bad_seq.push_back(Bytes::from_static(b"")); + assert!(decode_message(&id, bad_seq).is_none()); + + // Bad payload. + let bad_payload = helpers::build_multipart(1, vec![0xff, 0xfe]); + assert!(decode_message(&id, bad_payload).is_none()); + + // Happy path. + let payload = helpers::encode_all_blocks_cleared_batch(0.0, None); + let good = helpers::build_multipart(7, payload); + let event = decode_message(&id, good).expect("should decode"); + let (worker, seq, _batch) = helpers::expect_batch(event); + assert_eq!(seq, 7); + assert_eq!(worker, id); + } + + /// Restart-resume contract: after a worker is removed and then re-added + /// to the same endpoint, the new subscriber must connect and forward + /// fresh events. Confirms that `remove_worker` releases the SUB socket + /// cleanly enough that a same-endpoint reconnect succeeds within the + /// settle window, without leaking the previous task's state. + /// + /// Events published while the worker is detached are lost (ZMQ PUB/SUB + /// is fire-and-forget; no replay). Downstream cursor recovery happens + /// at the [`super::index::KvEventIndex`] layer, which clears the cursor + /// on `remove_worker` so the re-added worker's seq=1 is not filtered. + #[tokio::test] + async fn restart_after_remove_picks_up_new_events() { + let (mut pub_sock, port) = helpers::make_pub_bound().await; + let worker_url = "http://127.0.0.1:30000"; + let cfg = helpers::cfg_for(worker_url, port, 1); + + let (tx, mut rx) = mpsc::channel::(8); + let registry = KvEventSubscriberRegistry::new(tx); + + // First incarnation: publish + drain. + registry.add_worker(worker_url, &cfg).await; + helpers::settle().await; + let payload_a = helpers::encode_all_blocks_cleared_batch(1.0, Some(0)); + pub_sock + .send(helpers::build_multipart(1, payload_a)) + .await + .unwrap(); + let event_a = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("recv before remove timed out") + .expect("channel closed"); + let (_, seq_a, _) = helpers::expect_batch(event_a); + assert_eq!(seq_a, 1); + + // Detach the subscriber while the publisher keeps going. + registry.remove_worker(worker_url).await; + + // This batch is sent while no subscriber is attached; it must be + // dropped (ZMQ PUB without a connected SUB is fire-and-forget) and + // must not poison the next subscriber's view. + let payload_b = helpers::encode_all_blocks_cleared_batch(2.0, Some(0)); + pub_sock + .send(helpers::build_multipart(2, payload_b)) + .await + .unwrap(); + // Verify rx really has nothing buffered. + assert!( + timeout(Duration::from_millis(100), rx.recv()) + .await + .is_err(), + "no event must arrive while the worker is detached", + ); + + // Re-attach the SAME worker at the SAME endpoint. + registry.add_worker(worker_url, &cfg).await; + helpers::settle().await; + + // Fresh event from the publisher → must surface on the new subscriber. + let payload_c = helpers::encode_all_blocks_cleared_batch(3.0, Some(0)); + pub_sock + .send(helpers::build_multipart(3, payload_c)) + .await + .unwrap(); + let event_c = timeout(Duration::from_millis(500), rx.recv()) + .await + .expect("recv after re-add timed out") + .expect("channel closed"); + let (worker_c, seq_c, _) = helpers::expect_batch(event_c); + assert_eq!(seq_c, 3); + assert_eq!(worker_c.url, worker_url); + + registry.shutdown().await; + } +} diff --git a/experimental/sgl-router/src/policies/kv_events/tree.rs b/experimental/sgl-router/src/policies/kv_events/tree.rs new file mode 100644 index 000000000000..2409b6d10778 --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/tree.rs @@ -0,0 +1,1094 @@ +//! Hash-keyed radix tree for KV-cache event indexing. +//! +//! Each non-root node represents one block hash (`i64`). A node's children +//! are keyed by the *next* block hash in a chain, so a path from the root +//! down to depth `n` represents a chain of `n` block hashes. Every node +//! tracks the set of [`KvWorkerId`]s that hold the chain ending at that +//! node. +//! +//! The tree is fed by `BlockStored` / `BlockRemoved` / `AllBlocksCleared` +//! events from SGLang workers (decoded by [`super::wire`]) and is queried +//! via [`HashTree::match_prefix`] to find which workers already hold the +//! longest prefix of an incoming request's block-hash chain. +//! +//! # Concurrency +//! +//! The whole tree lives behind a single [`parking_lot::RwLock`] +//! ([`HashTree::state`]). The match path takes a read-lock and updates +//! `last_used` via an [`AtomicU64`] so that routing decisions across tokio +//! worker threads do not serialise on the lock. Mutations (insert / remove +//! / clear / evict) take a write-lock. We accept the coarse granularity +//! for v1 on the write side — correctness over throughput — and the +//! existing text-tree at `super::super::tree` is what serves the high-RPS +//! mesh-fallback path. This module is only on the cache-aware-from-events +//! path. +//! +//! # Reverse index +//! +//! `BlockRemoved` events carry only `block_hashes` and no parent context, +//! so without an index from `block_hash → set of nodes carrying that hash` +//! we'd have to walk the whole tree. We maintain that reverse index as +//! [`TreeState::by_hash`]. The same hash can legitimately appear at +//! multiple positions in the tree (e.g. as the last block of one chain and +//! as the second block of another), so each entry is a *set* of node IDs. +//! +//! # Pruning +//! +//! When a worker is dropped from a node and the node has no remaining +//! workers AND no children, we detach it from its parent and remove it +//! from the reverse index. Pruning cascades upward iteratively (chains +//! can be deep — the recursive form would risk stack-overflow for +//! pathological inputs). + +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::OnceLock; +use std::time::Instant; + +use parking_lot::RwLock; +use tracing::{debug, error}; + +/// Process-wide monotonic epoch used to derive cheap millisecond-resolution +/// timestamps for [`Node::last_used`]. Initialised lazily on first use. +static PROCESS_EPOCH: OnceLock = OnceLock::new(); + +/// Milliseconds elapsed since [`PROCESS_EPOCH`]. Truncates from `u128` to +/// `u64`; with `u64` ms we have ~584 million years of headroom which is +/// fine. +fn now_millis() -> u64 { + PROCESS_EPOCH + .get_or_init(Instant::now) + .elapsed() + .as_millis() as u64 +} + +/// Identifier for a worker endpoint, refined by DP-attention rank. +/// +/// Workers running with multiple DP-attention ranks emit independent event +/// streams (one per rank), and each rank holds a disjoint slice of the KV +/// cache. We therefore track them as separate cache-holders. +/// +/// The name is intentionally namespaced (`KvWorkerId`) to avoid collision +/// with [`crate::core::worker_registry::WorkerId`], which is a UUID-string +/// identity used by the worker registry. +/// +/// # Provenance +/// +/// Instances should only be minted by the kv_events module itself +/// (subscriber registry → pump → tree) so the `url` always comes from +/// the worker registry's authoritative URL. External callers can read +/// the fields and use them to query the tree, but constructing fresh +/// IDs from arbitrary URLs would let routing logic resolve to +/// non-registered endpoints. Use [`KvWorkerId::new`] when constructing +/// from a tested path; do not assemble struct literals from +/// user-controlled input. +#[derive(Clone, Eq, Hash, PartialEq, Debug)] +pub struct KvWorkerId { + pub url: String, + pub dp_rank: u32, +} + +impl KvWorkerId { + /// Explicit constructor — preferred over struct-literal syntax so + /// future tightening of provenance has a single chokepoint. + pub fn new(url: String, dp_rank: u32) -> Self { + Self { url, dp_rank } + } +} + +/// Result of [`HashTree::match_prefix`]. +#[derive(Debug, Clone)] +pub struct MatchResult { + /// Number of leading block hashes from the input slice that matched a + /// path from the root. + pub matched_blocks: usize, + /// Workers holding the deepest matched node. Empty when + /// `matched_blocks == 0`. + pub workers: HashSet, +} + +/// Internal stable handle to a tree node. +/// +/// We use an arena (`HashMap`) instead of `Arc>` +/// + `Weak` because: +/// 1. We need to enumerate every node (e.g. for `clear_worker` and +/// `evict_lru`); a flat map is direct and cheap. +/// 2. The reverse index needs a *stable* key per node — `Weak` would force +/// upgrades on every lookup and complicate prune semantics. +type NodeId = u64; + +/// A single tree node. Non-root nodes are keyed by their `block_hash` +/// (which is shared across siblings only insofar as the reverse index +/// records every position); within a single parent's children map there is +/// at most one child per `block_hash`. +/// +/// `last_used` is an [`AtomicU64`] of milliseconds since [`PROCESS_EPOCH`]. +/// Storing it atomically lets the match path mutate it under a *read* lock +/// on [`TreeState`], which is essential because matching is on the routing +/// hot path. `Relaxed` ordering is sufficient: eviction only needs +/// approximate freshness, and ties at the millisecond boundary tie-break +/// by [`NodeId`]. +#[derive(Debug)] +struct Node { + block_hash: i64, + /// Hash of the parent block on the chain that produced this node, or + /// `None` if this node hangs directly off the root sentinel. + /// Stored for diagnostic / chain-reconstruction only — the actual + /// parent pointer lives in [`Node::parent`]. Tests and future + /// inspectors read this; suppress dead-code warning in non-test builds. + #[allow(dead_code)] + parent_block_hash: Option, + /// `None` only for the root sentinel. + parent: Option, + workers: HashSet, + /// Children keyed by next-block hash. + children: HashMap, + last_used: AtomicU64, +} + +impl Node { + fn new_child(block_hash: i64, parent_block_hash: Option, parent: NodeId) -> Self { + Self { + block_hash, + parent_block_hash, + parent: Some(parent), + workers: HashSet::new(), + children: HashMap::new(), + last_used: AtomicU64::new(now_millis()), + } + } +} + +/// Inner mutable tree state. Single-lock for v1; document any cross-method +/// invariants here: +/// +/// * `nodes[ROOT_ID]` is always present and is the only node with +/// `parent == None`. +/// * For every non-root node `n`: `nodes[n.parent].children[&n.block_hash] +/// == n`'s id (i.e., parent's child pointer round-trips). +/// * `by_hash[h]` contains the id of every non-root node `n` with +/// `n.block_hash == h`. Root is never in `by_hash`. +/// * Pruning runs after every worker-removal that empties a node: prune +/// detaches from parent, removes from `by_hash`, and recurses upward. +#[derive(Debug)] +struct TreeState { + nodes: HashMap, + by_hash: HashMap>, + next_id: NodeId, +} + +const ROOT_ID: NodeId = 0; +/// Sentinel block_hash for the root. Real workers can in principle emit +/// `i64::MIN`, but the root is never looked up via `by_hash` so collisions +/// don't matter. +const ROOT_HASH_SENTINEL: i64 = i64::MIN; + +impl TreeState { + fn new() -> Self { + let mut nodes = HashMap::new(); + nodes.insert( + ROOT_ID, + Node { + block_hash: ROOT_HASH_SENTINEL, + parent_block_hash: None, + parent: None, + workers: HashSet::new(), + children: HashMap::new(), + last_used: AtomicU64::new(now_millis()), + }, + ); + Self { + nodes, + by_hash: HashMap::new(), + next_id: 1, + } + } + + fn alloc_id(&mut self) -> NodeId { + let id = self.next_id; + self.next_id += 1; + id + } + + /// Insert a brand-new child under `parent_id` and wire up the reverse + /// index. Caller is responsible for ensuring `parent_id`'s child slot + /// for `block_hash` is empty (else this overwrites it). + /// + /// Returns `None` if `parent_id` does not exist — an invariant + /// violation. The pump runs in a long-lived task; panicking here would + /// take down the entire cache-aware path, so we log and bail. + fn create_child( + &mut self, + parent_id: NodeId, + block_hash: i64, + parent_block_hash: Option, + ) -> Option { + let id = self.alloc_id(); + self.nodes.insert( + id, + Node::new_child(block_hash, parent_block_hash, parent_id), + ); + let Some(parent) = self.nodes.get_mut(&parent_id) else { + error!( + parent_id, + block_hash, + "tree invariant violation: create_child called with unknown parent_id; discarding new node", + ); + self.nodes.remove(&id); + return None; + }; + parent.children.insert(block_hash, id); + self.by_hash.entry(block_hash).or_default().insert(id); + Some(id) + } + + /// Pick the parent node id for an incoming `BlockStored` event. + /// + /// Resolution order (matches doc-comment on `HashTree::insert`): + /// 1. `parent_hash == None` → root. + /// 2. There's exactly one node carrying `parent_hash` → use it. + /// 3. Multiple candidates: prefer one already containing `worker`. + /// 4. None contain the worker: log at debug, fall back to root. The + /// new chain still carries `parent_hash` on its first node so that + /// if the parent's `BlockStored` arrives later we can reconstruct + /// the link via the reverse index. + fn resolve_parent(&self, worker: &KvWorkerId, parent_hash: Option) -> NodeId { + let Some(parent_hash) = parent_hash else { + return ROOT_ID; + }; + let Some(candidates) = self.by_hash.get(&parent_hash) else { + debug!( + worker = %worker.url, + dp_rank = worker.dp_rank, + parent_hash, + "parent_hash not in tree; attaching new chain to root", + ); + return ROOT_ID; + }; + if candidates.len() == 1 { + return *candidates.iter().next().unwrap(); + } + // Multiple candidates — prefer one this worker already holds. + for &cand in candidates { + if self + .nodes + .get(&cand) + .is_some_and(|n| n.workers.contains(worker)) + { + return cand; + } + } + debug!( + worker = %worker.url, + dp_rank = worker.dp_rank, + parent_hash, + n_candidates = candidates.len(), + "ambiguous parent_hash with no worker-owned candidate; attaching to root", + ); + ROOT_ID + } + + fn insert(&mut self, worker: &KvWorkerId, parent_hash: Option, block_hashes: &[i64]) { + if block_hashes.is_empty() { + return; + } + let mut current = self.resolve_parent(worker, parent_hash); + let mut prev_hash = parent_hash; + let now = now_millis(); + for &h in block_hashes { + let child_id = match self + .nodes + .get(¤t) + .and_then(|n| n.children.get(&h).copied()) + { + Some(id) => id, + None => match self.create_child(current, h, prev_hash) { + Some(id) => id, + None => return, + }, + }; + let Some(child) = self.nodes.get_mut(&child_id) else { + error!( + child_id, + block_hash = h, + "tree invariant violation: child node missing immediately after fetch/create; aborting chain", + ); + return; + }; + child.workers.insert(worker.clone()); + child.last_used.store(now, Ordering::Relaxed); + current = child_id; + prev_hash = Some(h); + } + } + + fn remove(&mut self, worker: &KvWorkerId, block_hashes: &[i64]) { + // Collect all node ids to touch (fixed snapshot — avoids iterator + // invalidation when pruning mutates `by_hash`). + let mut targets: Vec = Vec::new(); + for h in block_hashes { + if let Some(set) = self.by_hash.get(h) { + targets.extend(set.iter().copied()); + } + } + for id in targets { + // Node may already be gone if a previous prune in this batch + // cascaded through it — skip silently. + let still_present = match self.nodes.get_mut(&id) { + Some(node) => { + node.workers.remove(worker); + node.workers.is_empty() && node.children.is_empty() + } + None => false, + }; + if still_present { + self.prune_cascade(id); + } + } + } + + fn clear_worker(&mut self, worker: &KvWorkerId) { + // Snapshot ids before mutation. + let ids: Vec = self + .nodes + .keys() + .copied() + .filter(|&id| id != ROOT_ID) + .collect(); + let mut prune_candidates: Vec = Vec::new(); + for id in ids { + if let Some(node) = self.nodes.get_mut(&id) { + if node.workers.remove(worker) + && node.workers.is_empty() + && node.children.is_empty() + { + prune_candidates.push(id); + } + } + } + for id in prune_candidates { + // Re-check: cascading prune from a sibling may have already + // removed this id. + if self.nodes.contains_key(&id) { + self.prune_cascade(id); + } + } + } + + /// Detach `start` and walk up, pruning every ancestor that becomes + /// empty + childless. Iterative — chains can be long. + fn prune_cascade(&mut self, start: NodeId) { + let mut cursor = start; + loop { + if cursor == ROOT_ID { + return; + } + // Peek at the node before removal so we know its parent + hash. + let (parent_id, block_hash) = match self.nodes.get(&cursor) { + Some(n) => match n.parent { + Some(p) => (p, n.block_hash), + None => { + error!( + cursor, + "tree invariant violation: non-root node has no parent; aborting prune", + ); + return; + } + }, + None => return, + }; + // Confirm prune precondition (cheap defensive check). + let prunable = self + .nodes + .get(&cursor) + .map(|n| n.workers.is_empty() && n.children.is_empty()) + .unwrap_or(false); + if !prunable { + return; + } + // Detach from parent's children map. + if let Some(parent) = self.nodes.get_mut(&parent_id) { + parent.children.remove(&block_hash); + } + // Remove from reverse index. + if let Some(set) = self.by_hash.get_mut(&block_hash) { + set.remove(&cursor); + if set.is_empty() { + self.by_hash.remove(&block_hash); + } + } + // Drop the node itself. + self.nodes.remove(&cursor); + // Walk up. + cursor = parent_id; + // Stop unless the parent is now also empty + childless. + let parent_prunable = self + .nodes + .get(&cursor) + .map(|n| cursor != ROOT_ID && n.workers.is_empty() && n.children.is_empty()) + .unwrap_or(false); + if !parent_prunable { + return; + } + } + } + + /// Read-only match path. Takes `&self` (not `&mut self`) so the public + /// [`HashTree::match_prefix`] can hold only a read lock — matching is + /// the routing hot path and write-locking it would serialise all + /// routing decisions across tokio worker threads. `last_used` is an + /// [`AtomicU64`] specifically so the touch-on-descend can happen + /// through a shared reference. + /// + /// Note the asymmetry with [`TreeState::resolve_parent`] (used by + /// `insert`): that function disambiguates a multi-candidate + /// `parent_hash` by preferring a worker-owned node. This function has + /// no worker context to do the same, so multiple candidates fall back + /// to root. The asymmetry is intentional for v1; the public doc on + /// [`HashTree::match_prefix`] documents the policy for callers. + fn match_prefix(&self, parent_hash: Option, block_hashes: &[i64]) -> MatchResult { + if block_hashes.is_empty() { + return MatchResult { + matched_blocks: 0, + workers: HashSet::new(), + }; + } + // Determine starting node: root, or the unique node carrying + // `parent_hash`. Multiple matches: bail to root (caller should + // have a single canonical context). + let start = match parent_hash { + None => ROOT_ID, + Some(p) => match self.by_hash.get(&p) { + Some(set) if set.len() == 1 => *set.iter().next().unwrap(), + _ => ROOT_ID, + }, + }; + + let mut current = start; + let mut matched = 0usize; + let mut last_match_node: Option = None; + let now = now_millis(); + for &h in block_hashes { + let next = self + .nodes + .get(¤t) + .and_then(|n| n.children.get(&h).copied()); + match next { + Some(child_id) => { + // Touch as we descend. Atomic store under a shared + // borrow — no &mut needed. + if let Some(child) = self.nodes.get(&child_id) { + child.last_used.store(now, Ordering::Relaxed); + } + current = child_id; + matched += 1; + last_match_node = Some(child_id); + } + None => break, + } + } + let workers = match last_match_node { + Some(id) => self + .nodes + .get(&id) + .map(|n| n.workers.clone()) + .unwrap_or_default(), + None => HashSet::new(), + }; + MatchResult { + matched_blocks: matched, + workers, + } + } + + /// Approximate count of *non-root* nodes in the tree. + fn node_count(&self) -> usize { + // Subtract one for the root sentinel. + self.nodes.len().saturating_sub(1) + } + + fn evict_lru(&mut self, max_size: usize) -> usize { + // Fast-path: already under cap. + if self.node_count() <= max_size { + return 0; + } + // Count by total node-count delta so cascade prunes (which may + // remove multiple ancestors per `prune_cascade` call) are + // accounted for accurately, not just the cascade entry point. + let count_before = self.nodes.len(); + + // Phase 1: drop empty (no-worker) leaves first. These hang around + // only because of pruning races — they're free wins. + let empty_leaves: Vec = self + .nodes + .iter() + .filter_map(|(&id, n)| { + if id != ROOT_ID && n.workers.is_empty() && n.children.is_empty() { + Some(id) + } else { + None + } + }) + .collect(); + for id in empty_leaves { + if self.node_count() <= max_size { + break; + } + if self.nodes.contains_key(&id) { + self.prune_cascade(id); + } + } + + // Phase 2: evict oldest leaves (with workers) until we hit cap. + // We re-snapshot leaves each pass because pruning can promote a + // parent into "leaf" status. The outer loop bounds work to + // O(node_count) so we don't spin on a degenerate tree. + let mut iters = 0usize; + let max_iters = self.nodes.len().saturating_add(1); + while self.node_count() > max_size && iters < max_iters { + iters += 1; + // Find the LRU leaf. `last_used` is read with `Relaxed` — + // approximate freshness is fine for eviction. Equality at + // the millisecond boundary tie-breaks by NodeId. + let mut oldest: Option<(u64, NodeId)> = None; + for (&id, n) in &self.nodes { + if id == ROOT_ID || !n.children.is_empty() { + continue; + } + let ts = n.last_used.load(Ordering::Relaxed); + match oldest { + None => oldest = Some((ts, id)), + Some((cur, _)) if ts < cur => oldest = Some((ts, id)), + _ => {} + } + } + let Some((_, victim)) = oldest else { + break; // No leaves at all (shouldn't happen with non-empty tree). + }; + // Force-prune even if the leaf still holds workers — eviction + // intentionally evicts. We clear workers first so the cascade + // precondition holds. + if let Some(node) = self.nodes.get_mut(&victim) { + node.workers.clear(); + } + self.prune_cascade(victim); + } + count_before - self.nodes.len() + } +} + +/// Public hash-keyed radix tree. Cheap to clone an [`Arc`] of; the +/// underlying state is `Send + Sync` (single `RwLock`). +#[derive(Debug)] +pub struct HashTree { + state: RwLock, +} + +impl Default for HashTree { + fn default() -> Self { + Self::new() + } +} + +impl HashTree { + pub fn new() -> Self { + Self { + state: RwLock::new(TreeState::new()), + } + } + + /// Apply a `BlockStored` event. + /// + /// Walks from `parent_hash`'s node (or root) and descends along + /// `block_hashes`, marking every visited node as held by `worker`. + /// Empty `block_hashes` is a no-op. + pub fn insert(&self, worker: &KvWorkerId, parent_hash: Option, block_hashes: &[i64]) { + let mut state = self.state.write(); + state.insert(worker, parent_hash, block_hashes); + } + + /// Apply a `BlockRemoved` event. + /// + /// For every node carrying any hash in `block_hashes`, drop `worker` + /// from that node's worker set. Nodes that become empty AND childless + /// are pruned (cascading upward). + /// + /// Removing the worker from a node does NOT remove the node if other + /// workers still hold it. + pub fn remove(&self, worker: &KvWorkerId, block_hashes: &[i64]) { + let mut state = self.state.write(); + state.remove(worker, block_hashes); + } + + /// Apply an `AllBlocksCleared` event for `worker`. + pub fn clear_worker(&self, worker: &KvWorkerId) { + let mut state = self.state.write(); + state.clear_worker(worker); + } + + /// Find the longest path from the root that matches a prefix of + /// `block_hashes`, optionally starting from the node carrying + /// `parent_hash`. + /// + /// Returns the deepest matched node's worker set and how many blocks + /// matched. + /// + /// As a side-effect, touches `last_used` on every node visited along + /// the match — so frequently-matched paths are kept hot for + /// [`HashTree::evict_lru`]. The touch is an atomic `Relaxed` store, so + /// this method only needs a read lock and many threads can match + /// concurrently. + /// + /// # Ambiguous `parent_hash` + /// If `parent_hash == Some(p)` and `p` is carried by multiple nodes + /// (the "same hash in two chains" case), this method cannot + /// disambiguate and falls back to matching from the root. Callers + /// that need a specific chain should split the request or call with + /// `parent_hash = None`. (`insert` resolves the same ambiguity by + /// preferring a worker-owned candidate; `match_prefix` has no worker + /// context, so the asymmetry is intentional.) + pub fn match_prefix(&self, parent_hash: Option, block_hashes: &[i64]) -> MatchResult { + let state = self.state.read(); + state.match_prefix(parent_hash, block_hashes) + } + + /// Approximate number of non-root nodes in the tree (the root sentinel + /// is not counted). Useful for metrics and to decide when to call + /// [`HashTree::evict_lru`]. + pub fn node_count(&self) -> usize { + self.state.read().node_count() + } + + /// Number of distinct block-hash keys carried by the reverse index. + /// Exposed for invariant tests: when `node_count() == 0` this must + /// also be 0. A nonzero value here with zero nodes means a `prune` + /// path forgot to clean up `by_hash` and the index has leaked. + pub fn reverse_index_size(&self) -> usize { + self.state.read().by_hash.len() + } + + /// Evict least-recently-used nodes until `node_count() <= max_size`. + /// + /// Strategy: + /// 1. Drop already-empty leaves (no workers, no children) first. + /// 2. If still over cap, evict oldest leaves (force-clearing workers + /// on the victim) and cascade-prune. + /// + /// Returns the exact total number of nodes pruned, including any + /// ancestors removed by cascade-pruning. Suitable for wiring into a + /// metric counter. + pub fn evict_lru(&self, max_size: usize) -> usize { + let mut state = self.state.write(); + state.evict_lru(max_size) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn worker(url: &str, dp_rank: u32) -> KvWorkerId { + KvWorkerId { + url: url.to_string(), + dp_rank, + } + } + + fn workers(ids: &[&KvWorkerId]) -> HashSet { + ids.iter().map(|w| (*w).clone()).collect() + } + + #[test] + fn empty_match_returns_zero_no_workers() { + let tree = HashTree::new(); + let m = tree.match_prefix(None, &[]); + assert_eq!(m.matched_blocks, 0); + assert!(m.workers.is_empty()); + + let m2 = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m2.matched_blocks, 0); + assert!(m2.workers.is_empty()); + } + + #[test] + fn single_insert_and_match() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[1, 2, 3]); + + let m = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + + let m = tree.match_prefix(None, &[1, 2]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&a])); + + // Diverges at depth 3 (input asks for 4, tree has 3). + let m = tree.match_prefix(None, &[1, 2, 4]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&a])); + + // No match at root. + let m = tree.match_prefix(None, &[9, 9]); + assert_eq!(m.matched_blocks, 0); + assert!(m.workers.is_empty()); + } + + #[test] + fn two_workers_overlapping_prefix() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + let b = worker("http://b", 0); + tree.insert(&a, None, &[1, 2, 3]); + tree.insert(&b, None, &[1, 2, 4]); + + // Common prefix node carries both. + let m = tree.match_prefix(None, &[1, 2]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&a, &b])); + + // Divergent leaf carries only the matching worker. + let m = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + + let m = tree.match_prefix(None, &[1, 2, 4]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&b])); + } + + #[test] + fn continuation_insert_chains_via_parent_hash() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[1, 2]); + tree.insert(&a, Some(2), &[3]); + + let m = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + } + + #[test] + fn remove_specific_blocks_drops_worker_at_those_nodes() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[1, 2, 3]); + // Sanity. + assert_eq!(tree.node_count(), 3); + + // Remove A from the node carrying hash=2. Per spec: that node loses + // A; descendants are NOT recursively touched, but `match_prefix` + // returns the deepest matched *node*'s worker set. Node 2 still + // exists (it has child 3), but its worker set is now empty. + tree.remove(&a, &[2]); + + // Node 2 still in tree (has child 3). + // Match length 2 lands on node 2 (workers empty), so workers={}. + let m = tree.match_prefix(None, &[1, 2]); + assert_eq!(m.matched_blocks, 2); + assert!(m.workers.is_empty()); + + // Match length 3 lands on node 3 (workers still has A). + let m = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + + // Reverse-index sanity for hash 2: still present (node holds it). + { + let st = tree.state.read(); + assert!(st.by_hash.contains_key(&2)); + } + } + + #[test] + fn clear_worker_drops_exclusive_branches_keeps_shared_nodes() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + let b = worker("http://b", 0); + tree.insert(&a, None, &[1, 2, 3]); + tree.insert(&b, None, &[1, 2, 4]); + let n_before = tree.node_count(); + assert_eq!(n_before, 4); // 1, 2, 3, 4 + + tree.clear_worker(&a); + + // [1,2,3] no longer has A; node 3 prunes (only A held it). + let m = tree.match_prefix(None, &[1, 2, 3]); + // Node 3 was pruned, so only 2 levels match. + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&b])); + + // [1,2] now has only B (A was the only other holder of node 2; + // wait — actually A held 1 and 2 too. But B also holds 1 and 2.) + let m = tree.match_prefix(None, &[1, 2]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&b])); + + // Node count: root + 1 + 2 + 4 (no 3) = 3 non-root nodes. + assert_eq!(tree.node_count(), 3); + } + + #[test] + fn pruning_cascades_when_only_worker_clears() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[1, 2, 3]); + assert_eq!(tree.node_count(), 3); + + tree.clear_worker(&a); + // Whole chain prunes; only the root sentinel remains. + // node_count() returns *non-root* count, so it should be 0. + assert_eq!(tree.node_count(), 0); + // Reverse index for these hashes should be empty. + { + let st = tree.state.read(); + assert!(!st.by_hash.contains_key(&1)); + assert!(!st.by_hash.contains_key(&2)); + assert!(!st.by_hash.contains_key(&3)); + } + } + + #[test] + fn pruning_cascades_via_remove_blockhashes() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[1, 2, 3]); + + // Remove all of A's blocks at once. + tree.remove(&a, &[1, 2, 3]); + assert_eq!(tree.node_count(), 0); + } + + #[test] + fn same_hash_in_two_chains_both_tracked_in_reverse_index() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + // Two chains share hash=5 but at different positions. + tree.insert(&a, None, &[1, 5]); + tree.insert(&a, None, &[2, 5]); + + // Both chains exist independently. + let m = tree.match_prefix(None, &[1, 5]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&a])); + + let m = tree.match_prefix(None, &[2, 5]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&a])); + + // Reverse index for hash 5 has 2 distinct nodes. + { + let st = tree.state.read(); + assert_eq!(st.by_hash.get(&5).map(|s| s.len()), Some(2)); + } + + // BlockRemoved [5] should remove A from BOTH nodes-carrying-5. + // Both nodes are leaves, so both prune. + tree.remove(&a, &[5]); + // Remaining nodes: 1 and 2 (still hold A). + assert_eq!(tree.node_count(), 2); + let m = tree.match_prefix(None, &[1, 5]); + assert_eq!(m.matched_blocks, 1); + assert_eq!(m.workers, workers(&[&a])); + let m = tree.match_prefix(None, &[2, 5]); + assert_eq!(m.matched_blocks, 1); + assert_eq!(m.workers, workers(&[&a])); + } + + #[test] + fn dp_rank_distinguishes_workers() { + let tree = HashTree::new(); + let w0 = worker("http://u", 0); + let w1 = worker("http://u", 1); + tree.insert(&w0, None, &[1, 2, 3]); + tree.insert(&w1, None, &[1, 2, 4]); + + // Common prefix has both ranks. + let m = tree.match_prefix(None, &[1, 2]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&w0, &w1])); + + // Divergent leaves: each rank on its own. + let m = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&w0])); + + let m = tree.match_prefix(None, &[1, 2, 4]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&w1])); + } + + #[test] + fn parent_hash_resolution_picks_worker_owned_node() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + let b = worker("http://b", 0); + // Two nodes both end up carrying hash=5 (same trick as the + // "same hash in two chains" test). + tree.insert(&a, None, &[1, 5]); + tree.insert(&b, None, &[2, 5]); + // A continues from its 5. + tree.insert(&a, Some(5), &[7]); + + // The chain 1->5->7 must exist with A. + let m = tree.match_prefix(None, &[1, 5, 7]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + + // The chain 2->5 should NOT have a 7-child (we routed to A's branch). + let m = tree.match_prefix(None, &[2, 5, 7]); + assert_eq!(m.matched_blocks, 2); + assert_eq!(m.workers, workers(&[&b])); + } + + #[test] + fn ambiguous_parent_hash_unowned_falls_back_to_root() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + let b = worker("http://b", 0); + let c = worker("http://c", 0); + // Two nodes carry hash=5, neither is owned by C. + tree.insert(&a, None, &[1, 5]); + tree.insert(&b, None, &[2, 5]); + // C tries to extend with parent_hash=5; resolution should fall + // back to root with the new chain rooted at hash=9. + tree.insert(&c, Some(5), &[9]); + + // C is reachable as a fresh root child at hash=9. + let m = tree.match_prefix(None, &[9]); + assert_eq!(m.matched_blocks, 1); + assert_eq!(m.workers, workers(&[&c])); + } + + #[test] + fn reinsert_same_chain_idempotent() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[1, 2, 3]); + tree.insert(&a, None, &[1, 2, 3]); + + assert_eq!(tree.node_count(), 3); + let m = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + } + + #[test] + fn empty_block_hashes_insert_is_noop() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[]); + assert_eq!(tree.node_count(), 0); + } + + #[test] + fn eviction_smoke_drops_to_below_cap() { + let tree = HashTree::new(); + // 50 distinct chains of length 1. Each chain gets its own root child. + for i in 0..50i64 { + let w = worker("http://w", i as u32); + tree.insert(&w, None, &[i]); + } + assert_eq!(tree.node_count(), 50); + + let evicted = tree.evict_lru(10); + // Each leaf hangs directly off root, so cascade-pruning never + // cascades past the leaf itself: count must equal exactly the + // number of nodes we needed to drop. + assert_eq!(evicted, 40, "expected to evict exactly 40, got {evicted}"); + assert_eq!( + tree.node_count(), + 10, + "expected node_count == 10, got {}", + tree.node_count() + ); + } + + #[test] + fn eviction_under_cap_is_noop() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + tree.insert(&a, None, &[1, 2, 3]); + + let evicted = tree.evict_lru(100); + assert_eq!(evicted, 0); + assert_eq!(tree.node_count(), 3); + } + + #[test] + fn eviction_prefers_oldest_leaves() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + // First chain: oldest. + tree.insert(&a, None, &[100, 101, 102]); + // Tiny sleep to force last_used differentiation at millisecond + // resolution. The 2ms gap is generous vs. the 1ms tick. + std::thread::sleep(std::time::Duration::from_millis(2)); + // Second chain: newer. + tree.insert(&a, None, &[200, 201, 202]); + + // Match the newer chain to bump its last_used. + std::thread::sleep(std::time::Duration::from_millis(2)); + let _ = tree.match_prefix(None, &[200, 201, 202]); + + // Force eviction down to 3 nodes; the older chain should go first. + // The leaf 102 is the LRU; pruning it cascades up through 101 and + // 100 (each becomes empty + childless), so a single victim drops + // the whole older chain — exactly 3 nodes evicted. + let evicted = tree.evict_lru(3); + assert_eq!(evicted, 3, "expected to evict exactly 3, got {evicted}"); + assert_eq!(tree.node_count(), 3); + + // The newer chain should still match fully. + let m = tree.match_prefix(None, &[200, 201, 202]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + } + + #[test] + fn batched_block_stored_chains_correctly() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + // BlockStored carrying multiple hashes: each chains off its + // predecessor, and parent_hash applies to the FIRST. + tree.insert(&a, None, &[10, 20, 30]); + + let m = tree.match_prefix(None, &[10, 20, 30]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&a])); + + // Confirm parent_block_hash chain: node carrying 30 should record + // parent_block_hash = Some(20). + let st = tree.state.read(); + let n30_id = *st.by_hash.get(&30).unwrap().iter().next().unwrap(); + assert_eq!(st.nodes[&n30_id].parent_block_hash, Some(20)); + let n20_id = *st.by_hash.get(&20).unwrap().iter().next().unwrap(); + assert_eq!(st.nodes[&n20_id].parent_block_hash, Some(10)); + let n10_id = *st.by_hash.get(&10).unwrap().iter().next().unwrap(); + assert_eq!(st.nodes[&n10_id].parent_block_hash, None); + } + + #[test] + fn remove_does_not_drop_node_held_by_other_workers() { + let tree = HashTree::new(); + let a = worker("http://a", 0); + let b = worker("http://b", 0); + tree.insert(&a, None, &[1, 2, 3]); + tree.insert(&b, None, &[1, 2, 3]); + assert_eq!(tree.node_count(), 3); + + // A removes its blocks; B still holds them. + tree.remove(&a, &[1, 2, 3]); + assert_eq!(tree.node_count(), 3); + + let m = tree.match_prefix(None, &[1, 2, 3]); + assert_eq!(m.matched_blocks, 3); + assert_eq!(m.workers, workers(&[&b])); + } +} diff --git a/experimental/sgl-router/src/policies/kv_events/wire.rs b/experimental/sgl-router/src/policies/kv_events/wire.rs new file mode 100644 index 000000000000..a7160696235c --- /dev/null +++ b/experimental/sgl-router/src/policies/kv_events/wire.rs @@ -0,0 +1,866 @@ +//! Wire-format types for SGLang's KV cache event stream. +//! +//! SGLang's `ZmqEventPublisher` (Python: +//! `python/sglang/srt/disaggregation/kv_events.py`) encodes batches with +//! `msgspec.msgpack`. Two struct families are involved: +//! +//! * `EventBatch` (the outer payload) — declared with +//! `array_like=True, omit_defaults=True, gc=False` (no tag). +//! * `KVCacheEvent` (each inner event variant) — additionally declared +//! with `tag=True`. +//! +//! The combined effect on the wire: +//! +//! * Each struct is a msgpack **array** of its fields in declaration +//! order, not a map. +//! * `tag=True` on `KVCacheEvent` prepends a class-name string at index 0 +//! of each inner event array, so an event is +//! `[class_name_str, field1, field2, ...]`. The outer `EventBatch` +//! array does **not** carry a tag prefix. +//! * `omit_defaults=True` allows trailing fields whose values equal their +//! declared defaults to be dropped from the array. The decoder therefore +//! accepts variable-length sequences for each struct shape. +//! +//! This module deserializes those bytes into Rust types and exposes a single +//! [`decode_event_batch`] entry point. + +use std::fmt; + +use serde::de::{self, Deserializer, IgnoredAny, SeqAccess, Visitor}; +use serde::Deserialize; + +/// Top-level batch payload published by SGLang. +/// +/// Wire shape (`EventBatch`, `array_like`): +/// `[ts: f64, events: [...], attn_dp_rank: int_or_nil_or_omitted]`. +/// SGLang declares `attn_dp_rank` as a Python `Optional[int]`; we decode +/// it as `u32` since DP ranks are non-negative and bounded by the +/// publisher's `dp_size`. +#[derive(Debug, Clone, PartialEq)] +pub struct KvEventBatch { + /// Wall-clock timestamp from the publisher (seconds since epoch). + pub ts: f64, + /// Ordered list of cache events in this batch. + pub events: Vec, + /// Optional DP-attention rank that produced this batch. `None` if the + /// publisher emitted nil or omitted the field via `omit_defaults`. + pub attn_dp_rank: Option, +} + +/// A single KV cache event. The Python base class `KVCacheEvent` uses +/// `tag=True`, so each event on the wire is an array whose first element +/// is the class-name discriminator. +#[derive(Debug, Clone, PartialEq)] +pub enum KvCacheEvent { + /// `["BlockStored", block_hashes, parent_block_hash, token_ids, + /// block_size, lora_id, medium?]`. + BlockStored(BlockStored), + /// `["BlockRemoved", block_hashes, medium?]`. + BlockRemoved(BlockRemoved), + /// `["AllBlocksCleared"]`. + AllBlocksCleared, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct BlockStored { + /// 64-bit block hashes in declaration order. Hashes can exceed `i32` + /// range; signedness matches SGLang's Python `int`. + pub block_hashes: Vec, + /// Hash of the parent block, or `None` for the first block in a chain. + pub parent_block_hash: Option, + /// Tokens covered by this block. SGLang uses 32-bit token IDs. + pub token_ids: Vec, + /// Block size (tokens per block). + pub block_size: u32, + /// LoRA adapter ID this block is associated with, if any. + pub lora_id: Option, + /// Storage tier (`"GPU"`, `"CPU_PINNED"`, `"DISK"`, `"EXTERNAL"`). + /// Optional in the Python schema (`= None` default), so it may be + /// omitted entirely under `omit_defaults`. + pub medium: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct BlockRemoved { + pub block_hashes: Vec, + /// Same semantics as [`BlockStored::medium`]. + pub medium: Option, +} + +/// Maximum number of block hashes a single decoded `BlockStored` / +/// `BlockRemoved` event may carry. A misbehaving worker (or a corrupted +/// frame) could otherwise prompt a multi-gigabyte allocation in the +/// gateway. Workers are inside the trust boundary, so this is +/// defense-in-depth — but the cost of *not* capping is unbounded memory +/// amplification, so we cap. +pub(crate) const MAX_HASHES_PER_EVENT: usize = 65_536; +/// Same rationale as [`MAX_HASHES_PER_EVENT`], but for `token_ids`. A +/// 1M-token block list is already absurdly larger than any realistic +/// `BlockStored` payload — the cap exists to bound the worst case, not +/// to constrain normal operation. +pub(crate) const MAX_TOKENS_PER_EVENT: usize = 1_048_576; + +/// Errors produced by [`decode_event_batch`]. +#[derive(thiserror::Error, Debug)] +pub enum DecodeError { + /// The msgpack payload was malformed or did not match the expected schema. + #[error("failed to decode KV event batch: {0}")] + Msgpack(#[from] rmp_serde::decode::Error), + /// A single event's variable-length field exceeded its hard cap. We + /// surface this as an error rather than panicking so a single bad + /// payload only kills its batch, not the consumer task. + #[error("KV event field {field} length {len} exceeds cap {cap}")] + PayloadTooLarge { + field: &'static str, + len: usize, + cap: usize, + }, +} + +/// Sentinel string a custom visitor uses to encode a "field too large" +/// error through serde's `de::Error::custom` channel. We rewrap as the +/// typed [`DecodeError::PayloadTooLarge`] in [`decode_event_batch`]. +const PAYLOAD_TOO_LARGE_TAG: &str = "kv_events::wire::PAYLOAD_TOO_LARGE"; + +/// Decode a single ZMQ payload frame from SGLang's `ZmqEventPublisher`. +/// +/// The payload is the `payload` arg to `_pub.send_multipart((topic, seq, +/// payload))` — the topic and 8-byte big-endian sequence number are separate +/// frames and are NOT part of the msgpack input here. +/// +/// Caps the per-event `block_hashes` and `token_ids` lengths +/// ([`MAX_HASHES_PER_EVENT`], [`MAX_TOKENS_PER_EVENT`]) so a misbehaving +/// worker — or a corrupted msgpack length prefix — cannot trigger an +/// unbounded allocation in the gateway. +pub fn decode_event_batch(bytes: &[u8]) -> Result { + match rmp_serde::from_slice::(bytes) { + Ok(b) => Ok(b), + Err(e) => { + // Rewrap the size-cap sentinel into the typed variant. The + // sentinel string is set by `BoundedI64Vec` / `BoundedU32Vec` + // below; everything else is a true msgpack decode failure. + let s = e.to_string(); + if let Some(rest) = s.strip_prefix(PAYLOAD_TOO_LARGE_TAG) { + // Format: ":::" + let mut parts = rest.trim_start_matches(':').split(':'); + if let (Some(field), Some(len), Some(cap)) = + (parts.next(), parts.next(), parts.next()) + { + if let (Ok(len), Ok(cap)) = (len.parse::(), cap.parse::()) { + let field = match field { + "block_hashes" => "block_hashes", + "token_ids" => "token_ids", + // Unknown — fall through to Msgpack. + _ => return Err(DecodeError::Msgpack(e)), + }; + return Err(DecodeError::PayloadTooLarge { field, len, cap }); + } + } + } + Err(DecodeError::Msgpack(e)) + } + } +} + +/// Newtype wrapping `Vec` whose `Deserialize` impl rejects sequences +/// announcing more than [`MAX_HASHES_PER_EVENT`] elements *before* doing +/// the per-element work. Required because `rmp-serde` pre-sizes the +/// destination `Vec` from the msgpack length prefix; a malicious or +/// corrupted prefix would otherwise prompt a multi-gigabyte allocation. +#[derive(Debug, Clone, PartialEq)] +struct BoundedI64Vec(Vec); + +impl<'de> Deserialize<'de> for BoundedI64Vec { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct V; + impl<'de> Visitor<'de> for V { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a msgpack array of i64 values") + } + fn visit_seq(self, mut seq: A) -> Result, A::Error> + where + A: SeqAccess<'de>, + { + if let Some(hint) = seq.size_hint() { + if hint > MAX_HASHES_PER_EVENT { + return Err(de::Error::custom(format!( + "{PAYLOAD_TOO_LARGE_TAG}:block_hashes:{hint}:{MAX_HASHES_PER_EVENT}" + ))); + } + } + let mut out: Vec = match seq.size_hint() { + Some(h) => Vec::with_capacity(h), + None => Vec::new(), + }; + while let Some(v) = seq.next_element::()? { + if out.len() >= MAX_HASHES_PER_EVENT { + return Err(de::Error::custom(format!( + "{PAYLOAD_TOO_LARGE_TAG}:block_hashes:{}:{MAX_HASHES_PER_EVENT}", + out.len() + 1 + ))); + } + out.push(v); + } + Ok(out) + } + } + let v = deserializer.deserialize_seq(V)?; + Ok(BoundedI64Vec(v)) + } +} + +/// `BoundedI64Vec`'s `u32` twin. Same shape, different cap. +#[derive(Debug, Clone, PartialEq)] +struct BoundedU32Vec(Vec); + +impl<'de> Deserialize<'de> for BoundedU32Vec { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct V; + impl<'de> Visitor<'de> for V { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a msgpack array of u32 values") + } + fn visit_seq(self, mut seq: A) -> Result, A::Error> + where + A: SeqAccess<'de>, + { + if let Some(hint) = seq.size_hint() { + if hint > MAX_TOKENS_PER_EVENT { + return Err(de::Error::custom(format!( + "{PAYLOAD_TOO_LARGE_TAG}:token_ids:{hint}:{MAX_TOKENS_PER_EVENT}" + ))); + } + } + let mut out: Vec = match seq.size_hint() { + Some(h) => Vec::with_capacity(h), + None => Vec::new(), + }; + while let Some(v) = seq.next_element::()? { + if out.len() >= MAX_TOKENS_PER_EVENT { + return Err(de::Error::custom(format!( + "{PAYLOAD_TOO_LARGE_TAG}:token_ids:{}:{MAX_TOKENS_PER_EVENT}", + out.len() + 1 + ))); + } + out.push(v); + } + Ok(out) + } + } + let v = deserializer.deserialize_seq(V)?; + Ok(BoundedU32Vec(v)) + } +} + +// --------------------------------------------------------------------------- +// Custom Deserialize impls — msgspec encodes these structs as msgpack arrays +// (not maps), and `omit_defaults=True` means trailing optional fields may be +// absent. We therefore implement `Deserialize` by hand against `SeqAccess`. +// --------------------------------------------------------------------------- + +impl<'de> Deserialize<'de> for KvEventBatch { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct BatchVisitor; + + impl<'de> Visitor<'de> for BatchVisitor { + type Value = KvEventBatch; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a msgpack array [ts, events, attn_dp_rank?]") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let ts: f64 = seq + .next_element()? + .ok_or_else(|| de::Error::missing_field("ts"))?; + let events: Vec = seq + .next_element()? + .ok_or_else(|| de::Error::missing_field("events"))?; + // attn_dp_rank may be present-as-nil, present-as-int, or + // omitted entirely under msgspec's `omit_defaults`. + let attn_dp_rank: Option = seq.next_element()?.unwrap_or(None); + // Drain any extra trailing fields a future schema might add + // (forward-compat). + while seq.next_element::()?.is_some() {} + Ok(KvEventBatch { + ts, + events, + attn_dp_rank, + }) + } + } + + deserializer.deserialize_seq(BatchVisitor) + } +} + +impl<'de> Deserialize<'de> for KvCacheEvent { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct EventVisitor; + + impl<'de> Visitor<'de> for EventVisitor { + type Value = KvCacheEvent; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a tagged msgpack array [class_name, ...fields]") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let tag: String = seq + .next_element()? + .ok_or_else(|| de::Error::missing_field("event tag"))?; + + match tag.as_str() { + "BlockStored" => { + let block_hashes: BoundedI64Vec = seq + .next_element()? + .ok_or_else(|| de::Error::missing_field("block_hashes"))?; + let parent_block_hash: Option = seq.next_element()?.unwrap_or(None); + let token_ids: BoundedU32Vec = seq + .next_element()? + .ok_or_else(|| de::Error::missing_field("token_ids"))?; + let block_size: u32 = seq + .next_element()? + .ok_or_else(|| de::Error::missing_field("block_size"))?; + // `lora_id` is `Optional[int]` with no default — it's + // always emitted, but as nil when absent. + let lora_id: Option = seq.next_element()?.unwrap_or(None); + // `medium` defaults to None and may be omitted. + let medium: Option = seq.next_element()?.unwrap_or(None); + while seq.next_element::()?.is_some() {} + Ok(KvCacheEvent::BlockStored(BlockStored { + block_hashes: block_hashes.0, + parent_block_hash, + token_ids: token_ids.0, + block_size, + lora_id, + medium, + })) + } + "BlockRemoved" => { + let block_hashes: BoundedI64Vec = seq + .next_element()? + .ok_or_else(|| de::Error::missing_field("block_hashes"))?; + let medium: Option = seq.next_element()?.unwrap_or(None); + while seq.next_element::()?.is_some() {} + Ok(KvCacheEvent::BlockRemoved(BlockRemoved { + block_hashes: block_hashes.0, + medium, + })) + } + "AllBlocksCleared" => { + while seq.next_element::()?.is_some() {} + Ok(KvCacheEvent::AllBlocksCleared) + } + other => Err(de::Error::unknown_variant( + other, + &["BlockStored", "BlockRemoved", "AllBlocksCleared"], + )), + } + } + } + + deserializer.deserialize_seq(EventVisitor) + } +} + +// --------------------------------------------------------------------------- +// Tests — golden bytes are constructed via the `rmp` low-level encoder so +// they exercise the exact msgpack array layout SGLang emits, independent of +// any Rust-side serializer. +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + use rmp::encode as mp; + + /// Encode a tagged event header `[tag, ...]` array of `total_len` + /// elements (tag included). + fn write_event_array(buf: &mut Vec, tag: &str, total_len: u32) { + mp::write_array_len(buf, total_len).unwrap(); + mp::write_str(buf, tag).unwrap(); + } + + fn write_i64_array(buf: &mut Vec, values: &[i64]) { + mp::write_array_len(buf, values.len() as u32).unwrap(); + for v in values { + mp::write_sint(buf, *v).unwrap(); + } + } + + fn write_u32_array(buf: &mut Vec, values: &[u32]) { + mp::write_array_len(buf, values.len() as u32).unwrap(); + for v in values { + mp::write_uint(buf, *v as u64).unwrap(); + } + } + + /// Build a full BlockStored event as msgspec would emit it (all 7 + /// elements: tag + 6 fields). `medium` may be Some/None. + fn build_block_stored_bytes( + block_hashes: &[i64], + parent: Option, + token_ids: &[u32], + block_size: u32, + lora_id: Option, + medium: Option<&str>, + ) -> Vec { + let mut buf = Vec::new(); + write_event_array(&mut buf, "BlockStored", 7); + write_i64_array(&mut buf, block_hashes); + match parent { + Some(v) => { + mp::write_sint(&mut buf, v).unwrap(); + } + None => mp::write_nil(&mut buf).unwrap(), + } + write_u32_array(&mut buf, token_ids); + mp::write_uint(&mut buf, block_size as u64).unwrap(); + match lora_id { + Some(v) => { + mp::write_sint(&mut buf, v).unwrap(); + } + None => mp::write_nil(&mut buf).unwrap(), + } + match medium { + Some(s) => mp::write_str(&mut buf, s).unwrap(), + None => mp::write_nil(&mut buf).unwrap(), + } + buf + } + + fn build_block_removed_bytes(block_hashes: &[i64], medium: Option<&str>) -> Vec { + let mut buf = Vec::new(); + write_event_array(&mut buf, "BlockRemoved", 3); + write_i64_array(&mut buf, block_hashes); + match medium { + Some(s) => mp::write_str(&mut buf, s).unwrap(), + None => mp::write_nil(&mut buf).unwrap(), + } + buf + } + + fn build_all_blocks_cleared_bytes() -> Vec { + let mut buf = Vec::new(); + write_event_array(&mut buf, "AllBlocksCleared", 1); + buf + } + + /// Wrap pre-encoded event bytes into a top-level KVEventBatch array + /// `[ts, [event0_bytes, event1_bytes, ...], attn_dp_rank_or_nil]`. + fn build_batch_bytes( + ts: f64, + event_bufs: &[Vec], + attn_dp_rank: Option, + include_dp_field: bool, + ) -> Vec { + let mut buf = Vec::new(); + let total_len = if include_dp_field { 3 } else { 2 }; + mp::write_array_len(&mut buf, total_len).unwrap(); + mp::write_f64(&mut buf, ts).unwrap(); + mp::write_array_len(&mut buf, event_bufs.len() as u32).unwrap(); + for ev in event_bufs { + buf.extend_from_slice(ev); + } + if include_dp_field { + match attn_dp_rank { + Some(v) => { + mp::write_uint(&mut buf, v as u64).unwrap(); + } + None => mp::write_nil(&mut buf).unwrap(), + } + } + buf + } + + #[test] + fn decodes_block_stored_with_all_fields() { + let event = build_block_stored_bytes( + &[1234567890123_i64, -987654321_i64], + Some(42), + &[10, 20, 30, 40], + 4, + Some(7), + Some("GPU"), + ); + let bytes = build_batch_bytes(123.456, &[event], Some(2), true); + + let batch = decode_event_batch(&bytes).expect("decode"); + assert_eq!(batch.ts, 123.456); + assert_eq!(batch.attn_dp_rank, Some(2)); + assert_eq!(batch.events.len(), 1); + match &batch.events[0] { + KvCacheEvent::BlockStored(b) => { + assert_eq!(b.block_hashes, vec![1234567890123_i64, -987654321_i64]); + assert_eq!(b.parent_block_hash, Some(42)); + assert_eq!(b.token_ids, vec![10, 20, 30, 40]); + assert_eq!(b.block_size, 4); + assert_eq!(b.lora_id, Some(7)); + assert_eq!(b.medium.as_deref(), Some("GPU")); + } + other => panic!("expected BlockStored, got {:?}", other), + } + } + + #[test] + fn decodes_block_stored_with_nil_optionals() { + let event = build_block_stored_bytes(&[1, 2, 3], None, &[5, 6], 16, None, None); + let bytes = build_batch_bytes(0.0, &[event], None, true); + + let batch = decode_event_batch(&bytes).expect("decode"); + match &batch.events[0] { + KvCacheEvent::BlockStored(b) => { + assert_eq!(b.parent_block_hash, None); + assert_eq!(b.lora_id, None); + assert_eq!(b.medium, None); + assert_eq!(b.block_size, 16); + } + other => panic!("unexpected variant: {:?}", other), + } + } + + #[test] + fn decodes_block_removed() { + let event = build_block_removed_bytes(&[100, 200], Some("DISK")); + let bytes = build_batch_bytes(1.0, &[event], Some(0), true); + + let batch = decode_event_batch(&bytes).expect("decode"); + match &batch.events[0] { + KvCacheEvent::BlockRemoved(r) => { + assert_eq!(r.block_hashes, vec![100, 200]); + assert_eq!(r.medium.as_deref(), Some("DISK")); + } + other => panic!("unexpected variant: {:?}", other), + } + } + + #[test] + fn decodes_all_blocks_cleared() { + let event = build_all_blocks_cleared_bytes(); + let bytes = build_batch_bytes(2.0, &[event], None, true); + + let batch = decode_event_batch(&bytes).expect("decode"); + assert_eq!(batch.events.len(), 1); + assert!(matches!(batch.events[0], KvCacheEvent::AllBlocksCleared)); + } + + #[test] + fn decodes_mixed_batch_preserving_order() { + let stored = build_block_stored_bytes(&[10], Some(1), &[1, 2], 2, None, Some("GPU")); + let removed = build_block_removed_bytes(&[20], None); + let cleared = build_all_blocks_cleared_bytes(); + let bytes = build_batch_bytes(99.0, &[stored, removed, cleared], Some(3), true); + + let batch = decode_event_batch(&bytes).expect("decode"); + assert_eq!(batch.events.len(), 3); + assert!(matches!(batch.events[0], KvCacheEvent::BlockStored(_))); + assert!(matches!(batch.events[1], KvCacheEvent::BlockRemoved(_))); + assert!(matches!(batch.events[2], KvCacheEvent::AllBlocksCleared)); + assert_eq!(batch.attn_dp_rank, Some(3)); + } + + #[test] + fn attn_dp_rank_omitted_decodes_as_none() { + // msgspec's `omit_defaults=True` may drop attn_dp_rank entirely from + // the wire array when it equals its default of None. + let event = build_all_blocks_cleared_bytes(); + let bytes = build_batch_bytes(5.0, &[event], None, /* include_dp_field */ false); + + let batch = decode_event_batch(&bytes).expect("decode"); + assert_eq!(batch.attn_dp_rank, None); + assert_eq!(batch.events.len(), 1); + } + + #[test] + fn medium_omitted_in_block_stored_decodes_as_none() { + // BlockStored with `medium` omitted entirely (omit_defaults can drop + // the trailing default-None field). 6 elements instead of 7. + let mut buf = Vec::new(); + write_event_array(&mut buf, "BlockStored", 6); + write_i64_array(&mut buf, &[1]); + mp::write_nil(&mut buf).unwrap(); // parent_block_hash + write_u32_array(&mut buf, &[1, 2]); + mp::write_uint(&mut buf, 2).unwrap(); // block_size + mp::write_nil(&mut buf).unwrap(); // lora_id + let bytes = build_batch_bytes(0.0, &[buf], None, true); + + let batch = decode_event_batch(&bytes).expect("decode"); + match &batch.events[0] { + KvCacheEvent::BlockStored(b) => assert_eq!(b.medium, None), + other => panic!("unexpected variant: {:?}", other), + } + } + + #[test] + fn medium_omitted_in_block_removed_decodes_as_none() { + // BlockRemoved with only [tag, block_hashes] (medium omitted). + let mut buf = Vec::new(); + write_event_array(&mut buf, "BlockRemoved", 2); + write_i64_array(&mut buf, &[42]); + let bytes = build_batch_bytes(0.0, &[buf], None, true); + + let batch = decode_event_batch(&bytes).expect("decode"); + match &batch.events[0] { + KvCacheEvent::BlockRemoved(r) => { + assert_eq!(r.block_hashes, vec![42]); + assert_eq!(r.medium, None); + } + other => panic!("unexpected variant: {:?}", other), + } + } + + #[test] + fn unknown_event_tag_is_rejected() { + let mut buf = Vec::new(); + write_event_array(&mut buf, "MysteryEvent", 1); + let bytes = build_batch_bytes(0.0, &[buf], None, true); + + let err = decode_event_batch(&bytes).expect_err("should reject unknown variant"); + let msg = format!("{err}"); + assert!( + msg.contains("MysteryEvent") || msg.contains("unknown variant"), + "unexpected error message: {msg}" + ); + } + + /// Golden bytes captured from the actual SGLang Python publisher + /// (`msgspec.msgpack.Encoder().encode(KVEventBatch(...))`). These + /// hex strings are produced by msgspec 0.21.1 against the schema in + /// `python/sglang/srt/disaggregation/kv_events.py` and lock down the + /// exact wire format the decoder is expected to consume. Regenerated + /// with `python -c '...msgspec.msgpack.Encoder().encode(...)'`. + mod msgspec_golden { + use super::super::*; + + fn hex_to_bytes(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + #[test] + fn full_block_stored() { + // EventBatch(ts=123.456, events=[BlockStored([1234567890123, -987654321], + // parent=42, tokens=[10,20,30,40], block_size=4, lora=7, medium="GPU")], + // attn_dp_rank=2) + let bytes = hex_to_bytes( + "93cb405edd2f1a9fbe779197ab426c6f636b53746f72656492cf0000011f71fb04cbd2c521974f2a940a141e280407a347505502", + ); + let batch = decode_event_batch(&bytes).expect("decode msgspec golden"); + assert_eq!(batch.ts, 123.456); + assert_eq!(batch.attn_dp_rank, Some(2)); + assert_eq!(batch.events.len(), 1); + match &batch.events[0] { + KvCacheEvent::BlockStored(b) => { + assert_eq!(b.block_hashes, vec![1234567890123_i64, -987654321_i64]); + assert_eq!(b.parent_block_hash, Some(42)); + assert_eq!(b.token_ids, vec![10, 20, 30, 40]); + assert_eq!(b.block_size, 4); + assert_eq!(b.lora_id, Some(7)); + assert_eq!(b.medium.as_deref(), Some("GPU")); + } + other => panic!("expected BlockStored, got {:?}", other), + } + } + + #[test] + fn block_stored_with_nil_optionals() { + // ts=0.0, BlockStored([1,2,3], parent=None, tokens=[5,6], block_size=16, + // lora=None, medium=None), attn_dp_rank=None + let bytes = hex_to_bytes( + "93cb00000000000000009197ab426c6f636b53746f72656493010203c092050610c0c0c0", + ); + let batch = decode_event_batch(&bytes).expect("decode msgspec golden"); + assert_eq!(batch.attn_dp_rank, None); + match &batch.events[0] { + KvCacheEvent::BlockStored(b) => { + assert_eq!(b.block_hashes, vec![1, 2, 3]); + assert_eq!(b.parent_block_hash, None); + assert_eq!(b.token_ids, vec![5, 6]); + assert_eq!(b.block_size, 16); + assert_eq!(b.lora_id, None); + assert_eq!(b.medium, None); + } + other => panic!("unexpected: {:?}", other), + } + } + + #[test] + fn block_removed_with_medium() { + // ts=1.0, [BlockRemoved([100, 200], medium="DISK")], attn_dp_rank=0 + let bytes = hex_to_bytes( + "93cb3ff00000000000009193ac426c6f636b52656d6f7665649264ccc8a44449534b00", + ); + let batch = decode_event_batch(&bytes).expect("decode msgspec golden"); + assert_eq!(batch.ts, 1.0); + assert_eq!(batch.attn_dp_rank, Some(0)); + match &batch.events[0] { + KvCacheEvent::BlockRemoved(r) => { + assert_eq!(r.block_hashes, vec![100, 200]); + assert_eq!(r.medium.as_deref(), Some("DISK")); + } + other => panic!("unexpected: {:?}", other), + } + } + + #[test] + fn all_blocks_cleared() { + // ts=2.0, [AllBlocksCleared()], attn_dp_rank=None + let bytes = + hex_to_bytes("93cb40000000000000009191b0416c6c426c6f636b73436c6561726564c0"); + let batch = decode_event_batch(&bytes).expect("decode msgspec golden"); + assert_eq!(batch.ts, 2.0); + assert_eq!(batch.attn_dp_rank, None); + assert_eq!(batch.events.len(), 1); + assert!(matches!(batch.events[0], KvCacheEvent::AllBlocksCleared)); + } + + #[test] + fn mixed_batch() { + // ts=99.0, [BlockStored, BlockRemoved, AllBlocksCleared], attn_dp_rank=3 + let bytes = hex_to_bytes( + "93cb4058c000000000009397ab426c6f636b53746f726564910a0192010202c0a347505593ac426c6f636b52656d6f7665649114c091b0416c6c426c6f636b73436c656172656403", + ); + let batch = decode_event_batch(&bytes).expect("decode msgspec golden"); + assert_eq!(batch.ts, 99.0); + assert_eq!(batch.attn_dp_rank, Some(3)); + assert_eq!(batch.events.len(), 3); + match &batch.events[0] { + KvCacheEvent::BlockStored(b) => { + assert_eq!(b.block_hashes, vec![10]); + assert_eq!(b.parent_block_hash, Some(1)); + assert_eq!(b.token_ids, vec![1, 2]); + assert_eq!(b.block_size, 2); + assert_eq!(b.lora_id, None); + assert_eq!(b.medium.as_deref(), Some("GPU")); + } + other => panic!("unexpected: {:?}", other), + } + match &batch.events[1] { + KvCacheEvent::BlockRemoved(r) => { + assert_eq!(r.block_hashes, vec![20]); + assert_eq!(r.medium, None); + } + other => panic!("unexpected: {:?}", other), + } + assert!(matches!(batch.events[2], KvCacheEvent::AllBlocksCleared)); + } + } + + #[test] + fn empty_payload_is_an_error() { + let err = decode_event_batch(&[]).expect_err("empty payload should fail"); + // Just assert we surfaced a Msgpack decode error. + assert!(matches!(err, DecodeError::Msgpack(_))); + } + + /// A `BlockStored` event whose `block_hashes` array prefix exceeds + /// the per-event cap must be rejected with `PayloadTooLarge` so a + /// misbehaving worker (or a corrupted msgpack length prefix) cannot + /// trigger an unbounded allocation in the gateway. We don't fill the + /// whole array — the visitor refuses on the size_hint alone. + #[test] + fn block_stored_with_too_many_hashes_rejected() { + let claimed = (MAX_HASHES_PER_EVENT + 1) as u32; + + let mut event = Vec::new(); + write_event_array(&mut event, "BlockStored", 7); + // Oversize block_hashes prefix; only one real element. The + // visitor's size_hint check fires before reading anything. + mp::write_array_len(&mut event, claimed).unwrap(); + mp::write_sint(&mut event, 0).unwrap(); + // Trailing bytes are ignored — decoder errors out earlier. + + let bytes = build_batch_bytes(0.0, &[event], None, true); + + let err = decode_event_batch(&bytes).expect_err("oversize hashes should fail"); + match err { + DecodeError::PayloadTooLarge { field, len, cap } => { + assert_eq!(field, "block_hashes"); + assert_eq!(cap, MAX_HASHES_PER_EVENT); + assert_eq!(len, claimed as usize); + } + other => panic!("expected PayloadTooLarge, got {other:?}"), + } + } + + /// `token_ids` cap — uses an oversize msgpack array length prefix. + /// rmp-serde reports `size_hint` from the prefix (an `array_len` is a + /// known length), so the visitor refuses before reading any element. + /// We deliberately under-fill the array to keep the test cheap; the + /// decoder rejects on the prefix alone. + #[test] + fn block_stored_oversize_token_ids_prefix_rejected() { + let claimed = (MAX_TOKENS_PER_EVENT + 1) as u32; + + let mut event = Vec::new(); + write_event_array(&mut event, "BlockStored", 7); + write_i64_array(&mut event, &[42_i64]); // block_hashes (small) + mp::write_nil(&mut event).unwrap(); // parent_block_hash + // Oversize token_ids: announce huge length but only write a + // single element. The visitor's size_hint check fires + // immediately and we never reach the truncated payload. + mp::write_array_len(&mut event, claimed).unwrap(); + mp::write_uint(&mut event, 0).unwrap(); + // Trailing bytes after the truncated array are ignored — the + // decoder errors out on the size_hint check before reading them. + + let bytes = build_batch_bytes(0.0, &[event], None, true); + + let err = decode_event_batch(&bytes).expect_err("oversize token prefix should fail"); + match err { + DecodeError::PayloadTooLarge { field, len, cap } => { + assert_eq!(field, "token_ids"); + assert_eq!(cap, MAX_TOKENS_PER_EVENT); + assert_eq!(len, claimed as usize); + } + other => panic!("expected PayloadTooLarge, got {other:?}"), + } + } + + /// `BlockRemoved` is also covered. Uses the `block_hashes` cap. + #[test] + fn block_removed_with_too_many_hashes_rejected() { + let claimed = (MAX_HASHES_PER_EVENT + 1) as u32; + + let mut event = Vec::new(); + write_event_array(&mut event, "BlockRemoved", 3); + mp::write_array_len(&mut event, claimed).unwrap(); + mp::write_sint(&mut event, 0).unwrap(); + // Trailing bytes ignored — decoder errors on the size hint. + + let bytes = build_batch_bytes(0.0, &[event], None, true); + + let err = decode_event_batch(&bytes).expect_err("oversize hashes should fail"); + match err { + DecodeError::PayloadTooLarge { field, cap, .. } => { + assert_eq!(field, "block_hashes"); + assert_eq!(cap, MAX_HASHES_PER_EVENT); + } + other => panic!("expected PayloadTooLarge, got {other:?}"), + } + } +} diff --git a/experimental/sgl-router/src/policies/mod.rs b/experimental/sgl-router/src/policies/mod.rs new file mode 100644 index 000000000000..4bad0fb1d19c --- /dev/null +++ b/experimental/sgl-router/src/policies/mod.rs @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +pub mod active_load; +pub mod cache_aware_zmq; +pub mod factory; +pub mod kv_events; +pub mod power_of_two; +pub mod random; +pub mod registry; +pub mod round_robin; + +use crate::discovery::ModelId; +use crate::workers::Worker; +use dashmap::DashMap; +use std::sync::Arc; + +/// Selection input — carries the request body so that cache-aware policies +/// can hash prefix tokens without reshaping the [`Policy`] trait. Today's +/// policies (round-robin, random, power-of-two) only read `workers`. +/// +/// Constructed via [`Self::new`]; accessors expose immutable references so +/// callers cannot mutate the model id or swap in a different body without +/// going through the constructor. +pub struct SelectionContext<'a> { + model: &'a ModelId, + request_body: Option<&'a [u8]>, +} + +impl<'a> SelectionContext<'a> { + pub fn new(model: &'a ModelId, request_body: Option<&'a [u8]>) -> Self { + Self { + model, + request_body, + } + } + + pub fn model(&self) -> &ModelId { + self.model + } + + pub fn request_body(&self) -> Option<&[u8]> { + self.request_body + } +} + +pub trait Policy: Send + Sync + std::fmt::Debug { + fn select(&self, workers: &[Arc], ctx: &SelectionContext<'_>) -> Option>; +} + +#[derive(Debug, Default)] +pub struct PolicyRegistry { + by_model: DashMap>, +} + +impl PolicyRegistry { + pub fn insert(&self, model: ModelId, policy: Arc) { + self.by_model.insert(model, policy); + } + + pub fn get(&self, model: &ModelId) -> Option> { + self.by_model.get(model).map(|p| p.clone()) + } +} diff --git a/experimental/sgl-router/src/policies/power_of_two.rs b/experimental/sgl-router/src/policies/power_of_two.rs new file mode 100644 index 000000000000..a6c4aacf6343 --- /dev/null +++ b/experimental/sgl-router/src/policies/power_of_two.rs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::policies::{Policy, SelectionContext}; +use crate::workers::Worker; +use rand::seq::IteratorRandom; +use std::sync::Arc; + +#[derive(Debug, Default)] +pub struct PowerOfTwoChoicesPolicy; + +impl PowerOfTwoChoicesPolicy { + pub fn new() -> Self { + Self + } +} + +impl Policy for PowerOfTwoChoicesPolicy { + fn select(&self, workers: &[Arc], _ctx: &SelectionContext<'_>) -> Option> { + match workers.len() { + 0 => None, + 1 => Some(workers[0].clone()), + _ => { + let mut rng = rand::thread_rng(); + let mut chosen = workers.iter().choose_multiple(&mut rng, 2); + chosen.sort_by_key(|w| w.active_load()); + Some(chosen[0].clone()) + } + } + } +} diff --git a/experimental/sgl-router/src/policies/random.rs b/experimental/sgl-router/src/policies/random.rs new file mode 100644 index 000000000000..05547429b93b --- /dev/null +++ b/experimental/sgl-router/src/policies/random.rs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::policies::{Policy, SelectionContext}; +use crate::workers::Worker; +use rand::seq::SliceRandom; +use std::sync::Arc; + +#[derive(Debug, Default)] +pub struct RandomPolicy; + +impl RandomPolicy { + pub fn new() -> Self { + Self + } +} + +impl Policy for RandomPolicy { + fn select(&self, workers: &[Arc], _ctx: &SelectionContext<'_>) -> Option> { + workers.choose(&mut rand::thread_rng()).cloned() + } +} diff --git a/experimental/sgl-router/src/policies/registry.rs b/experimental/sgl-router/src/policies/registry.rs new file mode 100644 index 000000000000..a2aa1e3a1cc7 --- /dev/null +++ b/experimental/sgl-router/src/policies/registry.rs @@ -0,0 +1,741 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Per-model PD pool resolution. +//! +//! Carries forward the fix from `sgl-project/sglang#25184`: in +//! prefill/decode (PD) disaggregation deployments, prefill traffic must +//! never select a decode worker and vice versa. This module is the +//! single chokepoint that classifies a model as PD or non-PD and exposes +//! pool-restricted candidate sets. +//! +//! # Classification +//! +//! A model is **PD-mode** if its [`WorkerRegistry`] contains workers +//! with [`WorkerMode::Prefill`] OR [`WorkerMode::Decode`]. A model is +//! **plain-mode** if it has only `WorkerMode::Plain` workers (or no +//! workers — both queries return empty for an unknown model). The +//! `(prefill, decode, plain)` partition is computed eagerly per call; +//! tests show this is cheaper than maintaining a side-table and +//! avoiding a race against discovery events. +//! +//! # Why not `Worker::mode` directly in the chat handler? +//! +//! Two reasons: +//! +//! 1. The classification is a *cohort* decision (does the model use PD?), +//! not a per-worker decision. Putting it in the handler means every +//! request route reimplements the same "are any of these prefill?" +//! walk. A central [`PdPoolResolver`] returns the same answer with +//! one call. +//! 2. Errors. The handler needs to distinguish "no workers at all" +//! (existing `NoHealthyWorkers`) from "no prefill workers +//! available for a PD-mode model" (new `NoPrefillWorkersAvailable`) +//! — only the resolver has the cohort context to tell which is which. + +use crate::discovery::{ModelId, WorkerMode}; +use crate::workers::{Worker, WorkerRegistry}; +use std::sync::Arc; + +/// Multiplier over the median decode-pool load above which a same-host +/// decode peer is considered "too hot" — we fall back to the lowest-load +/// peer outside the affinity preference. Two-times-median keeps short +/// load bursts on the same host (NCCL chatter, GPU sharing) from being +/// treated as overload while still avoiding pinning to a wedged peer. +const AFFINITY_LOAD_TOLERANCE: f64 = 2.0; + +/// Resolution result for a single request route. The handler picks +/// `prefill` / `decode` based on whether it is dispatching prefill or +/// decode traffic; `plain` is for non-PD models. +#[derive(Debug)] +pub enum PdPools { + /// Non-PD deployment: the model is served by plain workers. + Plain { workers: Vec> }, + /// PD-disaggregation deployment: the model has prefill and/or decode + /// workers. Either OR BOTH pools may be empty (e.g. every prefill + /// worker's circuit breaker is open, or every PD worker on the + /// model is currently unhealthy). The `*_candidates` helpers are + /// the only safe consumers — they map an empty pool to the + /// appropriate `NoPrefillWorkersAvailable` / `NoDecodeWorkersAvailable` + /// error. Callers that read this variant directly MUST treat an + /// empty pool as a transient failure, not as "zero work". + Pd { + prefill: Vec>, + decode: Vec>, + }, +} + +/// Reason the resolver could not satisfy a request — exposed so the +/// handler can map to the right HTTP error code. +#[derive(Debug, PartialEq, Eq)] +pub enum PdResolveError { + /// The model has no workers registered at all, healthy or not. + /// Surfaced as 503 `no_healthy_workers`. + NoHealthyWorkers, + /// PD-mode deployment whose prefill pool is empty (all + /// breakers-open or no prefill workers ever registered). + /// Surfaced as 503 `no_prefill_workers_available`. + NoPrefillWorkersAvailable, + /// PD-mode deployment whose decode pool is empty. + /// Surfaced as 503 `no_decode_workers_available`. + NoDecodeWorkersAvailable, +} + +/// Thin façade over [`WorkerRegistry`] that returns the per-pool +/// candidate sets for a model. Cheap to construct; the registry is +/// shared. +#[derive(Debug, Clone)] +pub struct PdPoolResolver { + workers: Arc, +} + +impl PdPoolResolver { + pub fn new(workers: Arc) -> Self { + Self { workers } + } + + /// Classify a model and return its pool partition over healthy + /// workers. Workers whose circuit breaker is open are filtered out + /// at this layer so the policy never has to re-check. + /// + /// Returns `Err(NoHealthyWorkers)` only when the model has zero + /// **registered** workers (healthy or not). When the model is + /// registered as PD but every PD worker is currently unhealthy + /// (any failure path that flips `breaker.allow()` to false), + /// returns `Ok(Pd { prefill: [], decode: [] })` so + /// `prefill_candidates` / `decode_candidates` can surface the more + /// specific `NoPrefillWorkersAvailable` / `NoDecodeWorkersAvailable` + /// code — operators alerting on partial-pool failures see the same + /// code whether the empty pool is empty by registration or by + /// transient health state. + pub fn resolve(&self, model: &ModelId) -> Result { + let all = self.workers.healthy_workers_for(model); + if all.is_empty() { + // No healthy workers — distinguish "model never registered" + // (true 404-ish, operator misconfiguration) from "PD model + // with all breakers currently open" (transient health + // issue, deserves the per-pool code). + let registered = self.workers.workers_for(model); + let pd_intent = registered + .iter() + .any(|w| matches!(w.mode(), WorkerMode::Prefill | WorkerMode::Decode)); + return if pd_intent { + Ok(PdPools::Pd { + prefill: Vec::new(), + decode: Vec::new(), + }) + } else { + Err(PdResolveError::NoHealthyWorkers) + }; + } + let mut prefill = Vec::new(); + let mut decode = Vec::new(); + let mut plain = Vec::new(); + for w in all { + match w.mode() { + WorkerMode::Prefill => prefill.push(w), + WorkerMode::Decode => decode.push(w), + WorkerMode::Plain => plain.push(w), + } + } + // PD-mode iff any prefill OR any decode worker exists. Mixing + // plain + prefill on the same model_id is a discovery-level + // misconfiguration we do not try to repair here — we treat any + // role tag at all as PD intent. The plain workers in that case + // become unreachable, which is loud enough at the metrics layer + // for operators to notice. + if !prefill.is_empty() || !decode.is_empty() { + Ok(PdPools::Pd { prefill, decode }) + } else { + Ok(PdPools::Plain { workers: plain }) + } + } + + /// Convenience for the prefill dispatch path. Returns the prefill + /// pool for a PD model, or the full plain pool for a non-PD model. + /// Errors when the relevant pool is empty. + pub fn prefill_candidates(&self, model: &ModelId) -> Result>, PdResolveError> { + match self.resolve(model)? { + PdPools::Plain { workers } => Ok(workers), + PdPools::Pd { prefill, .. } => { + if prefill.is_empty() { + Err(PdResolveError::NoPrefillWorkersAvailable) + } else { + Ok(prefill) + } + } + } + } + + /// Convenience for the decode dispatch path. Mirror of + /// [`Self::prefill_candidates`]. + pub fn decode_candidates(&self, model: &ModelId) -> Result>, PdResolveError> { + match self.resolve(model)? { + PdPools::Plain { workers } => Ok(workers), + PdPools::Pd { decode, .. } => { + if decode.is_empty() { + Err(PdResolveError::NoDecodeWorkersAvailable) + } else { + Ok(decode) + } + } + } + } + + /// Pick a decode worker for a PD-mode handoff with **host affinity** + /// to the prefill worker. Resolves the decode pool for `model`, then + /// applies the affinity rules in [`select_decode_with_affinity`]. + /// + /// Returns `Err(NoDecodeWorkersAvailable)` if the decode pool is + /// empty (PD-mode partial failure) — the chat handler then maps to + /// 503 `no_decode_workers_available`. For non-PD (plain) models + /// this is a no-op call — there is no decode peer to find — and + /// the caller should NOT use this helper. + pub fn decode_with_affinity( + &self, + model: &ModelId, + prefill_url: &str, + ) -> Result, PdResolveError> { + let candidates = self.decode_candidates(model)?; + select_decode_with_affinity(prefill_url, &candidates) + .ok_or(PdResolveError::NoDecodeWorkersAvailable) + } +} + +/// Pick a decode worker from `candidates` preferring the one whose URL +/// shares a host with `prefill_url`. Falls back to lowest-load when no +/// same-host peer exists, when the same-host peer's breaker is open, +/// or when the same-host peer is overloaded relative to the pool. +/// +/// # Rules +/// +/// 1. **Same-host preference.** Parse the host portion of both URLs +/// (`url::Url::host_str`). If any candidate shares the host AND has +/// a closed circuit breaker AND has `active_load <= +/// AFFINITY_LOAD_TOLERANCE × median(decode_pool_load)`, return it. +/// 2. **Fallback: min-load among closed-breaker candidates.** No +/// same-host peer, or the same-host peer was filtered by rule 1's +/// health/load gates. +/// 3. **Last resort: min-load over ALL candidates.** Every candidate +/// has its breaker open; the next dispatch will likely fail too, +/// but a min-load fallback keeps the selection function total. +/// Callers should observe the breaker-open error and surface it as +/// `BreakerOpen`, not silently retry. +/// +/// Returns `None` only when `candidates` is empty. +/// +/// # Why a free-standing function vs a `Policy::select` extension? +/// +/// The current `Policy` trait carries `(workers, ctx)`; adding an +/// `affinity_hint` argument would touch every policy implementation +/// (`round_robin`, `random`, `power_of_two`, `cache_aware_zmq`). +/// Affinity is a PD-routing concern — orthogonal to the in-pool +/// scoring the trait abstracts — so keeping it as a sibling helper +/// keeps the trait's responsibility narrow. +pub fn select_decode_with_affinity( + prefill_url: &str, + candidates: &[Arc], +) -> Option> { + if candidates.is_empty() { + return None; + } + let prefill_host = host_of(prefill_url); + + // Build the closed-breaker subset once; both the affinity branch + // and the fallback branch read from it. `would_allow` (non-mutating) + // is the right filter — `allow()` would claim a half-open probe for + // every candidate we look at, including ones we never dispatch to. + let healthy: Vec<&Arc> = candidates + .iter() + .filter(|w| w.breaker.would_allow()) + .collect(); + + // Compute the median load over the closed-breaker subset. Empty + // subset → median is 0 (means: every peer's breaker is open; the + // affinity gate is moot, we'll fall through to the last-resort + // branch). + let load_tolerance = if healthy.is_empty() { + 0 + } else { + let mut loads: Vec = healthy.iter().map(|w| w.active_load()).collect(); + loads.sort_unstable(); + let median = loads[loads.len() / 2]; + ((median as f64) * AFFINITY_LOAD_TOLERANCE).ceil() as usize + }; + + // Rule 1: same-host AND healthy AND not overloaded. + if let Some(host) = prefill_host.as_deref() { + let affinity_peer = healthy.iter().find(|w| { + host_of(&w.url).as_deref() == Some(host) + && (load_tolerance == 0 || w.active_load() <= load_tolerance) + }); + if let Some(w) = affinity_peer { + return Some(Arc::clone(w)); + } + } + + // Rule 2: min-load among healthy. + if let Some(w) = healthy.iter().min_by_key(|w| w.active_load()) { + return Some(Arc::clone(w)); + } + + // Rule 3: last-resort min-load over all candidates (every + // breaker is open). The caller's dispatch will likely fail and + // surface `BreakerOpen`, but the selection function stays total. + candidates.iter().min_by_key(|w| w.active_load()).cloned() +} + +/// Parse the host portion of a worker URL. Returns `None` when the URL +/// fails to parse or has no host (rare; discovery emits URLs the proxy +/// has already used at least once for /server_info, so this is mostly +/// defensive). +fn host_of(worker_url: &str) -> Option { + url::Url::parse(worker_url) + .ok()? + .host_str() + .map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::{ModelId, WorkerId, WorkerSpec}; + + fn spec(id: &str, mode: WorkerMode, model: &str) -> WorkerSpec { + WorkerSpec { + id: WorkerId(id.into()), + url: format!("http://{id}"), + mode, + model_ids: vec![ModelId(model.into())], + bootstrap_port: None, + } + } + + fn registry(specs: &[WorkerSpec]) -> Arc { + let r = Arc::new(WorkerRegistry::default()); + for s in specs { + let _ = r.add(s.clone()); + } + r + } + + /// Model with only Plain workers → Plain partition. + #[test] + fn plain_mode_returns_all_plain_workers() { + let r = registry(&[ + spec("w1", WorkerMode::Plain, "m"), + spec("w2", WorkerMode::Plain, "m"), + ]); + let res = PdPoolResolver::new(r) + .resolve(&ModelId("m".into())) + .unwrap(); + match res { + PdPools::Plain { workers } => assert_eq!(workers.len(), 2), + PdPools::Pd { .. } => panic!("expected Plain"), + } + } + + /// Model with prefill + decode → Pd partition, both pools populated. + #[test] + fn pd_mode_returns_distinct_pools() { + let r = registry(&[ + spec("p1", WorkerMode::Prefill, "m"), + spec("d1", WorkerMode::Decode, "m"), + spec("d2", WorkerMode::Decode, "m"), + ]); + let res = PdPoolResolver::new(r) + .resolve(&ModelId("m".into())) + .unwrap(); + match res { + PdPools::Pd { prefill, decode } => { + assert_eq!(prefill.len(), 1); + assert_eq!(decode.len(), 2); + // No cross-contamination: each worker carries the + // right mode. + assert!(prefill.iter().all(|w| w.mode() == WorkerMode::Prefill)); + assert!(decode.iter().all(|w| w.mode() == WorkerMode::Decode)); + } + PdPools::Plain { .. } => panic!("expected Pd"), + } + } + + /// Unknown model → NoHealthyWorkers. + #[test] + fn unknown_model_returns_no_healthy_workers() { + let r = Arc::new(WorkerRegistry::default()); + let err = PdPoolResolver::new(r) + .resolve(&ModelId("ghost".into())) + .unwrap_err(); + assert_eq!(err, PdResolveError::NoHealthyWorkers); + } + + /// Gap closer #1: PD mode with no prefill workers → resolve() + /// returns a Pd partition with an empty prefill pool, and + /// `prefill_candidates()` errors with NoPrefillWorkersAvailable. + #[test] + fn pd_mode_with_no_prefill_errors_on_prefill_dispatch() { + let r = registry(&[ + spec("d1", WorkerMode::Decode, "m"), + spec("d2", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + let model = ModelId("m".into()); + // resolve() succeeds — we have decode workers. + match resolver.resolve(&model).unwrap() { + PdPools::Pd { prefill, decode } => { + assert!(prefill.is_empty()); + assert_eq!(decode.len(), 2); + } + other => panic!("expected Pd, got {other:?}"), + } + // prefill_candidates errors. + let err = resolver.prefill_candidates(&model).unwrap_err(); + assert_eq!(err, PdResolveError::NoPrefillWorkersAvailable); + // decode_candidates succeeds. + let decode = resolver.decode_candidates(&model).unwrap(); + assert_eq!(decode.len(), 2); + } + + /// PD mode where every breaker is open (e.g. the upstream pool went + /// hard down) must NOT collapse to the generic `NoHealthyWorkers` + /// code. Both `prefill_candidates` and `decode_candidates` should + /// still surface the per-pool variant so operators can alert on + /// "prefill tier degraded" independently from "model misconfigured". + #[test] + fn pd_mode_all_breakers_open_keeps_per_pool_codes() { + let r = registry(&[ + spec("p1", WorkerMode::Prefill, "m"), + spec("d1", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + let model = ModelId("m".into()); + // Trip both breakers. Loop on `allow()` (not a fixed count) so + // the test stays correct if the default `CircuitBreakerConfig` + // threshold ever changes. + for w in resolver.workers.workers_for(&model) { + while w.breaker.allow() { + w.breaker.record_failure(); + } + } + // resolve() still returns a PD shape (both pools empty) — the + // PD intent is preserved across the breaker-open state. + match resolver.resolve(&model).unwrap() { + PdPools::Pd { prefill, decode } => { + assert!(prefill.is_empty()); + assert!(decode.is_empty()); + } + other => panic!("expected Pd, got {other:?}"), + } + // prefill dispatch → NoPrefillWorkersAvailable (not NoHealthyWorkers). + assert_eq!( + resolver.prefill_candidates(&model).unwrap_err(), + PdResolveError::NoPrefillWorkersAvailable, + ); + // decode dispatch → NoDecodeWorkersAvailable (not NoHealthyWorkers). + assert_eq!( + resolver.decode_candidates(&model).unwrap_err(), + PdResolveError::NoDecodeWorkersAvailable, + ); + } + + /// Symmetric: PD mode with no decode workers → decode dispatch + /// errors. + #[test] + fn pd_mode_with_no_decode_errors_on_decode_dispatch() { + let r = registry(&[ + spec("p1", WorkerMode::Prefill, "m"), + spec("p2", WorkerMode::Prefill, "m"), + ]); + let resolver = PdPoolResolver::new(r); + let model = ModelId("m".into()); + let err = resolver.decode_candidates(&model).unwrap_err(); + assert_eq!(err, PdResolveError::NoDecodeWorkersAvailable); + } + + /// PR #25184 carry-forward: separate models don't cross-contaminate. + /// One model is PD, the other is plain; resolving one must not return + /// workers from the other's pool. + #[test] + fn distinct_models_isolated_across_pd_and_plain() { + let r = registry(&[ + spec("plain1", WorkerMode::Plain, "plainmodel"), + spec("p1", WorkerMode::Prefill, "pdmodel"), + spec("d1", WorkerMode::Decode, "pdmodel"), + ]); + let resolver = PdPoolResolver::new(r); + match resolver.resolve(&ModelId("plainmodel".into())).unwrap() { + PdPools::Plain { workers } => assert_eq!(workers.len(), 1), + _ => panic!("plainmodel should resolve to Plain"), + } + match resolver.resolve(&ModelId("pdmodel".into())).unwrap() { + PdPools::Pd { prefill, decode } => { + assert_eq!(prefill.len(), 1); + assert_eq!(decode.len(), 1); + } + _ => panic!("pdmodel should resolve to Pd"), + } + } + + /// Plain-mode prefill_candidates returns the plain pool (non-PD + /// shorthand: dispatch helpers Just Work for plain models). + #[test] + fn plain_mode_prefill_candidates_returns_plain_pool() { + let r = registry(&[spec("w1", WorkerMode::Plain, "m")]); + let resolver = PdPoolResolver::new(r); + let v = resolver.prefill_candidates(&ModelId("m".into())).unwrap(); + assert_eq!(v.len(), 1); + assert_eq!(v[0].mode(), WorkerMode::Plain); + } + + // === Decoder affinity (Task C) === + + /// Build a `WorkerSpec` with an explicit URL — the affinity tests + /// distinguish workers by host, so they care about the URL string + /// directly, not the generated `http://{id}` form. + fn spec_with_url(id: &str, url: &str, mode: WorkerMode, model: &str) -> WorkerSpec { + WorkerSpec { + id: WorkerId(id.into()), + url: url.into(), + mode, + model_ids: vec![ModelId(model.into())], + bootstrap_port: None, + } + } + + /// Same-host affinity: a request that lands on `prefill@host_a` + /// picks `decode@host_a` even when `decode@host_b` has lower load. + /// Pin: the affinity branch wins over load tiebreak when both + /// candidates are healthy and not overloaded. + #[test] + fn decoder_picks_same_host_when_available() { + let r = registry(&[ + spec_with_url("p1", "http://host_a:30000", WorkerMode::Prefill, "m"), + spec_with_url("d1", "http://host_a:30001", WorkerMode::Decode, "m"), + spec_with_url("d2", "http://host_b:30001", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + let prefill_url = "http://host_a:30000"; + + let chosen = resolver + .decode_with_affinity(&ModelId("m".into()), prefill_url) + .unwrap(); + assert_eq!( + chosen.url, "http://host_a:30001", + "same-host decode peer must win over remote peer", + ); + } + + /// Affinity peer's breaker is open → fall back to the remote + /// healthy peer. Pin: the affinity rule must not pin a request to + /// a known-bad worker just because the host matches. + #[test] + fn decoder_falls_back_when_affinity_peer_breaker_open() { + let r = registry(&[ + spec_with_url("p1", "http://host_a:30000", WorkerMode::Prefill, "m"), + spec_with_url("d1", "http://host_a:30001", WorkerMode::Decode, "m"), + spec_with_url("d2", "http://host_b:30001", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + + // Trip d1's breaker by saturating record_failure() against the + // default config (threshold = 3). The breaker then denies + // `allow()` until the cooldown elapses. + let d1 = resolver + .workers + .healthy_workers_for(&ModelId("m".into())) + .into_iter() + .find(|w| w.url == "http://host_a:30001") + .unwrap(); + for _ in 0..3 { + d1.breaker.record_failure(); + } + assert!(!d1.breaker.allow(), "d1 breaker must be open"); + + let chosen = resolver + .decode_with_affinity(&ModelId("m".into()), "http://host_a:30000") + .unwrap(); + assert_eq!( + chosen.url, "http://host_b:30001", + "breaker-open affinity peer must fall back to the remote healthy peer", + ); + } + + /// Affinity peer is overloaded (load > 2× median) → fall back to + /// the remote lower-load peer. Pin: the load gate prevents a single + /// host's wedged decode worker from absorbing every co-located + /// prefill request. + #[test] + fn decoder_falls_back_when_affinity_peer_load_imbalance() { + let r = registry(&[ + spec_with_url("p1", "http://host_a:30000", WorkerMode::Prefill, "m"), + spec_with_url("d1", "http://host_a:30001", WorkerMode::Decode, "m"), + spec_with_url("d2", "http://host_b:30001", WorkerMode::Decode, "m"), + spec_with_url("d3", "http://host_c:30001", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + + // Loads: d1=20, d2=2, d3=2. Median = 2. 2× tolerance = 4. + // d1 is overloaded (20 > 4) → affinity rule rejects d1. + let decode_pool = resolver + .workers + .healthy_workers_for(&ModelId("m".into())) + .into_iter() + .filter(|w| w.mode() == WorkerMode::Decode) + .collect::>(); + let d1 = decode_pool + .iter() + .find(|w| w.url == "http://host_a:30001") + .unwrap(); + let d2 = decode_pool + .iter() + .find(|w| w.url == "http://host_b:30001") + .unwrap(); + let d3 = decode_pool + .iter() + .find(|w| w.url == "http://host_c:30001") + .unwrap(); + let mut guards = Vec::new(); + for _ in 0..20 { + guards.push(d1.load_guard()); + } + for _ in 0..2 { + guards.push(d2.load_guard()); + guards.push(d3.load_guard()); + } + + let chosen = resolver + .decode_with_affinity(&ModelId("m".into()), "http://host_a:30000") + .unwrap(); + assert!( + chosen.url == "http://host_b:30001" || chosen.url == "http://host_c:30001", + "overloaded affinity peer must fall back to a remote min-load peer, got: {}", + chosen.url, + ); + // Drop guards explicitly so the test cleanup doesn't depend on + // RAII order against the resolver / registry. + drop(guards); + } + + /// No same-host decode peer exists → fall back to min-load remote. + #[test] + fn decoder_falls_back_when_no_same_host_peer() { + let r = registry(&[ + spec_with_url("p1", "http://host_a:30000", WorkerMode::Prefill, "m"), + spec_with_url("d1", "http://host_b:30001", WorkerMode::Decode, "m"), + spec_with_url("d2", "http://host_c:30001", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + + // Bump d1 to 1, d2 stays at 0 — min-load picks d2. + let pool = resolver + .workers + .healthy_workers_for(&ModelId("m".into())) + .into_iter() + .filter(|w| w.mode() == WorkerMode::Decode) + .collect::>(); + let d1 = pool + .iter() + .find(|w| w.url == "http://host_b:30001") + .unwrap(); + let _g = d1.load_guard(); + + let chosen = resolver + .decode_with_affinity(&ModelId("m".into()), "http://host_a:30000") + .unwrap(); + assert_eq!( + chosen.url, "http://host_c:30001", + "no same-host peer → min-load fallback over remote candidates", + ); + } + + /// Empty decode pool → `NoDecodeWorkersAvailable`. The chat + /// handler maps this to 503 `no_decode_workers_available`. + #[test] + fn decoder_with_affinity_returns_error_when_pool_empty() { + let r = registry(&[spec_with_url( + "p1", + "http://host_a:30000", + WorkerMode::Prefill, + "m", + )]); + let resolver = PdPoolResolver::new(r); + let err = resolver + .decode_with_affinity(&ModelId("m".into()), "http://host_a:30000") + .unwrap_err(); + assert_eq!(err, PdResolveError::NoDecodeWorkersAvailable); + } + + /// Prefill URL is malformed (no host) → still picks a min-load + /// decode peer. Affinity is best-effort; a parse failure must not + /// kill the request. + #[test] + fn decoder_handles_malformed_prefill_url_via_min_load_fallback() { + let r = registry(&[ + spec_with_url("d1", "http://host_a:30001", WorkerMode::Decode, "m"), + spec_with_url("d2", "http://host_b:30001", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + let chosen = resolver + .decode_with_affinity(&ModelId("m".into()), "not-a-url") + .unwrap(); + // Both d1 and d2 are at load 0 → either is acceptable. The + // assertion is only that the function returns Some, not None + // / panic. + assert!( + chosen.url == "http://host_a:30001" || chosen.url == "http://host_b:30001", + "unexpected decode worker chosen: {}", + chosen.url, + ); + } + + /// All decode peers' breakers are open → `decode_with_affinity` + /// surfaces `NoDecodeWorkersAvailable` (the per-pool variant), not + /// the generic `NoHealthyWorkers`. The PD intent is preserved + /// through `resolve` so operators alerting on "decode tier down" + /// see the same code regardless of whether the pool is empty by + /// registration or by breaker state. + /// + /// The lower-level helper [`select_decode_with_affinity`] is total + /// even when every candidate's breaker is open (rule 3 in the + /// docstring): tests that call it directly with breaker-open + /// candidates get a min-load result. + #[test] + fn decoder_with_affinity_errors_when_all_breakers_open() { + let r = registry(&[ + spec_with_url("d1", "http://host_a:30001", WorkerMode::Decode, "m"), + spec_with_url("d2", "http://host_b:30001", WorkerMode::Decode, "m"), + ]); + let resolver = PdPoolResolver::new(r); + let pool = resolver + .workers + .workers_for(&ModelId("m".into())) + .into_iter() + .filter(|w| w.mode() == WorkerMode::Decode) + .collect::>(); + // Trip every decode breaker; loop on `allow()` for threshold + // resilience. + for w in &pool { + while w.breaker.allow() { + w.breaker.record_failure(); + } + } + // resolver path: healthy_workers_for returns empty, but the + // model is registered as PD (decode peers exist), so resolve() + // preserves PD shape and decode_with_affinity surfaces the + // per-pool code. + let err = resolver + .decode_with_affinity(&ModelId("m".into()), "http://host_a:30000") + .unwrap_err(); + assert_eq!(err, PdResolveError::NoDecodeWorkersAvailable); + + // helper path with a non-empty (but all-breaker-open) slice + // returns Some via the last-resort branch — selection function + // stays total, caller sees `BreakerOpen` on dispatch. + let any = select_decode_with_affinity("http://host_a:30000", &pool).unwrap(); + assert!( + any.url == "http://host_a:30001" || any.url == "http://host_b:30001", + "last-resort path must return some candidate, got: {}", + any.url, + ); + } +} diff --git a/experimental/sgl-router/src/policies/round_robin.rs b/experimental/sgl-router/src/policies/round_robin.rs new file mode 100644 index 000000000000..a023e74dd14e --- /dev/null +++ b/experimental/sgl-router/src/policies/round_robin.rs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::policies::{Policy, SelectionContext}; +use crate::workers::Worker; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +#[derive(Debug, Default)] +pub struct RoundRobinPolicy { + counter: AtomicUsize, +} + +impl RoundRobinPolicy { + pub fn new() -> Self { + Self::default() + } +} + +impl Policy for RoundRobinPolicy { + fn select(&self, workers: &[Arc], _ctx: &SelectionContext<'_>) -> Option> { + if workers.is_empty() { + return None; + } + let i = self.counter.fetch_add(1, Ordering::Relaxed) % workers.len(); + Some(workers[i].clone()) + } +} diff --git a/experimental/sgl-router/src/proxy/mod.rs b/experimental/sgl-router/src/proxy/mod.rs new file mode 100644 index 000000000000..a04877683983 --- /dev/null +++ b/experimental/sgl-router/src/proxy/mod.rs @@ -0,0 +1,250 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP proxy — forwards requests to the upstream SGLang worker. + +pub mod sse; + +use crate::health::circuit_breaker::CircuitBreaker; +use crate::server::error::ApiError; +use crate::server::header_utils::should_forward_request_header; +use anyhow::Context; +use axum::body::Body; +use axum::http::{HeaderMap, HeaderName, HeaderValue, Response}; +use bytes::Bytes; +use reqwest::{Client, Url}; +use std::sync::Arc; +use std::time::Duration; + +/// Parse a worker URL emitted by discovery. On failure, trip the worker's +/// circuit breaker so the malformed worker drops out of subsequent +/// `healthy_workers_for(...)` selection, then surface the error as +/// `ApiError::WorkerMisconfigured`. +fn parse_worker_url(worker_url: &str, breaker: &CircuitBreaker) -> Result { + Url::parse(worker_url).map_err(|e| { + breaker.record_failure(); + ApiError::WorkerMisconfigured { + worker: worker_url.to_string(), + source: anyhow::Error::new(e).context("parse worker URL"), + } + }) +} + +#[derive(Debug)] +pub struct Proxy { + pub client: Client, + /// Wall-clock timeout applied to non-streaming upstream requests. Streaming + /// requests deliberately do not use this (long generations are valid). + pub request_timeout: Duration, +} + +impl Proxy { + /// Build a proxy. `request_timeout` is the per-request wall-clock budget for + /// non-streaming forwards. Connect timeout is hard-coded to 5 s — even a + /// streaming request fails fast at TCP setup if the worker is unreachable. + pub fn new(request_timeout: Duration) -> Result { + let client = Client::builder() + .pool_max_idle_per_host(64) + .connect_timeout(Duration::from_secs(5)) + .build() + .context("build reqwest client")?; + Ok(Self { + client, + request_timeout, + }) + } + + /// Classify a reqwest error into the right `ApiError` variant, given an + /// explicit worker URL. Called from the breaker-gated `forward_*_to` + /// methods, which carry per-request worker URLs (not a single proxy-level + /// URL). + /// + /// Walks the full source chain to detect timeouts, because reqwest wraps + /// hyper which wraps `std::io::Error` — a top-level `is_timeout()` check + /// misses both the wrapped reqwest timeout and the `io::ErrorKind::TimedOut` + /// cases. + fn classify_reqwest_error_for(worker: Url, e: reqwest::Error, path: &str) -> ApiError { + let source = anyhow::Error::new(e).context(format!("worker {worker}: post {path}")); + let is_timeout = source.chain().any(|c| { + c.downcast_ref::() + .is_some_and(|r| r.is_timeout()) + }) || source.chain().any(|c| { + c.downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::TimedOut) + }); + if is_timeout { + ApiError::UpstreamTimeout { worker } + } else { + ApiError::UpstreamUnreachable { worker, source } + } + } + + /// Breaker-gated JSON POST: checks `breaker.allow()` first, records + /// success/failure based on response status, and returns + /// `ApiError::BreakerOpen` immediately when the breaker is Open. + /// + /// `worker_url` is the discovery-emitted worker URL string. It's parsed + /// to [`reqwest::Url`] internally so we can use [`Url::join`] for clean + /// path concatenation (no double-slash) and pass a typed URL to the + /// split error variants (`UpstreamUnreachable` / `UpstreamTimeout` / + /// `UpstreamStatus`). + pub async fn forward_json_to( + &self, + worker_url: &str, + breaker: &CircuitBreaker, + path: &str, + headers: &HeaderMap, + body: Bytes, + ) -> Result, ApiError> { + if !breaker.allow() { + return Err(ApiError::BreakerOpen { + worker: worker_url.to_string(), + }); + } + let worker_url = parse_worker_url(worker_url, breaker)?; + let url = worker_url.join(path).map_err(|e| { + ApiError::Internal(anyhow::Error::new(e).context(format!("join worker path {path}"))) + })?; + let mut req = self.client.post(url.clone()).body(body); + for (k, v) in headers { + if should_forward_request_header(k) { + req = req.header(k, v); + } + } + req = req + .header("content-type", "application/json") + .timeout(self.request_timeout); + let resp = req.send().await.map_err(|e| { + breaker.record_failure(); + Self::classify_reqwest_error_for(worker_url.clone(), e, path) + })?; + let status = resp.status(); + // Defer breaker recording until after the body completes — a + // worker that returns 2xx headers and then drops mid-body is + // still failing the request, and crediting it as healthy lets + // a misbehaving worker stay eligible. For 5xx the early bail is + // safe (no body to consume meaningfully), but we still wait + // until after the read attempt to record exactly once. + let bytes = match resp.bytes().await { + Ok(b) => b, + Err(e) => { + tracing::warn!( + upstream = %url, + status = %status, + error = ?e, + "upstream dropped connection mid-body", + ); + breaker.record_failure(); + return Err(ApiError::UpstreamStatus { status }); + } + }; + if status.is_server_error() { + breaker.record_failure(); + } else { + breaker.record_success(); + } + let mut out = Response::new(Body::from(bytes)); + *out.status_mut() = status; + out.headers_mut().insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/json"), + ); + Ok(out) + } + + /// Breaker-gated streaming POST: checks `breaker.allow()` first, records + /// success/failure, and returns `ApiError::BreakerOpen` when Open. + /// + /// `stream_guards` — when `Some`, the value is threaded into the SSE + /// pump task and held for the entire body lifetime (headers → last byte + /// / client disconnect). The proxy does not inspect the boxed value; it + /// relies entirely on `Drop` semantics, so callers typically pack + /// `(LoadGuard, ActiveLoadGuard)` here. This keeps both the per-worker + /// `active_requests` counter and the per-request active-load entry alive + /// for the full streaming lifetime — without which a long-running SSE + /// response would under-report load. + pub async fn forward_streaming_to( + &self, + worker_url: &str, + breaker: &Arc, + path: &str, + headers: &HeaderMap, + body: Bytes, + stream_guards: Option>, + ) -> Result, ApiError> { + if !breaker.allow() { + return Err(ApiError::BreakerOpen { + worker: worker_url.to_string(), + }); + } + let worker_url = parse_worker_url(worker_url, breaker)?; + let url = worker_url.join(path).map_err(|e| { + ApiError::Internal(anyhow::Error::new(e).context(format!("join worker path {path}"))) + })?; + let mut req = self.client.post(url.clone()).body(body); + for (k, v) in headers { + if should_forward_request_header(k) { + req = req.header(k, v); + } + } + req = req + .header("content-type", "application/json") + .header("accept", "text/event-stream"); + let resp = req.send().await.map_err(|e| { + breaker.record_failure(); + Self::classify_reqwest_error_for(worker_url.clone(), e, path) + })?; + let status = resp.status(); + let upstream_ct = resp + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("application/json") + .to_string(); + let content_type = if status.is_success() { + "text/event-stream".to_string() + } else { + upstream_ct + }; + // Breaker recording is deferred to the pump's completion hook so + // an upstream that returns 2xx headers and then drops mid-stream + // is recorded as a failure. For 5xx headers we record_failure + // up front and skip the pump hook (the body we surface is the + // error response — its stream completing is not a worker win). + let on_complete: Option> = + if status.is_server_error() { + breaker.record_failure(); + None + } else { + let breaker_for_hook = Arc::clone(breaker); + Some(Box::new(move |ok| { + if ok { + breaker_for_hook.record_success(); + } else { + breaker_for_hook.record_failure(); + } + })) + }; + let body = sse::bytes_stream_to_body(resp.bytes_stream(), stream_guards, on_complete); + let mut out = Response::new(body); + *out.status_mut() = status; + out.headers_mut().insert( + HeaderName::from_static("content-type"), + HeaderValue::from_str(&content_type) + .unwrap_or_else(|_| HeaderValue::from_static("application/json")), + ); + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[tokio::test] + async fn new_returns_result_not_panic() { + let p = Proxy::new(Duration::from_secs(5)).unwrap(); + assert_eq!(p.request_timeout, Duration::from_secs(5)); + } +} diff --git a/experimental/sgl-router/src/proxy/sse.rs b/experimental/sgl-router/src/proxy/sse.rs new file mode 100644 index 000000000000..63f082718669 --- /dev/null +++ b/experimental/sgl-router/src/proxy/sse.rs @@ -0,0 +1,324 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! SSE passthrough — bridges a reqwest `bytes_stream()` into an axum Body. + +use std::panic::AssertUnwindSafe; +use std::sync::Arc; + +use axum::body::Body; +use bytes::Bytes; +use futures::{FutureExt, StreamExt}; +use tokio_stream::wrappers::ReceiverStream; + +/// Bridge a byte stream into an axum Body that streams chunks unchanged. +/// +/// Spawns one tokio task per stream so the handler can return immediately. +/// Uses a **bounded** 64-slot channel so `tx.send().await` naturally +/// backpressures the upstream read when the client (axum Body consumer) falls +/// behind — an unbounded channel would buffer hundreds of MB for a slow client +/// receiving a long completion. +/// +/// # Backpressure note +/// The channel bound of 64 absorbs short bursts while still limiting +/// worst-case outstanding bytes to 64 × chunk_size (typically a few MB). +/// +/// # Client disconnect +/// When the axum Body is dropped the receiver is closed; `tx.send()` then +/// returns `Err`, which breaks the loop — no upstream bytes are read after the +/// client disconnects. +/// +/// # Panic safety +/// The pump future is wrapped in `AssertUnwindSafe(..).catch_unwind()`. If the +/// upstream stream panics, we surface a loud `io::Error` to the client; without +/// this, the body would EOF cleanly and clients couldn't distinguish that from +/// success — the worst failure class (truncated output that looks complete). +/// +/// # Stream guards +/// When `stream_guards` is `Some`, the value is **moved into the spawned task** +/// and held for the entire body lifetime. It is dropped only when the SSE +/// pump finishes (stream exhausted, client disconnects, or upstream errors). +/// The opaque `Box` accepts any drop-only payload — most +/// commonly a tuple of [`crate::workers::LoadGuard`] and +/// [`crate::policies::active_load::ActiveLoadGuard`]. The proxy does not +/// inspect the value; it relies entirely on `Drop` semantics, so callers can +/// pack arbitrary cleanup state in. Pass `None` for callers that manage the +/// guard externally (e.g. non-streaming paths where the handler itself is the +/// guard scope). +/// +/// # Completion hook +/// When `on_complete` is `Some`, the closure runs exactly once when the +/// pump task finishes. The bool argument is `true` on clean stream end +/// (including a clean client disconnect after at least the headers +/// landed cleanly), `false` on upstream stream error or pump panic. +/// `forward_streaming_to` passes a closure that records the worker's +/// circuit-breaker outcome — without this hook, a worker that returns +/// 2xx headers and then drops the stream mid-flight would stay credited +/// as healthy. +pub fn bytes_stream_to_body( + stream: S, + stream_guards: Option>, + on_complete: Option>, +) -> Body +where + S: futures::Stream> + Send + Unpin + 'static, + E: std::fmt::Display + Send + Sync + 'static, +{ + let (tx, rx) = tokio::sync::mpsc::channel(64); + tokio::spawn(async move { + let tx_for_panic = tx.clone(); + // Capture the pump's outcome so we can report it through `on_complete` + // AFTER `pump.catch_unwind()` settles. The closure inside owns + // `outcome_setter`; the outer scope reads `outcome_holder` once. + let outcome_holder = Arc::new(parking_lot::Mutex::new(true)); + let outcome_setter = Arc::clone(&outcome_holder); + let pump = AssertUnwindSafe(async move { + // Hold the guards for the task's lifetime — dropped when this + // block exits (stream done or client disconnect). Leading + // underscore suppresses the "unused variable" lint while + // keeping intent explicit. + let _hold = stream_guards; + let mut s = stream; + while let Some(chunk) = s.next().await { + let item: Result = chunk.map_err(|e| { + let msg = e.to_string(); + tracing::warn!(error = %msg, "upstream SSE stream errored mid-flight"); + std::io::Error::other(msg) + }); + let is_err_chunk = item.is_err(); + if is_err_chunk { + *outcome_setter.lock() = false; + } + if tx.send(item).await.is_err() { + // Receiver dropped. If we were about to ship an upstream + // error there's nothing left to report; otherwise this is + // a clean client-side disconnect — log at debug since it's + // not a router-side fault. + if !is_err_chunk { + tracing::debug!("SSE client disconnected mid-stream"); + } + break; + } + if is_err_chunk { + // Surfaced upstream error to client; stop reading. + break; + } + } + }); + let pump_result = pump.catch_unwind().await; + let panicked = pump_result.is_err(); + if let Err(panic_payload) = pump_result { + let msg = panic_payload + .downcast_ref::<&'static str>() + .map(|s| (*s).to_string()) + .or_else(|| panic_payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "".to_string()); + tracing::error!(error = %msg, "SSE pump task panicked"); + let _ = tx_for_panic + .send(Err(std::io::Error::other(format!( + "SSE pump panicked: {msg}" + )))) + .await; + } + if let Some(hook) = on_complete { + let ok = !panicked && *outcome_holder.lock(); + hook(ok); + } + }); + Body::from_stream(ReceiverStream::new(rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use futures::stream; + use http_body_util::BodyExt; + + #[tokio::test] + async fn passes_through_a_simple_byte_stream() { + let chunks = vec![ + Ok::(Bytes::from_static(b"hello ")), + Ok(Bytes::from_static(b"world")), + ]; + let s = stream::iter(chunks); + let body = bytes_stream_to_body(s, None, None); + let bytes = body.collect().await.unwrap().to_bytes(); + assert_eq!(&bytes[..], b"hello world"); + } + + #[tokio::test] + async fn upstream_error_surfaces_to_consumer() { + let chunks: Vec> = vec![ + Ok(Bytes::from_static(b"ok-chunk")), + Err(std::io::Error::other("upstream blew up mid-stream")), + ]; + let s = stream::iter(chunks); + let body = bytes_stream_to_body(s, None, None); + // Collecting a body that terminates with an error must return Err. + let result = body.collect().await; + assert!( + result.is_err(), + "expected body collect to surface upstream error, got Ok" + ); + } + + /// A stream that yields one Ok chunk on the first poll, then panics on the + /// second poll. Used to exercise the pump's panic-catch path. + struct PanicOnSecondPoll { + polls: usize, + } + + impl futures::Stream for PanicOnSecondPoll { + type Item = Result; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.polls += 1; + match self.polls { + 1 => std::task::Poll::Ready(Some(Ok(Bytes::from_static(b"first-chunk")))), + _ => panic!("synthetic pump panic from stream poll"), + } + } + } + + /// A stream that yields one Ok chunk, then panics with a non-string + /// payload (`i32`). Used to exercise the `` + /// fallback in the downcast ladder — the existing + /// `PanicOnSecondPoll` test only covers the `&'static str` arm. + struct PanicAnyOnSecondPoll { + polls: usize, + } + + impl futures::Stream for PanicAnyOnSecondPoll { + type Item = Result; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.polls += 1; + match self.polls { + 1 => std::task::Poll::Ready(Some(Ok(Bytes::from_static(b"first-chunk")))), + _ => std::panic::panic_any(42_i32), + } + } + } + + #[tokio::test] + async fn bytes_stream_to_body_handles_non_string_panic_payload() { + // `panic_any(42_i32)` skips the formatter entirely — neither the + // `&'static str` nor the `String` downcast arms match, so the + // catch_unwind handler must fall through to the + // `""` literal. If a refactor deletes + // that arm, the closure unwrap-or-elses would panic itself or + // produce an empty message, which this test catches. + let s = PanicAnyOnSecondPoll { polls: 0 }; + let body = bytes_stream_to_body(s, None, None); + let result = body.collect().await; + assert!( + result.is_err(), + "expected body collect to surface non-string panic as Err, got Ok" + ); + let err = result.err().unwrap(); + let msg = format!("{err}"); + assert!( + msg.contains(""), + "expected fallback message for non-string panic payload, got: {msg}" + ); + assert!( + msg.contains("SSE pump panicked"), + "expected wrapper message to remain, got: {msg}" + ); + } + + #[tokio::test] + async fn bytes_stream_to_body_propagates_pump_panic() { + // The pump task panics mid-stream. The client must see a loud Err, + // NOT a silently-truncated success. + let s = PanicOnSecondPoll { polls: 0 }; + let body = bytes_stream_to_body(s, None, None); + let result = body.collect().await; + assert!( + result.is_err(), + "expected body collect to surface pump panic as Err, got Ok (silent truncation)" + ); + let err = result.err().unwrap(); + let msg = format!("{err}"); + assert!( + msg.contains("pump panicked") || msg.contains("SSE pump panicked"), + "expected error message to mention pump panic, got: {msg}" + ); + } + + /// Regression guard for the backpressure-via-disconnect invariant. + /// + /// The doc on `bytes_stream_to_body` claims "when the axum Body is dropped + /// the receiver is closed; `tx.send()` then returns `Err`, which breaks the + /// loop — no upstream bytes are read after the client disconnects." This + /// test pins that contract: a refactor that swaps the `if tx.send().await. + /// is_err() { break; }` for `let _ = tx.send().await;` would silently + /// regress (leaked upstream reads on every client cancel, visible only as + /// ops-side memory growth). + #[tokio::test] + async fn bytes_stream_to_body_breaks_on_client_disconnect() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + // A stream that yields N Ok chunks readily, counting polls via a shared + // atomic. After we read 1 chunk and drop the body, the pump must hit + // tx.send-err and break — not drain all 1000 chunks. + struct CountingStream { + polls: Arc, + yielded: usize, + max: usize, + } + + impl futures::Stream for CountingStream { + type Item = Result; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.polls.fetch_add(1, Ordering::SeqCst); + if self.yielded >= self.max { + return std::task::Poll::Ready(None); + } + self.yielded += 1; + std::task::Poll::Ready(Some(Ok(Bytes::from_static(b"chunk")))) + } + } + + let polls = Arc::new(AtomicUsize::new(0)); + let stream = CountingStream { + polls: polls.clone(), + yielded: 0, + max: 1000, // way more than we'll let it consume + }; + let body = bytes_stream_to_body(stream, None, None); + + // Read exactly one frame, then drop the body to simulate client disconnect. + let mut data_stream = body.into_data_stream(); + let first = data_stream.next().await; + assert!(first.is_some(), "expected at least one chunk before drop"); + drop(data_stream); + + // Give the pump generous time to make additional polls if its break is + // broken. Healthy code: pump fills the 64-slot channel, then on the + // next iteration tx.send().await detects receiver-drop and breaks. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + let final_polls = polls.load(Ordering::SeqCst); + assert!( + final_polls <= 70, + "pump kept polling upstream after client disconnect: {final_polls} polls (expected <=70, channel bound + slack)" + ); + // And: the pump must NOT have drained all 1000 chunks. + assert!( + final_polls < 1000, + "pump drained the entire upstream after client disconnect ({final_polls} polls); the break-on-tx.send-err path is dead" + ); + } +} diff --git a/experimental/sgl-router/src/server/app.rs b/experimental/sgl-router/src/server/app.rs new file mode 100644 index 000000000000..c727762c5c0a --- /dev/null +++ b/experimental/sgl-router/src/server/app.rs @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::server::app_context::AppContext; +use crate::server::routes::chat::MAX_CHAT_BODY_BYTES; +use axum::extract::{DefaultBodyLimit, Request}; +use axum::http::StatusCode; +use axum::middleware::{self, Next}; +use axum::response::Response; +use axum::routing::{get, post}; +use axum::Router; +use std::sync::Arc; + +/// Middleware: log 413 PAYLOAD_TOO_LARGE responses with the request method +/// and URI so an operator investigating "client X gets 413s" has a +/// server-side breadcrumb. The 413 is produced by axum's `DefaultBodyLimit` +/// layer BEFORE the handler runs, so without this we would have no record +/// of which request was rejected. +async fn log_413(req: Request, next: Next) -> Response { + let method = req.method().clone(); + let uri = req.uri().clone(); + let resp = next.run(req).await; + if resp.status() == StatusCode::PAYLOAD_TOO_LARGE { + tracing::warn!( + %method, + %uri, + "request rejected with 413 PAYLOAD_TOO_LARGE (body exceeded route limit)", + ); + } + resp +} + +pub fn build_router(ctx: Arc) -> Router { + Router::new() + .route("/healthz", get(crate::server::routes::health::healthz)) + .route("/readyz", get(crate::server::routes::health::readyz)) + .route("/metrics", get(crate::server::routes::metrics::metrics)) + .route( + "/v1/models", + get(crate::server::routes::models::list_models), + ) + .route( + "/v1/tokenize", + post(crate::server::routes::tokenize::tokenize), + ) + .route( + "/v1/detokenize", + post(crate::server::routes::tokenize::detokenize), + ) + .route( + "/v1/chat/completions", + post(crate::server::routes::chat::chat_completions) + .layer(DefaultBodyLimit::max(MAX_CHAT_BODY_BYTES)) + .layer(middleware::from_fn(log_413)), + ) + .with_state(ctx) +} diff --git a/experimental/sgl-router/src/server/app_context.rs b/experimental/sgl-router/src/server/app_context.rs new file mode 100644 index 000000000000..15bc289c420b --- /dev/null +++ b/experimental/sgl-router/src/server/app_context.rs @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::config::Config; + +use crate::policies::active_load::ActiveLoadRegistry; +use crate::policies::PolicyRegistry; +use crate::proxy::Proxy; +use crate::server::metrics::MetricsRegistry; +use crate::tokenizer::TokenizerRegistry; +use crate::workers::WorkerRegistry; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +#[derive(Debug)] +pub struct AppContext { + pub config: Config, + pub tokenizers: Arc, + pub proxy: Arc, + pub registry: Arc, + pub policies: Arc, + /// Per-worker active-load bookkeeping. Shared between the proxy + /// (which mints guards on the request hot path), the cache-aware + /// policy (which reads per-worker load when scoring candidates), and + /// the stale-request janitor (which sweeps expired entries). + pub active_load: Arc, + /// Lightweight Prometheus-format metrics registry served via + /// `/metrics`. Shared with the chat handler (requests_total), + /// cache-aware-zmq policy (overlap_blocks), active-load registry + /// (active_load gauge + stale_requests_total), and PD resolver + /// (decode_affinity_total). + pub metrics: Arc, + ready: AtomicBool, +} + +impl AppContext { + pub fn new( + config: Config, + tokenizers: Arc, + proxy: Arc, + registry: Arc, + policies: Arc, + ) -> Self { + Self::with_active_load( + config, + tokenizers, + proxy, + registry, + policies, + ActiveLoadRegistry::with_defaults(), + ) + } + + /// Construct an [`AppContext`] with an explicit [`ActiveLoadRegistry`]. + /// Production wires the default (5-minute timeout, SystemTimeClock) + /// via [`Self::new`]; tests that exercise the janitor pass a registry + /// built with a `MockClock`. + pub fn with_active_load( + config: Config, + tokenizers: Arc, + proxy: Arc, + registry: Arc, + policies: Arc, + active_load: Arc, + ) -> Self { + let metrics = MetricsRegistry::new(); + // Wire the per-worker active-load gauge so `sgl_router_active_load` + // mirrors the live counter on every register / drop / sweep. + // Without this, the metric is permanently 0 in production even + // though the chat handler is faithfully calling `register`. + active_load.attach_metrics(Arc::clone(&metrics)); + Self { + config, + tokenizers, + proxy, + registry, + policies, + active_load, + metrics, + ready: AtomicBool::new(false), + } + } + + pub fn mark_ready(&self) { + // Relaxed: this flag does not synchronize other state; readers only + // care about eventual visibility, not happens-before with surrounding ops. + self.ready.store(true, Ordering::Relaxed); + } + + pub fn is_ready(&self) -> bool { + self.ready.load(Ordering::Relaxed) + } + + #[cfg(test)] + pub fn stub() -> Self { + Self { + config: Config { + server: crate::config::ServerConfig { + host: "x".into(), + port: 0, + }, + observability: Default::default(), + models: vec![], + discovery: crate::config::DiscoveryConfig { + backend: crate::config::DiscoveryBackend::StaticUrls( + crate::config::StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }, + ), + }, + proxy: crate::config::ProxyConfig::default(), + active_load: crate::config::ActiveLoadConfig::default(), + }, + tokenizers: Arc::new(TokenizerRegistry::default()), + proxy: Arc::new(Proxy::new(std::time::Duration::from_secs(60)).expect("stub proxy")), + registry: Arc::new(WorkerRegistry::default()), + policies: Arc::new(PolicyRegistry::default()), + active_load: ActiveLoadRegistry::with_defaults(), + metrics: MetricsRegistry::new(), + ready: AtomicBool::new(false), + } + } +} diff --git a/experimental/sgl-router/src/server/error.rs b/experimental/sgl-router/src/server/error.rs new file mode 100644 index 000000000000..bce6d631f0b5 --- /dev/null +++ b/experimental/sgl-router/src/server/error.rs @@ -0,0 +1,426 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use axum::http::{HeaderName, HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::Json; +use serde::Serialize; +use thiserror::Error; + +pub const X_ROUTER_ERROR_CODE: HeaderName = HeaderName::from_static("x-router-error-code"); + +#[derive(Debug, Error)] +pub enum ApiError { + #[error("bad request: {0}")] + BadRequest(String), + + #[error("model not found: {0}")] + ModelNotFound(String), + + /// Could not reach the upstream worker (connect refused, DNS, TLS, request + /// build error). `source` captures the full anyhow chain for server-side + /// logging; clients see a generic message. + /// + /// `worker` is the typed `reqwest::Url` so we don't re-stringify a value + /// that is already a `Url` at the construction site. Rendering goes + /// through `Display`, which produces the same canonical form as + /// `Url::as_str()`. + #[error("upstream unreachable: worker {worker}")] + UpstreamUnreachable { + worker: reqwest::Url, + #[source] + source: anyhow::Error, + }, + + /// The worker started a response (status + headers received) but failed + /// to deliver the full body — mid-body socket drop, framing error, etc. + /// Distinct from `UpstreamUnreachable` (no reply at all) and from a + /// well-formed non-2xx (which `Proxy` forwards verbatim with the worker's + /// own body). + #[error("upstream returned status {status}")] + UpstreamStatus { status: StatusCode }, + + /// Wall-clock timeout exceeded while waiting for the upstream worker's + /// response (per-request `request_timeout`). + /// + /// `worker` is the typed `reqwest::Url` for the same reason as + /// `UpstreamUnreachable`. + #[error("upstream timed out: worker {worker}")] + UpstreamTimeout { worker: reqwest::Url }, + + /// No healthy worker is available for `model`: either none were ever + /// registered, or every candidate's circuit breaker is open. Clients + /// should retry; operators should check discovery + worker health. + #[error("no healthy workers for model {model}")] + NoHealthyWorkers { model: String }, + + /// PD-mode deployment whose prefill pool has zero healthy workers. + /// Distinct from `NoHealthyWorkers` because the decode pool may + /// still be healthy — the failure is pool-specific, and surfacing + /// the distinct code lets operators alert on prefill-fleet outages + /// independently of full-model outages. + #[error("no prefill workers available for model {model}")] + NoPrefillWorkersAvailable { model: String }, + + /// PD-mode deployment whose decode pool has zero healthy workers. + /// Mirror of [`Self::NoPrefillWorkersAvailable`]. + #[error("no decode workers available for model {model}")] + NoDecodeWorkersAvailable { model: String }, + + /// A request whose lifetime exceeded `stale_request_timeout` — the + /// active-load janitor force-expired the in-flight bookkeeping + /// AND fired the per-request cancellation token, which the chat + /// handler `select!`-races against the upstream fetch. When the + /// token wins, the handler returns this variant → HTTP 504 → + /// client sees `stale_request_expired`. + /// + /// Mapped to 504 (not 503) because the failure is a router-side + /// gateway timeout from the client's perspective: the upstream + /// worker is still potentially fine, the router gave up because + /// the per-request budget elapsed. + #[error("stale request expired for model {model}")] + StaleRequestExpired { model: String }, + + /// The per-model policy returned `None` despite the candidate set + /// being non-empty. Almost always a router bug or an unsupported + /// policy state; surfaced as 503 (not 500) so retry-on-failure clients + /// can drain through a rotation rather than fail-fast on internal_error. + #[error("policy selected no worker for model {model}")] + PolicySelectionFailed { model: String }, + + /// The worker's circuit breaker was open at the moment of dispatch. + /// Surfaced post-policy-selection (race with `healthy_workers_for`); + /// the next selection will skip this worker. + #[error("worker circuit breaker open: {worker}")] + BreakerOpen { worker: String }, + + /// The worker URL emitted by discovery failed to parse. Always a + /// config / discovery-backend bug, not a transient infra issue — but + /// from the client's perspective the worker is unreachable, so 503. + /// The forwarder trips the circuit breaker before returning so the + /// malformed worker drops out of subsequent selection. + #[error("worker misconfigured: {worker}")] + WorkerMisconfigured { + worker: String, + #[source] + source: anyhow::Error, + }, + + #[error("internal: {0}")] + Internal(#[from] anyhow::Error), +} + +impl ApiError { + fn status_and_code(&self) -> (StatusCode, &'static str) { + match self { + ApiError::BadRequest(_) => (StatusCode::BAD_REQUEST, "bad_request"), + ApiError::ModelNotFound(_) => (StatusCode::NOT_FOUND, "model_not_found"), + ApiError::UpstreamUnreachable { .. } => { + (StatusCode::BAD_GATEWAY, "upstream_unreachable") + } + ApiError::UpstreamStatus { .. } => (StatusCode::BAD_GATEWAY, "upstream_status"), + ApiError::UpstreamTimeout { .. } => (StatusCode::BAD_GATEWAY, "upstream_timeout"), + ApiError::NoHealthyWorkers { .. } => { + (StatusCode::SERVICE_UNAVAILABLE, "no_healthy_workers") + } + ApiError::NoPrefillWorkersAvailable { .. } => ( + StatusCode::SERVICE_UNAVAILABLE, + "no_prefill_workers_available", + ), + ApiError::NoDecodeWorkersAvailable { .. } => ( + StatusCode::SERVICE_UNAVAILABLE, + "no_decode_workers_available", + ), + ApiError::StaleRequestExpired { .. } => { + (StatusCode::GATEWAY_TIMEOUT, "stale_request_expired") + } + ApiError::PolicySelectionFailed { .. } => { + (StatusCode::SERVICE_UNAVAILABLE, "policy_selection_failed") + } + ApiError::BreakerOpen { .. } => (StatusCode::SERVICE_UNAVAILABLE, "breaker_open"), + ApiError::WorkerMisconfigured { .. } => { + (StatusCode::SERVICE_UNAVAILABLE, "worker_misconfigured") + } + ApiError::Internal(_) => (StatusCode::INTERNAL_SERVER_ERROR, "internal_error"), + } + } +} + +#[derive(Serialize)] +struct ErrorEnvelope<'a> { + error: ErrorBody<'a>, +} + +#[derive(Serialize)] +struct ErrorBody<'a> { + #[serde(rename = "type")] + typ: &'static str, + code: &'a str, + message: String, +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let (status, code) = self.status_and_code(); + let typ = match status.as_u16() { + 400..=499 => "invalid_request_error", + _ => "server_error", + }; + // Pick a client-facing message that NEVER leaks worker URLs or raw + // source chains; full structured details are logged server-side. + let message = match &self { + ApiError::Internal(e) => { + // `{:#}` prints the anyhow chain (top error + sources) — `?e` + // would only show the outermost message. + tracing::error!("internal error serving request: {e:#}"); + "internal error".to_string() + } + ApiError::UpstreamUnreachable { worker, source } => { + tracing::warn!( + upstream = %worker, + error = %format_args!("{source:#}"), + "upstream worker unreachable", + ); + "upstream unavailable".to_string() + } + ApiError::UpstreamStatus { status } => { + tracing::warn!( + upstream_status = %status, + "upstream returned an error status", + ); + "upstream returned an error status".to_string() + } + ApiError::UpstreamTimeout { worker } => { + tracing::warn!(upstream = %worker, "upstream request timed out"); + "upstream request timed out".to_string() + } + ApiError::NoHealthyWorkers { model } => { + tracing::warn!(model = %model, reason = "no_healthy_workers", "service unavailable"); + "no healthy workers for the requested model".to_string() + } + ApiError::NoPrefillWorkersAvailable { model } => { + tracing::warn!( + model = %model, + reason = "no_prefill_workers_available", + "service unavailable", + ); + "no prefill workers available for the requested model".to_string() + } + ApiError::NoDecodeWorkersAvailable { model } => { + tracing::warn!( + model = %model, + reason = "no_decode_workers_available", + "service unavailable", + ); + "no decode workers available for the requested model".to_string() + } + ApiError::StaleRequestExpired { model } => { + tracing::warn!( + model = %model, + reason = "stale_request_expired", + "stale-request janitor expired in-flight request", + ); + "request expired before completion".to_string() + } + ApiError::PolicySelectionFailed { model } => { + tracing::warn!(model = %model, reason = "policy_selection_failed", "service unavailable"); + "service unavailable".to_string() + } + ApiError::BreakerOpen { worker } => { + tracing::warn!(upstream = %worker, reason = "breaker_open", "service unavailable"); + "service unavailable".to_string() + } + ApiError::WorkerMisconfigured { worker, source } => { + tracing::error!( + upstream = %worker, + error = %format_args!("{source:#}"), + "worker URL emitted by discovery is malformed", + ); + "service unavailable".to_string() + } + ApiError::BadRequest(_) | ApiError::ModelNotFound(_) => self.to_string(), + }; + let mut resp = ( + status, + Json(ErrorEnvelope { + error: ErrorBody { typ, code, message }, + }), + ) + .into_response(); + resp.headers_mut() + .insert(X_ROUTER_ERROR_CODE, HeaderValue::from_static(code)); + resp + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::response::IntoResponse; + use http_body_util::BodyExt; + use serde::Deserialize; + + fn collect_body(resp: Response) -> String { + let bytes = tokio::runtime::Runtime::new() + .unwrap() + .block_on(async { BodyExt::collect(resp.into_body()).await.unwrap().to_bytes() }); + String::from_utf8_lossy(&bytes).into_owned() + } + + /// Pin the exact JSON envelope shape that clients see. Renaming any of + /// these fields (or removing one) breaks every downstream consumer + /// silently, so we deserialize into a fixed struct rather than + /// regex-matching the rendered JSON. + #[derive(Deserialize)] + struct ErrEnv { + error: ErrField, + } + + #[derive(Deserialize)] + struct ErrField { + #[serde(rename = "type")] + typ: String, + code: String, + message: String, + } + + fn parse_envelope(resp: Response) -> (StatusCode, Option, ErrEnv) { + let status = resp.status(); + let code_header = resp + .headers() + .get("x-router-error-code") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + let body_str = collect_body(resp); + let env: ErrEnv = serde_json::from_str(&body_str) + .unwrap_or_else(|e| panic!("envelope did not match expected shape: {e}: {body_str}")); + (status, code_header, env) + } + + #[test] + fn upstream_unreachable_envelope_has_code_and_no_leak() { + let worker_str = "http://10.0.0.42:30000/"; + let worker = reqwest::Url::parse(worker_str).unwrap(); + let secret = "TLS_HANDSHAKE_FAILED at /etc/secret_ca.pem"; + let err = ApiError::UpstreamUnreachable { + worker: worker.clone(), + source: anyhow::anyhow!("{secret}"), + }; + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + resp.headers() + .get("x-router-error-code") + .and_then(|v| v.to_str().ok()), + Some("upstream_unreachable"), + ); + let body = collect_body(resp); + assert!(body.contains("\"code\":\"upstream_unreachable\""), "{body}"); + assert!(body.contains("\"type\":\"server_error\""), "{body}"); + assert!( + !body.contains(worker_str) && !body.contains(secret), + "client body must NOT leak worker URL or reqwest source chain; got: {body}", + ); + } + + #[test] + fn upstream_status_envelope_has_code() { + let err = ApiError::UpstreamStatus { + status: StatusCode::INTERNAL_SERVER_ERROR, + }; + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + resp.headers() + .get("x-router-error-code") + .and_then(|v| v.to_str().ok()), + Some("upstream_status"), + ); + let body = collect_body(resp); + assert!(body.contains("\"code\":\"upstream_status\""), "{body}"); + } + + #[test] + fn upstream_timeout_envelope_has_code_and_no_leak() { + let worker_str = "http://10.0.0.42:30000/"; + let worker = reqwest::Url::parse(worker_str).unwrap(); + let err = ApiError::UpstreamTimeout { + worker: worker.clone(), + }; + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + resp.headers() + .get("x-router-error-code") + .and_then(|v| v.to_str().ok()), + Some("upstream_timeout"), + ); + let body = collect_body(resp); + assert!(body.contains("\"code\":\"upstream_timeout\""), "{body}"); + assert!( + !body.contains(worker_str), + "client body must NOT leak worker URL; got: {body}", + ); + } + + #[test] + fn bad_request_envelope_has_expected_shape() { + let msg = "invalid_request: body must be an object"; + let err = ApiError::BadRequest(msg.into()); + let resp = err.into_response(); + let (status, code_header, env) = parse_envelope(resp); + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(code_header.as_deref(), Some("bad_request")); + assert_eq!(env.error.code, "bad_request"); + assert_eq!(env.error.typ, "invalid_request_error"); + assert!( + !env.error.message.is_empty(), + "message must not be empty: {:?}", + env.error.message, + ); + assert_ne!(env.error.code, "internal_error"); + assert_ne!(env.error.code, "model_not_found"); + } + + #[test] + fn model_not_found_envelope_has_expected_shape() { + let err = ApiError::ModelNotFound("ghost-7b".into()); + let resp = err.into_response(); + let (status, code_header, env) = parse_envelope(resp); + + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(code_header.as_deref(), Some("model_not_found")); + assert_eq!(env.error.code, "model_not_found"); + assert_eq!(env.error.typ, "invalid_request_error"); + assert!( + !env.error.message.is_empty(), + "message must not be empty: {:?}", + env.error.message, + ); + assert_ne!(env.error.code, "internal_error"); + assert_ne!(env.error.code, "bad_request"); + } + + #[test] + fn internal_error_response_sanitizes_anyhow_chain() { + let secret_msg = "internal /opt/secret/credential.json missing"; + let err = ApiError::Internal(anyhow::anyhow!("{secret_msg}")); + let resp = err.into_response(); + let body_str = collect_body(resp); + // Generic to client: + assert!( + body_str.contains("\"code\":\"internal_error\""), + "body: {body_str}" + ); + assert!( + body_str.contains("\"type\":\"server_error\""), + "body: {body_str}" + ); + // No leak of the original anyhow message: + assert!( + !body_str.contains(secret_msg), + "ApiError::Internal must not leak anyhow chain to client; got: {body_str}" + ); + } +} diff --git a/experimental/sgl-router/src/server/header_utils.rs b/experimental/sgl-router/src/server/header_utils.rs new file mode 100644 index 000000000000..4865a1c48252 --- /dev/null +++ b/experimental/sgl-router/src/server/header_utils.rs @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Header forwarding whitelist — mirrors SMG semantics. + +use axum::http::HeaderName; + +/// True if a request header from the inbound client should be forwarded +/// to the upstream worker. Mirrors SMG's whitelist semantics. +pub fn should_forward_request_header(name: &HeaderName) -> bool { + let n = name.as_str(); + matches!( + n, + "authorization" | "x-request-id" | "x-correlation-id" | "traceparent" | "tracestate" + ) || n.starts_with("x-request-id-") + || n.starts_with("x-sgl-") +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderName; + + #[test] + fn whitelist_basics() { + // Whitelisted headers + assert!(should_forward_request_header(&HeaderName::from_static( + "authorization" + ))); + assert!(should_forward_request_header(&HeaderName::from_static( + "x-request-id" + ))); + assert!(should_forward_request_header(&HeaderName::from_static( + "x-correlation-id" + ))); + assert!(should_forward_request_header(&HeaderName::from_static( + "traceparent" + ))); + assert!(should_forward_request_header(&HeaderName::from_static( + "tracestate" + ))); + assert!(should_forward_request_header(&HeaderName::from_static( + "x-sgl-route-key" + ))); + assert!(should_forward_request_header(&HeaderName::from_static( + "x-request-id-extra" + ))); + + // Stripped headers + assert!(!should_forward_request_header(&HeaderName::from_static( + "host" + ))); + assert!(!should_forward_request_header(&HeaderName::from_static( + "content-length" + ))); + assert!(!should_forward_request_header(&HeaderName::from_static( + "cookie" + ))); + assert!(!should_forward_request_header(&HeaderName::from_static( + "connection" + ))); + assert!(!should_forward_request_header(&HeaderName::from_static( + "transfer-encoding" + ))); + } + + /// Prefix-match negatives: names that LOOK similar to `x-request-id-*` + /// or `x-sgl-*` but must NOT be forwarded. Guards against a future + /// regression that loosens the rule (e.g., a `contains` instead of + /// `starts_with`, or a missing hyphen anchor). + #[test] + fn whitelist_prefix_negatives() { + // `x-request-id` itself is an exact match and MUST forward — + // pin this so a future "tighten prefix to require trailing hyphen" + // refactor doesn't silently drop the canonical name. + assert!( + should_forward_request_header(&HeaderName::from_static("x-request-id")), + "x-request-id (exact match) must forward", + ); + + // No trailing hyphen between `id` and the suffix: not a child of + // `x-request-id-*`, must NOT forward. + assert!( + !should_forward_request_header(&HeaderName::from_static("x-request-id2")), + "x-request-id2 (no hyphen separator) must not forward", + ); + assert!( + !should_forward_request_header(&HeaderName::from_static("x-request-idfoo")), + "x-request-idfoo (no hyphen separator) must not forward", + ); + + // Typo of the `x-sgl-` prefix (missing 'l'): must NOT forward. + assert!( + !should_forward_request_header(&HeaderName::from_static("x-sg-foo")), + "x-sg-foo (typo of x-sgl-) must not forward", + ); + + // Extra leading character: `xx-request-id-foo` does not start with + // `x-request-id-`, must NOT forward. + assert!( + !should_forward_request_header(&HeaderName::from_static("xx-request-id-foo")), + "xx-request-id-foo (extra leading char) must not forward", + ); + // Same shape for the x-sgl- family. + assert!( + !should_forward_request_header(&HeaderName::from_static("xx-sgl-foo")), + "xx-sgl-foo (extra leading char) must not forward", + ); + + // Substring-but-not-prefix: must NOT forward (guards against a + // `contains`-based regression). + assert!( + !should_forward_request_header(&HeaderName::from_static("foo-x-sgl-bar")), + "foo-x-sgl-bar (substring, not prefix) must not forward", + ); + } +} diff --git a/experimental/sgl-router/src/server/metrics.rs b/experimental/sgl-router/src/server/metrics.rs new file mode 100644 index 000000000000..678b10d1ec59 --- /dev/null +++ b/experimental/sgl-router/src/server/metrics.rs @@ -0,0 +1,551 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Lightweight in-process Prometheus exposition. +//! +//! We deliberately do NOT pull in the `metrics` + `metrics-exporter-prometheus` +//! crates: the observability surface is small enough that a hand-written +//! counter + histogram + gauge family is cheaper than a new dependency, and +//! it lets us label/serialise exactly the way the convergence and PD-affinity +//! tests want. +//! +//! All operations are concurrent — counters and gauges use +//! [`std::sync::atomic`], histograms use a [`Mutex>`] over a +//! fixed bucket set. Tests sub-second; production scrapes are 15s +//! cadence. Lock contention is not a concern at these rates. +//! +//! # Metrics surface +//! +//! | Metric | Type | Labels | +//! |---|---|---| +//! | `sgl_router_requests_total` | Counter | `worker_url`, `model_id`, `mode`, `outcome` | +//! | `sgl_router_overlap_blocks` | Histogram | `model_id` | +//! | `sgl_router_active_load` | Gauge | `worker_url`, `kind` | +//! | `sgl_router_stale_requests_total` | Counter | `outcome` | +//! | `sgl_router_decode_affinity_total` | Counter | `outcome` | +//! +//! The exposition is text/plain; version=0.0.4 per the Prometheus spec. + +use parking_lot::Mutex; +use std::collections::HashMap; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; +use std::sync::Arc; + +/// Histogram bucket upper bounds for `sgl_router_overlap_blocks`. Chosen to +/// span 0 → ~1k blocks: blocks are 32–64 tokens each, and our `MAX_CHAT_BODY_BYTES` +/// cap (1 MiB ≈ 250 k tokens) implies an upper bound around 4–8 k blocks for +/// a maximum-length context. The `+Inf` bucket catches everything beyond +/// 1000. +const OVERLAP_BLOCKS_BUCKETS: &[f64] = &[ + 0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1000.0, +]; + +/// Recordable outcome for a request — narrowed to a handful of variants so +/// the label cardinality stays bounded. +#[derive(Debug, Clone, Copy)] +pub enum RequestOutcome { + Success, + Error, + Cancelled, +} + +impl RequestOutcome { + fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Error => "error", + Self::Cancelled => "cancelled", + } + } +} + +/// Worker dispatch mode label — narrowed to the three modes the policy +/// resolver distinguishes. The `Plain` variant covers the non-PD case. +#[derive(Debug, Clone, Copy)] +pub enum WorkerModeLabel { + Prefill, + Decode, + Plain, +} + +impl WorkerModeLabel { + fn as_str(self) -> &'static str { + match self { + Self::Prefill => "prefill", + Self::Decode => "decode", + Self::Plain => "plain", + } + } +} + +/// Decode-affinity outcome — see `select_decode_with_affinity` for the +/// three reasons the affinity may not be honored. +#[derive(Debug, Clone, Copy)] +pub enum DecodeAffinityOutcome { + SameHostPicked, + FallbackBreaker, + FallbackLoadImbalance, +} + +impl DecodeAffinityOutcome { + fn as_str(self) -> &'static str { + match self { + Self::SameHostPicked => "same_host_picked", + Self::FallbackBreaker => "fallback_breaker", + Self::FallbackLoadImbalance => "fallback_load_imbalance", + } + } +} + +/// Stale-request outcome label. +#[derive(Debug, Clone, Copy)] +pub enum StaleRequestOutcome { + Expired, +} + +impl StaleRequestOutcome { + fn as_str(self) -> &'static str { + match self { + Self::Expired => "expired", + } + } +} + +/// Active-load kind label — separates the two axes of per-worker load. +#[derive(Debug, Clone, Copy)] +pub enum ActiveLoadKind { + PrefillTokens, + DecodeBlocks, +} + +impl ActiveLoadKind { + fn as_str(self) -> &'static str { + match self { + Self::PrefillTokens => "prefill_tokens", + Self::DecodeBlocks => "decode_blocks", + } + } +} + +/// The shared metrics registry, held on `AppContext`. Cheap to clone — all +/// internal state is `Arc`/`Atomic`/`Mutex`-protected. +#[derive(Debug, Default)] +pub struct MetricsRegistry { + requests_total: Mutex>>, + overlap_blocks: Mutex>, + active_load: Mutex>>, + stale_requests_total: Mutex>>, + decode_affinity_total: Mutex>>, +} + +#[derive(Debug, Hash, Eq, PartialEq, Clone)] +struct RequestKey { + worker_url: String, + model_id: String, + mode: &'static str, + outcome: &'static str, +} + +#[derive(Debug, Hash, Eq, PartialEq, Clone)] +struct ActiveLoadKey { + worker_url: String, + kind: &'static str, +} + +#[derive(Debug)] +struct Histogram { + /// One counter per bucket boundary in [`OVERLAP_BLOCKS_BUCKETS`], plus + /// one for `+Inf`. Buckets are cumulative on render but stored as + /// non-cumulative counts here. + buckets: Vec, + sum: f64, + count: u64, +} + +impl Histogram { + fn new() -> Self { + Self { + buckets: vec![0; OVERLAP_BLOCKS_BUCKETS.len() + 1], + sum: 0.0, + count: 0, + } + } + + fn observe(&mut self, value: f64) { + let mut placed = false; + for (i, &bound) in OVERLAP_BLOCKS_BUCKETS.iter().enumerate() { + if value <= bound { + self.buckets[i] += 1; + placed = true; + break; + } + } + if !placed { + // +Inf bucket + let last = self.buckets.len() - 1; + self.buckets[last] += 1; + } + self.sum += value; + self.count += 1; + } +} + +impl MetricsRegistry { + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// Bump `sgl_router_requests_total` for the given worker / model / mode / outcome. + pub fn record_request( + &self, + worker_url: &str, + model_id: &str, + mode: WorkerModeLabel, + outcome: RequestOutcome, + ) { + let key = RequestKey { + worker_url: worker_url.to_owned(), + model_id: model_id.to_owned(), + mode: mode.as_str(), + outcome: outcome.as_str(), + }; + let mut guard = self.requests_total.lock(); + let counter = guard + .entry(key) + .or_insert_with(|| Arc::new(AtomicU64::new(0))) + .clone(); + drop(guard); + counter.fetch_add(1, Ordering::Relaxed); + } + + /// Observe an overlap-blocks count for `sgl_router_overlap_blocks`. + pub fn observe_overlap_blocks(&self, model_id: &str, blocks: u64) { + let mut guard = self.overlap_blocks.lock(); + let hist = guard + .entry(model_id.to_owned()) + .or_insert_with(Histogram::new); + hist.observe(blocks as f64); + } + + /// Set `sgl_router_active_load` for the given worker + kind. Replaces the + /// previous value (gauge semantics). + pub fn set_active_load(&self, worker_url: &str, kind: ActiveLoadKind, value: i64) { + let key = ActiveLoadKey { + worker_url: worker_url.to_owned(), + kind: kind.as_str(), + }; + let mut guard = self.active_load.lock(); + let gauge = guard + .entry(key) + .or_insert_with(|| Arc::new(AtomicI64::new(0))) + .clone(); + drop(guard); + gauge.store(value, Ordering::Relaxed); + } + + /// Bump `sgl_router_stale_requests_total{outcome}`. + pub fn record_stale_request(&self, outcome: StaleRequestOutcome) { + let mut guard = self.stale_requests_total.lock(); + let counter = guard + .entry(outcome.as_str()) + .or_insert_with(|| Arc::new(AtomicU64::new(0))) + .clone(); + drop(guard); + counter.fetch_add(1, Ordering::Relaxed); + } + + /// Bump `sgl_router_decode_affinity_total{outcome}`. + pub fn record_decode_affinity(&self, outcome: DecodeAffinityOutcome) { + let mut guard = self.decode_affinity_total.lock(); + let counter = guard + .entry(outcome.as_str()) + .or_insert_with(|| Arc::new(AtomicU64::new(0))) + .clone(); + drop(guard); + counter.fetch_add(1, Ordering::Relaxed); + } + + /// Render the registry as a Prometheus 0.0.4 exposition-format string. + pub fn render(&self) -> String { + let mut out = String::new(); + + // requests_total + out.push_str( + "# HELP sgl_router_requests_total Total chat-completions requests dispatched to a worker.\n", + ); + out.push_str("# TYPE sgl_router_requests_total counter\n"); + let guard = self.requests_total.lock(); + // Sort for stable output — easier for tests. + let mut entries: Vec<(&RequestKey, u64)> = guard + .iter() + .map(|(k, v)| (k, v.load(Ordering::Relaxed))) + .collect(); + entries.sort_by(|a, b| { + (&a.0.worker_url, &a.0.model_id, a.0.mode, a.0.outcome).cmp(&( + &b.0.worker_url, + &b.0.model_id, + b.0.mode, + b.0.outcome, + )) + }); + for (key, value) in entries { + out.push_str(&format!( + "sgl_router_requests_total{{worker_url=\"{}\",model_id=\"{}\",mode=\"{}\",outcome=\"{}\"}} {}\n", + escape_label(&key.worker_url), + escape_label(&key.model_id), + key.mode, + key.outcome, + value, + )); + } + drop(guard); + + // overlap_blocks histogram + out.push_str( + "# HELP sgl_router_overlap_blocks Overlap-block count observed at cache-aware-zmq policy selection.\n", + ); + out.push_str("# TYPE sgl_router_overlap_blocks histogram\n"); + let guard = self.overlap_blocks.lock(); + let mut models: Vec<&String> = guard.keys().collect(); + models.sort(); + for model_id in models { + let hist = guard.get(model_id).unwrap(); + let mut cumulative: u64 = 0; + for (i, &bound) in OVERLAP_BLOCKS_BUCKETS.iter().enumerate() { + cumulative += hist.buckets[i]; + out.push_str(&format!( + "sgl_router_overlap_blocks_bucket{{model_id=\"{}\",le=\"{}\"}} {}\n", + escape_label(model_id), + bound, + cumulative, + )); + } + cumulative += hist.buckets[OVERLAP_BLOCKS_BUCKETS.len()]; + out.push_str(&format!( + "sgl_router_overlap_blocks_bucket{{model_id=\"{}\",le=\"+Inf\"}} {}\n", + escape_label(model_id), + cumulative, + )); + out.push_str(&format!( + "sgl_router_overlap_blocks_sum{{model_id=\"{}\"}} {}\n", + escape_label(model_id), + hist.sum, + )); + out.push_str(&format!( + "sgl_router_overlap_blocks_count{{model_id=\"{}\"}} {}\n", + escape_label(model_id), + hist.count, + )); + } + drop(guard); + + // active_load gauge + out.push_str( + "# HELP sgl_router_active_load Per-worker active load (prefill_tokens or decode_blocks).\n", + ); + out.push_str("# TYPE sgl_router_active_load gauge\n"); + let guard = self.active_load.lock(); + let mut entries: Vec<(&ActiveLoadKey, i64)> = guard + .iter() + .map(|(k, v)| (k, v.load(Ordering::Relaxed))) + .collect(); + entries.sort_by(|a, b| (&a.0.worker_url, a.0.kind).cmp(&(&b.0.worker_url, b.0.kind))); + for (key, value) in entries { + out.push_str(&format!( + "sgl_router_active_load{{worker_url=\"{}\",kind=\"{}\"}} {}\n", + escape_label(&key.worker_url), + key.kind, + value, + )); + } + drop(guard); + + // stale_requests_total + out.push_str( + "# HELP sgl_router_stale_requests_total Total stale-request cancellations fired by the janitor.\n", + ); + out.push_str("# TYPE sgl_router_stale_requests_total counter\n"); + let guard = self.stale_requests_total.lock(); + let mut entries: Vec<(&&str, u64)> = guard + .iter() + .map(|(k, v)| (k, v.load(Ordering::Relaxed))) + .collect(); + entries.sort_by_key(|e| *e.0); + for (outcome, value) in entries { + out.push_str(&format!( + "sgl_router_stale_requests_total{{outcome=\"{}\"}} {}\n", + outcome, value, + )); + } + drop(guard); + + // decode_affinity_total + out.push_str( + "# HELP sgl_router_decode_affinity_total Decode-affinity outcomes from select_decode_with_affinity.\n", + ); + out.push_str("# TYPE sgl_router_decode_affinity_total counter\n"); + let guard = self.decode_affinity_total.lock(); + let mut entries: Vec<(&&str, u64)> = guard + .iter() + .map(|(k, v)| (k, v.load(Ordering::Relaxed))) + .collect(); + entries.sort_by_key(|e| *e.0); + for (outcome, value) in entries { + out.push_str(&format!( + "sgl_router_decode_affinity_total{{outcome=\"{}\"}} {}\n", + outcome, value, + )); + } + drop(guard); + + out + } +} + +/// Prometheus label-value escape rule per +/// https://prometheus.io/docs/instrumenting/exposition_formats/. +/// We only escape `\`, `"`, and newline — the three characters the +/// reference parser rejects unescaped. +fn escape_label(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '\\' => out.push_str(r"\\"), + '"' => out.push_str(r#"\""#), + '\n' => out.push_str(r"\n"), + other => out.push(other), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_registry_renders_only_help_lines() { + let reg = MetricsRegistry::new(); + let out = reg.render(); + // Should at least carry HELP / TYPE for every metric family. + assert!(out.contains("# TYPE sgl_router_requests_total counter")); + assert!(out.contains("# TYPE sgl_router_overlap_blocks histogram")); + assert!(out.contains("# TYPE sgl_router_active_load gauge")); + assert!(out.contains("# TYPE sgl_router_stale_requests_total counter")); + assert!(out.contains("# TYPE sgl_router_decode_affinity_total counter")); + } + + #[test] + fn record_request_emits_labelled_counter_line() { + let reg = MetricsRegistry::new(); + reg.record_request( + "http://worker-a:30000", + "tiny", + WorkerModeLabel::Prefill, + RequestOutcome::Success, + ); + reg.record_request( + "http://worker-a:30000", + "tiny", + WorkerModeLabel::Prefill, + RequestOutcome::Success, + ); + let out = reg.render(); + assert!( + out.contains(r#"sgl_router_requests_total{worker_url="http://worker-a:30000",model_id="tiny",mode="prefill",outcome="success"} 2"#), + "render did not include the expected counter line; got:\n{out}", + ); + } + + #[test] + fn observe_overlap_blocks_writes_buckets_and_count() { + let reg = MetricsRegistry::new(); + reg.observe_overlap_blocks("tiny", 3); + reg.observe_overlap_blocks("tiny", 9); + reg.observe_overlap_blocks("tiny", 50); + let out = reg.render(); + // 3 observations -> count=3, sum=62 + assert!(out.contains(r#"sgl_router_overlap_blocks_count{model_id="tiny"} 3"#)); + assert!(out.contains(r#"sgl_router_overlap_blocks_sum{model_id="tiny"} 62"#)); + // The le=64 bucket is cumulative: 3 is <=4, 9 is <=16, 50 is <=64. + assert!( + out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="tiny",le="64"} 3"#), + "bucket le=64 should be 3 (cumulative); got:\n{out}", + ); + // The le=4 bucket should include only the 3. + assert!( + out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="tiny",le="4"} 1"#), + "bucket le=4 should be 1; got:\n{out}", + ); + } + + #[test] + fn set_active_load_gauge_overwrites() { + let reg = MetricsRegistry::new(); + reg.set_active_load("http://w:30000", ActiveLoadKind::PrefillTokens, 100); + reg.set_active_load("http://w:30000", ActiveLoadKind::PrefillTokens, 250); + let out = reg.render(); + assert!(out.contains( + r#"sgl_router_active_load{worker_url="http://w:30000",kind="prefill_tokens"} 250"#, + )); + // First write must NOT appear. + assert!(!out.contains( + r#"sgl_router_active_load{worker_url="http://w:30000",kind="prefill_tokens"} 100"#, + )); + } + + #[test] + fn stale_request_counter_increments() { + let reg = MetricsRegistry::new(); + reg.record_stale_request(StaleRequestOutcome::Expired); + reg.record_stale_request(StaleRequestOutcome::Expired); + reg.record_stale_request(StaleRequestOutcome::Expired); + let out = reg.render(); + assert!(out.contains(r#"sgl_router_stale_requests_total{outcome="expired"} 3"#)); + } + + #[test] + fn decode_affinity_counter_emits_three_outcomes() { + let reg = MetricsRegistry::new(); + reg.record_decode_affinity(DecodeAffinityOutcome::SameHostPicked); + reg.record_decode_affinity(DecodeAffinityOutcome::SameHostPicked); + reg.record_decode_affinity(DecodeAffinityOutcome::FallbackBreaker); + reg.record_decode_affinity(DecodeAffinityOutcome::FallbackLoadImbalance); + let out = reg.render(); + assert!(out.contains(r#"sgl_router_decode_affinity_total{outcome="same_host_picked"} 2"#)); + assert!(out.contains(r#"sgl_router_decode_affinity_total{outcome="fallback_breaker"} 1"#)); + assert!(out + .contains(r#"sgl_router_decode_affinity_total{outcome="fallback_load_imbalance"} 1"#,)); + } + + #[test] + fn label_values_escape_quotes_and_backslashes() { + let reg = MetricsRegistry::new(); + reg.record_request( + r#"http://"weird":30000"#, + r"back\slash", + WorkerModeLabel::Plain, + RequestOutcome::Error, + ); + let out = reg.render(); + assert!( + out.contains(r#"worker_url="http://\"weird\":30000""#), + "render did not escape double-quote; got:\n{out}", + ); + assert!( + out.contains(r#"model_id="back\\slash""#), + "render did not escape backslash; got:\n{out}", + ); + } + + #[test] + fn histogram_plus_inf_bucket_catches_overflow() { + let reg = MetricsRegistry::new(); + // 1001 is just above the last finite bucket (1000); it should land + // in +Inf only. + reg.observe_overlap_blocks("m", 1001); + let out = reg.render(); + assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="1000"} 0"#)); + assert!(out.contains(r#"sgl_router_overlap_blocks_bucket{model_id="m",le="+Inf"} 1"#)); + } +} diff --git a/experimental/sgl-router/src/server/mod.rs b/experimental/sgl-router/src/server/mod.rs new file mode 100644 index 000000000000..800500718c3b --- /dev/null +++ b/experimental/sgl-router/src/server/mod.rs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +pub mod app; +pub mod app_context; +pub mod error; +pub mod header_utils; +pub mod metrics; +pub mod routes; diff --git a/experimental/sgl-router/src/server/routes/chat.rs b/experimental/sgl-router/src/server/routes/chat.rs new file mode 100644 index 000000000000..868f84850802 --- /dev/null +++ b/experimental/sgl-router/src/server/routes/chat.rs @@ -0,0 +1,678 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::discovery::{ModelId, WorkerMode}; +use crate::policies::registry::{PdPoolResolver, PdResolveError}; +use crate::policies::SelectionContext; +use crate::server::app_context::AppContext; +use crate::server::error::ApiError; +use crate::server::metrics::{RequestOutcome, StaleRequestOutcome, WorkerModeLabel}; +use crate::workers::{LoadGuard, Worker}; +use axum::body::Body; +use axum::extract::State; +use axum::http::{HeaderMap, HeaderName, HeaderValue, Response}; +use bytes::Bytes; +use serde::de::IgnoredAny; +use serde::Deserialize; +use std::collections::HashMap; +use std::sync::Arc; + +/// Observability header carrying the decode-pool URL selected via host +/// affinity for a PD-disaggregated request. The router fans the +/// bootstrap-injected request body to BOTH the prefill and the decode +/// worker concurrently; this header lets the prefill log the chosen +/// peer, and is mirrored onto the response so sidecars / tests can +/// observe affinity without sniffing the proxy hop. The `x-sgl-` +/// prefix matches `x-sgl-router-error-code` so router-emitted metadata +/// stays grouped. +const X_SGL_DECODE_URL: HeaderName = HeaderName::from_static("x-sgl-decode-url"); + +/// Coarse char-count → token-count divisor used to estimate prefill load +/// from the request body when no real tokenizer count is available. Four +/// bytes per token is the standard SGLang upstream estimate; it +/// overcounts ASCII and undercounts CJK but stays within an order of +/// magnitude of the real token count, which is plenty for load +/// scoring. The active-load counters' role is relative ordering across +/// workers — not absolute accuracy — so the estimate is fit for +/// purpose. +const CHARS_PER_TOKEN_ESTIMATE: usize = 4; + +/// Per-route body-size cap on `/v1/chat/completions`. 1 MiB is comfortable +/// for normal chat traffic (a 200 k-token context tokenized as JSON is well +/// under this) while preventing a hostile client from forcing the router to +/// heap-allocate hundreds of MiB before forwarding. The cap is wired in +/// `crate::server::app::build_router` as a route-level `DefaultBodyLimit` +/// layer; axum's `Bytes` extractor enforces it and returns 413 +/// PAYLOAD_TOO_LARGE before this handler runs. +pub const MAX_CHAT_BODY_BYTES: usize = 1 << 20; + +/// Minimal probe over the request body — we only need the `stream` field +/// and the `model` field to decide between buffered vs SSE forwarding and +/// to select a worker. Deserializing into this struct (vs `serde_json::Value`) +/// does two things: +/// +/// 1. Avoids the per-field heap allocation of `Value` for a 1 MiB body. +/// 2. Pins the contract: the body MUST be a JSON object. Degenerate +/// shapes (`null`, `[]`, `"hi"`) fail at this step rather than being +/// silently forwarded with `stream=false`. +/// +/// All other fields are ignored — the worker is authoritative for the +/// full request schema. +#[derive(Debug, Deserialize)] +struct RequestProbe { + #[serde(default)] + stream: Option, + #[serde(default)] + model: Option, +} + +/// POST /v1/chat/completions — parse model from body, select a healthy +/// worker via the per-model policy, then proxy the request. If the +/// request opts into streaming (`stream: true`), we pipe SSE bytes back; +/// otherwise buffer. +pub async fn chat_completions( + State(ctx): State>, + headers: HeaderMap, + body: Bytes, +) -> Result, ApiError> { + let probe = parse_probe(&body)?; + let streaming = probe.stream.unwrap_or(false); + let model_str = probe + .model + .ok_or_else(|| ApiError::BadRequest("missing `model` field".into()))?; + let model_id = ModelId(model_str.clone()); + + // PD pool isolation: for PD-mode deployments, prefill traffic + // selects from the prefill pool only. Plain-mode deployments fall + // through to the full candidate set. Partial-failure errors + // (`no_prefill_workers_available`) are surfaced as 503 with a + // distinct error code so operators can alert independently. + let resolver = PdPoolResolver::new(Arc::clone(&ctx.registry)); + let workers = resolver + .prefill_candidates(&model_id) + .map_err(|e| match e { + PdResolveError::NoHealthyWorkers => ApiError::NoHealthyWorkers { + model: model_str.clone(), + }, + PdResolveError::NoPrefillWorkersAvailable => ApiError::NoPrefillWorkersAvailable { + model: model_str.clone(), + }, + PdResolveError::NoDecodeWorkersAvailable => ApiError::NoDecodeWorkersAvailable { + model: model_str.clone(), + }, + })?; + + let policy = ctx + .policies + .get(&model_id) + .ok_or_else(|| ApiError::ModelNotFound(model_str.clone()))?; + let selection_ctx = SelectionContext::new(&model_id, Some(&body)); + let worker = + policy + .select(&workers, &selection_ctx) + .ok_or_else(|| ApiError::PolicySelectionFailed { + model: model_str.clone(), + })?; + + // PD-mode decoder affinity. When the selected prefill worker is + // part of a PD-disagg deployment, also resolve the matching decode + // peer (same host where possible, falling back to min-load via + // `select_decode_with_affinity`). Both workers receive the SAME + // request body — augmented with the three flat `bootstrap_*` + // fields below — so the SGLang engine can match incoming KV + // transfers via `bootstrap_room`. + // + // Plain-mode workers skip the decode resolution entirely (no + // decode peer to find). PD-mode requests that fail to resolve a + // decode peer (`NoDecodeWorkersAvailable`) bubble up as 503 so + // operators can alert on prefill-vs-decode pool imbalance. + let decode_peer: Option> = if worker.mode() == WorkerMode::Prefill { + Some( + resolver + .decode_with_affinity(&model_id, &worker.url) + .map_err(|e| match e { + PdResolveError::NoHealthyWorkers => ApiError::NoHealthyWorkers { + model: model_str.clone(), + }, + PdResolveError::NoDecodeWorkersAvailable => { + ApiError::NoDecodeWorkersAvailable { + model: model_str.clone(), + } + } + PdResolveError::NoPrefillWorkersAvailable => { + ApiError::NoPrefillWorkersAvailable { + model: model_str.clone(), + } + } + })?, + ) + } else { + None + }; + let decode_hint_url: Option = decode_peer.as_ref().map(|d| d.url.clone()); + let mut request_headers = headers; + if let Some(url) = &decode_hint_url { + match HeaderValue::from_str(url) { + Ok(v) => { + request_headers.insert(X_SGL_DECODE_URL, v); + } + Err(e) => { + // Discovery emits URLs the proxy has already used; a + // header-value parse failure here means the URL + // contains a control character (e.g. CR / LF) — drop + // the header but keep the request: bootstrap injection + // below carries the host/port the engine actually + // needs; the header is purely observability. + tracing::warn!( + decode_url = %url, + error = %e, + "decode worker URL rejected by header parser; sending request without decode hint", + ); + } + } + } + let headers = request_headers; + + // Per-worker `active_requests` guard. The `ActiveLoadGuard` below + // sits beside this one: both track in-flight load, but the + // ActiveLoadGuard entry is per-request (with timeout-based janitor) + // while the worker-scoped counter is what the cache-aware policy + // reads. Both must drop at the same time — when the response stream + // ends, the client disconnects, or the handler returns an error. In + // PD mode the pair moves into the spawned prefill task so prefill + // load is tracked for the full duration of the KV transfer; in plain + // mode the pair stays in this handler. Decode-load contribution is + // 0 here: the active-load registry's decode axis is reserved for a + // future decode-side scheduler — current decode selection is + // host-affinity only. + let guard = worker.load_guard(); + let prefill_load = estimate_prefill_tokens(&body); + let active_guard = + ctx.active_load + .register(worker.id.clone(), worker.url.clone(), prefill_load, 0); + // Snapshot the stale-request cancel token BEFORE moving the guard + // into the spawned prefill task / streaming pump / response future. + // The token is cheap to clone (it's an `Arc<...>` internally) and + // the chat handler races the client-facing fetch against + // `token.cancelled()` to surface a 504 `stale_request_expired` if + // the janitor expires the request mid-flight. + let stale_token = active_guard.cancel_token().clone(); + + // Snapshot the labels we need for metrics BEFORE moving the worker + // / model_str values into the per-branch fetch futures. + let metrics_worker_url = worker.url.clone(); + let metrics_mode = match worker.mode() { + WorkerMode::Prefill => WorkerModeLabel::Prefill, + WorkerMode::Decode => WorkerModeLabel::Decode, + WorkerMode::Plain => WorkerModeLabel::Plain, + }; + let metrics_model = model_str.clone(); + + let result = if let Some(decode_worker) = decode_peer { + // PD-disagg dispatch (Pattern B — spawn prefill, await decode). + // + // SGLang's HTTP-mode disagg-prefill requires three flat + // top-level fields on the request body: `bootstrap_host`, + // `bootstrap_port` (the prefill worker's bootstrap-server + // address) and `bootstrap_room` (a per-request 63-bit u64 ID + // used by both sides to pair up the KV transfer). We inject + // these here and fan the same modified body to both the + // prefill and decode workers concurrently. + // + // **Why spawn-and-forget for prefill instead of + // `tokio::join!`?** All three peer SGLang-HTTP-PD routers + // (Dynamo / llm-d / aibrix) converged on this shape: the + // prefill request must outlive the client connection because + // tying prefill to the client future opens a cancel-race + // window where the engine's NIXL RPC teardown can leak KV + // block refs (NVBugs 5969206 in Dynamo). The detached task + // also keeps the LoadGuard + ActiveLoadGuard alive for the full + // prefill duration — KV transfer can run for tens of seconds + // even when the client gave up. + // + // No watchdog for fail-fast on prefill 5xx: llm-d / aibrix both + // ship without one. On prefill failure the client experiences + // the SGLang decode-side bootstrap_room timeout (~30–60 s by + // default) instead of an immediate 502. A follow-up can wire a + // `tokio::sync::watch` channel if telemetry shows it matters. + // + // **Scope of the "detached" guarantee.** The spawn protects + // against client disconnect — the handler future being dropped + // does NOT cancel the prefill HTTP request. It does NOT protect + // against router shutdown: when `AppContext` tears down, the + // tokio runtime cancels all unfinished tasks including this + // one. A future follow-up could thread a `TaskTracker` / + // `JoinSet` through `AppContext` for graceful shutdown drain; + // the current implementation ships without one (matching SMG's + // shutdown behaviour). + let bootstrap_room = generate_room_id(); + let injected_body = inject_bootstrap_fields( + &body, + worker.bootstrap_host(), + worker.bootstrap_port(), + bootstrap_room, + )?; + + let prefill_url = worker.url.clone(); + let prefill_breaker = Arc::clone(&worker.breaker); + let prefill_headers = headers.clone(); + let prefill_body = injected_body.clone(); + let prefill_proxy = Arc::clone(&ctx.proxy); + let prefill_holds: (LoadGuard, _) = (guard, active_guard); + tokio::spawn(async move { + // The tuple binding extends both guards' lifetime to the + // end of this async block, which lasts until the prefill + // HTTP request returns (success / error / engine-side + // bootstrap_room timeout). The result is logged and + // swallowed — no channel back to the client. See the big + // comment above for the rationale. + let _hold = prefill_holds; + match prefill_proxy + .forward_json_to( + &prefill_url, + &prefill_breaker, + "/v1/chat/completions", + &prefill_headers, + prefill_body, + ) + .await + { + Ok(_) => tracing::debug!( + prefill_url = %prefill_url, + bootstrap_room, + "prefill side completed", + ), + Err(e) => tracing::warn!( + prefill_url = %prefill_url, + bootstrap_room, + error = %e, + "prefill request failed; decode will time out on bootstrap_room", + ), + } + }); + + // Synchronously await the decode worker. Its response is what + // the client sees. The decode side gets its own LoadGuard so + // per-worker `active_requests` reflects decode-pool load for + // cache-aware-zmq decisions on the decode side. + let decode_guard = decode_worker.load_guard(); + if streaming { + let stream_guards: Box = Box::new(decode_guard); + let fetch = ctx.proxy.forward_streaming_to( + &decode_worker.url, + &decode_worker.breaker, + "/v1/chat/completions", + &headers, + injected_body, + Some(stream_guards), + ); + tokio::select! { + biased; + r = fetch => r, + _ = stale_token.cancelled() => Err(ApiError::StaleRequestExpired { model: model_str }), + } + } else { + let _decode_hold = decode_guard; + let fetch = ctx.proxy.forward_json_to( + &decode_worker.url, + &decode_worker.breaker, + "/v1/chat/completions", + &headers, + injected_body, + ); + tokio::select! { + biased; + r = fetch => r, + _ = stale_token.cancelled() => Err(ApiError::StaleRequestExpired { model: model_str }), + } + } + } else if streaming { + // Plain mode, streaming. Both guards ride the SSE pump until + // the body completes — see the matching comment in the + // non-streaming arm. + let stream_guards: Box = Box::new((guard, active_guard)); + let fetch = ctx.proxy.forward_streaming_to( + &worker.url, + &worker.breaker, + "/v1/chat/completions", + &headers, + body, + Some(stream_guards), + ); + // Bias `fetch` over the cancellation branch: a successful + // response that completes in the same poll as the token firing + // MUST win (returning 504 for a request that already has + // headers is a correctness regression). The cancellation + // branch only matters when fetch is still pending — at that + // point biasing the order is a wash. + tokio::select! { + biased; + r = fetch => r, + _ = stale_token.cancelled() => Err(ApiError::StaleRequestExpired { model: model_str }), + } + } else { + // Plain mode, non-streaming. The handler awaits the full + // buffered response, so both guards live correctly in this + // scope. The tuple binding exists only to extend the guards' + // lifetime to the end of the function — the `forward_json_to` + // future does not need them (it does not return until the + // body is buffered). + let _holds: (LoadGuard, _) = (guard, active_guard); + let fetch = ctx.proxy.forward_json_to( + &worker.url, + &worker.breaker, + "/v1/chat/completions", + &headers, + body, + ); + // Same `biased` order as the streaming arm. + tokio::select! { + biased; + r = fetch => r, + _ = stale_token.cancelled() => Err(ApiError::StaleRequestExpired { model: model_str }), + } + }; + + // Record the dispatch outcome AFTER we know whether the upstream + // accepted the request. A 504 from the stale-request branch counts as + // `cancelled` — semantically distinct from upstream errors that bubble + // through as `error`. The metric is per-worker so convergence tests + // can scrape `/metrics` and assert that ≥N requests landed on a + // single prefill worker. + let outcome = match &result { + Ok(_) => RequestOutcome::Success, + Err(ApiError::StaleRequestExpired { .. }) => { + // The janitor fired the stale-cancel and we observed it + // user-side; record both the per-request `cancelled` outcome + // AND the global `expired` count. The two views are useful for + // different alerts: per-worker request_total{cancelled} flags a + // worker that's hanging, while stale_requests_total{expired} + // tracks the global health of the janitor. + ctx.metrics + .record_stale_request(StaleRequestOutcome::Expired); + RequestOutcome::Cancelled + } + Err(_) => RequestOutcome::Error, + }; + ctx.metrics + .record_request(&metrics_worker_url, &metrics_model, metrics_mode, outcome); + + // Mirror the upstream `x-sgl-decode-url` hint onto the response so + // external tests / sidecars can observe PD decode affinity without + // sniffing the proxy hop. The request-side header was set above for + // the prefill worker; copying it here makes the affinity observable + // end-to-end. Plain-mode requests skip this (no decode peer was + // resolved). A malformed URL was already rejected at the + // request-side parse — we only reach this branch when the URL was + // header-valid, so the second parse is safe. + match (result, decode_hint_url) { + (Ok(mut response), Some(url)) => { + match HeaderValue::from_str(&url) { + Ok(v) => { + response.headers_mut().insert(X_SGL_DECODE_URL, v); + } + Err(e) => { + // Already-validated upstream; defensive log only. + tracing::warn!( + decode_url = %url, + error = %e, + "decode worker URL rejected by header parser on response; omitting response-side hint", + ); + } + } + Ok(response) + } + (other, _) => other, + } +} + +/// Estimate prefill-token count from the raw request body for use as +/// the active-load `prefill_load` counter. Returns 1 at minimum so +/// a registered request always shows up as "load > 0" — under-counting +/// to zero would hide the request from the cache-aware policy's +/// load-imbalance fast-path. +/// +/// This is a coarse approximation: we count the body length in bytes +/// and divide by [`CHARS_PER_TOKEN_ESTIMATE`]. A future improvement is +/// to thread the tokenizer's actual token count through (the +/// cache-aware-zmq policy already tokenizes the prompt for tree +/// matching — that count could be reused here). +fn estimate_prefill_tokens(body: &Bytes) -> usize { + (body.len() / CHARS_PER_TOKEN_ESTIMATE).max(1) +} + +/// Mint a fresh `bootstrap_room` for a PD-disagg request. +/// +/// SGLang's disagg-prefill stores the room as a signed `i64` internally +/// (see `python/sglang/srt/disaggregation/utils.py` — `bootstrap_room` +/// metadata buffer is allocated as `torch.int64`). Generating in +/// `[0, i64::MAX]` keeps the value safely positive when reinterpreted +/// signed. Mirrors SMG's `pd_types::generate_room_id`, Dynamo's +/// `rand::random_range(0..=i64::MAX.cast_unsigned())`, and SGLang's +/// own Python-side `random.randint(0, 2**63 - 1)`. +fn generate_room_id() -> u64 { + rand::random::() & (i64::MAX as u64) +} + +/// Inject the three flat top-level fields SGLang's HTTP disagg-prefill +/// validator requires: +/// +/// * `bootstrap_host` — the prefill worker's hostname; decode connects +/// to this address for the KV transfer. +/// * `bootstrap_port` — the prefill worker's bootstrap server port +/// (may be `null` if the worker is misconfigured; the engine will +/// reject the request with a clear error). +/// * `bootstrap_room` — a 63-bit random `u64` identifying this request +/// on both prefill and decode sides. +/// +/// The body must already be a JSON object (the chat handler's +/// `parse_probe` guarantees this); we re-parse into a `Map` here to +/// mutate top-level keys without walking nested values into a full +/// `serde_json::Value`. A malformed body is mapped to +/// `ApiError::BadRequest` — the parse_probe layer should already have +/// caught this, but defending against TOCTOU keeps the error path +/// honest. +fn inject_bootstrap_fields( + body: &Bytes, + bootstrap_host: &str, + bootstrap_port: Option, + bootstrap_room: u64, +) -> Result { + let mut obj: serde_json::Map = serde_json::from_slice(body) + .map_err(|e| { + tracing::debug!(error = %e, "re-parse for bootstrap injection failed"); + ApiError::BadRequest("invalid request: body must be a JSON object".to_string()) + })?; + obj.insert( + "bootstrap_host".to_string(), + serde_json::Value::String(bootstrap_host.to_string()), + ); + obj.insert( + "bootstrap_port".to_string(), + match bootstrap_port { + Some(p) => serde_json::Value::Number(p.into()), + None => serde_json::Value::Null, + }, + ); + obj.insert( + "bootstrap_room".to_string(), + serde_json::Value::Number(bootstrap_room.into()), + ); + let bytes = serde_json::to_vec(&obj).map_err(|e| { + ApiError::Internal(anyhow::Error::new(e).context("re-serialize bootstrap-injected body")) + })?; + Ok(Bytes::from(bytes)) +} + +fn parse_probe(body: &Bytes) -> Result { + // We deliberately do NOT echo the serde error into the client-visible + // message — that risks leaking field-level detail and is also of little + // help to a real client (which already has its own JSON validator). + // Server-side, the full error is logged with `tracing::debug!` for + // operator triage. + // + // Two-step deserialize: + // 1. `Map` *anchors* the shape to a JSON object. + // This rejects `null` / `[]` / `"hi"` (all valid JSON but not + // request shape) without walking the full value into a + // `serde_json::Value` per field. + // 2. `RequestProbe` (struct of `Option` + `Option`) + // lifts out only the fields we care about — `stream` and `model`. + // Other fields are ignored; the worker is authoritative for the + // rest of the schema. + let _: HashMap = serde_json::from_slice(body).map_err(|e| { + tracing::debug!(error = %e, "chat-completions body rejected as non-object JSON"); + ApiError::BadRequest("invalid request: body must be a JSON object".to_string()) + })?; + let probe: RequestProbe = serde_json::from_slice(body).map_err(|e| { + tracing::debug!(error = %e, "chat-completions request-probe deserialize failed"); + ApiError::BadRequest("invalid request: body must be a JSON object".to_string()) + })?; + Ok(probe) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `generate_room_id` MUST return values in `[0, i64::MAX]`. The + /// SGLang prefill stores `bootstrap_room` as `torch.int64`; a u64 + /// with the top bit set would wrap negative on the engine side. + /// Sample many times to defend against future refactors of the + /// mask (e.g. someone "simplifying" to plain `rand::random::()`). + #[test] + fn generate_room_id_stays_in_63_bit_range() { + for _ in 0..10_000 { + let r = generate_room_id(); + assert!( + r <= i64::MAX as u64, + "generate_room_id() returned {r} > i64::MAX; would wrap negative as torch.int64", + ); + } + } + + /// When the prefill worker has no `bootstrap_port` configured + /// (a misconfiguration the engine will reject loudly), the + /// injected field MUST be JSON `null` — not omitted, not 0. + /// SGLang's validator distinguishes "missing field" from + /// "null field" in some code paths. + #[test] + fn inject_bootstrap_fields_emits_null_for_missing_port() { + let body = Bytes::from_static(br#"{"model":"x","messages":[]}"#); + let injected = inject_bootstrap_fields(&body, "host", None, 42).unwrap(); + let parsed: serde_json::Value = serde_json::from_slice(&injected).unwrap(); + assert_eq!(parsed.get("bootstrap_port"), Some(&serde_json::Value::Null)); + assert_eq!( + parsed.get("bootstrap_host"), + Some(&serde_json::Value::String("host".into())) + ); + assert_eq!( + parsed.get("bootstrap_room"), + Some(&serde_json::Value::Number(42.into())) + ); + } + + #[test] + fn parse_probe_reads_stream_bool_from_object() { + let b = Bytes::from_static(br#"{"stream": true, "model": "tiny"}"#); + assert_eq!(parse_probe(&b).unwrap().stream, Some(true)); + let b = Bytes::from_static(br#"{"stream": false, "model": "tiny"}"#); + assert_eq!(parse_probe(&b).unwrap().stream, Some(false)); + } + + #[test] + fn parse_probe_defaults_when_stream_absent() { + // Existing happy-path contract: well-formed object missing `stream` + // must default to None (caller picks false). The minimal `RequestProbe` + // (Option + #[serde(default)]) must NOT break this. + let b = Bytes::from_static(br#"{"model": "tiny", "messages": []}"#); + let p = parse_probe(&b).unwrap(); + assert_eq!(p.stream, None); + assert_eq!(p.model.as_deref(), Some("tiny")); + } + + #[test] + fn parse_probe_rejects_non_object_shapes() { + // Pin the contract: degenerate JSON (valid JSON but wrong shape) + // must be rejected, not silently forwarded with `stream=false`. + for bad in [&b"null"[..], &b"[]"[..], &b"\"hi\""[..], &b"42"[..]] { + let b = Bytes::copy_from_slice(bad); + let err = parse_probe(&b).unwrap_err(); + match err { + ApiError::BadRequest(_) => {} + other => panic!("expected BadRequest for {bad:?}, got {other:?}"), + } + } + } + + #[test] + fn parse_probe_rejects_malformed_json() { + let b = Bytes::from_static(b"{not json}"); + let err = parse_probe(&b).unwrap_err(); + assert!(matches!(err, ApiError::BadRequest(_))); + } + + #[test] + fn parse_probe_handles_nested_messages_with_stream_true() { + // Well-formed object with nested arrays/objects (real chat-completions + // payloads carry `messages: [{role, content: [{type, text}]}]`). The + // two-step deserialize must not balk on this — only the top-level + // object shape and the `stream`/`model` fields matter. + let b = Bytes::from_static( + br#"{ + "model": "x", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + "stream": true + }"#, + ); + assert_eq!(parse_probe(&b).unwrap().stream, Some(true)); + } + + #[test] + fn parse_probe_handles_nested_messages_with_stream_false() { + let b = Bytes::from_static( + br#"{ + "model": "x", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + "stream": false + }"#, + ); + assert_eq!(parse_probe(&b).unwrap().stream, Some(false)); + } + + #[test] + fn parse_probe_handles_duplicate_stream_keys() { + // RFC 8259 says "names within an object SHOULD be unique" but a + // parser MAY accept duplicates. Step 1 (HashMap) silently + // last-wins, but step 2 deserializes into the typed `RequestProbe` + // struct, and `serde_json`'s `#[derive(Deserialize)]` REJECTS + // duplicate fields with a `duplicate field` error. + // + // We map that to `BadRequest` (same path as other malformed input). + // Pinning "reject" rather than "last-wins" is intentional — + // ambiguous bodies should fail loudly at the edge, not silently + // route based on which copy serde happened to see last. + let b = Bytes::from_static(br#"{"stream": true, "stream": false}"#); + let err = parse_probe(&b).unwrap_err(); + match err { + ApiError::BadRequest(_) => {} + other => panic!("expected BadRequest on duplicate `stream` key, got {other:?}"), + } + } + + #[test] + fn parse_probe_bad_request_message_does_not_leak_serde_detail() { + // Info-leak guard: the client-visible message must be a fixed + // string, not the serde error (which can contain line/column + // detail or hint at field shape). + let b = Bytes::from_static(br#"{"stream": "not-a-bool"}"#); + let err = parse_probe(&b).unwrap_err(); + match err { + ApiError::BadRequest(msg) => assert_eq!( + msg, "invalid request: body must be a JSON object", + "client-visible message must be fixed; got: {msg}" + ), + other => panic!("expected BadRequest, got {other:?}"), + } + } +} diff --git a/experimental/sgl-router/src/server/routes/health.rs b/experimental/sgl-router/src/server/routes/health.rs new file mode 100644 index 000000000000..2bcd6af8b28e --- /dev/null +++ b/experimental/sgl-router/src/server/routes/health.rs @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::server::app_context::AppContext; +use axum::extract::State; +use axum::http::StatusCode; +use std::sync::Arc; + +/// Always returns 200 — liveness probe. +pub async fn healthz() -> StatusCode { + StatusCode::OK +} + +/// Readiness probe — 200 only when the pod can actually serve traffic. +/// +/// Requires BOTH: +/// 1. `AppContext::mark_ready()` was called by main (process bootstrap +/// finished — config loaded, tokenizers built, server bound), AND +/// 2. At least one worker is registered. Without this second check, +/// `/readyz` flips green before the first `DiscoveryEvent::Added` +/// has been processed — the Service starts sending traffic to a +/// pod whose registry is empty, and every request returns 503 +/// `no_healthy_workers`. +pub async fn readyz(State(ctx): State>) -> StatusCode { + if ctx.is_ready() && !ctx.registry.is_empty() { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use tower::ServiceExt; + + #[tokio::test] + async fn healthz_always_200() { + let app = crate::server::app::build_router(test_ctx(false, false)); + let res = app + .oneshot( + Request::builder() + .uri("/healthz") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + } + + #[tokio::test] + async fn readyz_503_when_not_ready() { + let app = crate::server::app::build_router(test_ctx(false, true)); + let res = app + .oneshot( + Request::builder() + .uri("/readyz") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::SERVICE_UNAVAILABLE); + } + + #[tokio::test] + async fn readyz_503_when_ready_but_registry_empty() { + // Regression: `/readyz` previously returned 200 the moment + // `mark_ready()` was called, even with an empty worker + // registry. The Service would route traffic to a pod that + // could only return 503 no_healthy_workers. + let app = crate::server::app::build_router(test_ctx(true, false)); + let res = app + .oneshot( + Request::builder() + .uri("/readyz") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + res.status(), + StatusCode::SERVICE_UNAVAILABLE, + "ready=true + empty registry must still be 503" + ); + } + + #[tokio::test] + async fn readyz_200_when_ready_and_worker_registered() { + let app = crate::server::app::build_router(test_ctx(true, true)); + let res = app + .oneshot( + Request::builder() + .uri("/readyz") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + } + + fn test_ctx(ready: bool, with_worker: bool) -> Arc { + use crate::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; + let ctx = AppContext::stub(); + if ready { + ctx.mark_ready(); + } + if with_worker { + ctx.registry + .add(WorkerSpec { + id: WorkerId("test-w".into()), + url: "http://test:30000".into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("test".into())], + bootstrap_port: None, + }) + .expect("test worker accepted"); + } + Arc::new(ctx) + } +} diff --git a/experimental/sgl-router/src/server/routes/metrics.rs b/experimental/sgl-router/src/server/routes/metrics.rs new file mode 100644 index 000000000000..3d45897bfa5c --- /dev/null +++ b/experimental/sgl-router/src/server/routes/metrics.rs @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! `/metrics` endpoint — Prometheus 0.0.4 exposition. +//! +//! Returns the live snapshot of [`crate::server::metrics::MetricsRegistry`]. +//! Plain-text body; charset is utf-8. We deliberately don't gate this on +//! readiness — scrapers should be able to read the metrics surface even +//! while the router is warming up so the "router started but no workers +//! discovered" failure mode is observable. + +use crate::server::app_context::AppContext; +use axum::extract::State; +use axum::http::header::CONTENT_TYPE; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use std::sync::Arc; + +/// Content-Type per Prometheus exposition format spec. +const PROMETHEUS_CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; + +pub async fn metrics(State(ctx): State>) -> impl IntoResponse { + let body = ctx.metrics.render(); + ( + StatusCode::OK, + [(CONTENT_TYPE, PROMETHEUS_CONTENT_TYPE)], + body, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::server::metrics::{RequestOutcome, WorkerModeLabel}; + use axum::body::Body; + use axum::http::Request; + use http_body_util::BodyExt; + use tower::ServiceExt; + + #[tokio::test] + async fn metrics_endpoint_returns_prometheus_text() { + let ctx = Arc::new(AppContext::stub()); + let app = crate::server::app::build_router(ctx.clone()); + let res = app + .oneshot( + Request::builder() + .uri("/metrics") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let content_type = res + .headers() + .get(CONTENT_TYPE) + .expect("content-type header") + .to_str() + .unwrap() + .to_owned(); + assert!( + content_type.starts_with("text/plain"), + "expected text/plain, got {content_type}", + ); + let body = res.into_body().collect().await.unwrap().to_bytes(); + let body = std::str::from_utf8(&body).unwrap(); + // Every metric family should at least carry its HELP/TYPE lines. + assert!(body.contains("# TYPE sgl_router_requests_total counter")); + assert!(body.contains("# TYPE sgl_router_overlap_blocks histogram")); + assert!(body.contains("# TYPE sgl_router_active_load gauge")); + } + + #[tokio::test] + async fn metrics_endpoint_reflects_recorded_counters() { + let ctx = Arc::new(AppContext::stub()); + ctx.metrics.record_request( + "http://w-test:30000", + "tiny", + WorkerModeLabel::Prefill, + RequestOutcome::Success, + ); + let app = crate::server::app::build_router(ctx.clone()); + let res = app + .oneshot( + Request::builder() + .uri("/metrics") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = res.into_body().collect().await.unwrap().to_bytes(); + let body = std::str::from_utf8(&body).unwrap(); + assert!( + body.contains(r#"worker_url="http://w-test:30000""#), + "metrics did not include the recorded worker_url; got:\n{body}", + ); + } +} diff --git a/experimental/sgl-router/src/server/routes/mod.rs b/experimental/sgl-router/src/server/routes/mod.rs new file mode 100644 index 000000000000..c5d1845f4a13 --- /dev/null +++ b/experimental/sgl-router/src/server/routes/mod.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +pub mod chat; +pub mod health; +pub mod metrics; +pub mod models; +pub mod tokenize; diff --git a/experimental/sgl-router/src/server/routes/models.rs b/experimental/sgl-router/src/server/routes/models.rs new file mode 100644 index 000000000000..eed5ab19f789 --- /dev/null +++ b/experimental/sgl-router/src/server/routes/models.rs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::server::app_context::AppContext; +use axum::extract::State; +use axum::Json; +use serde::Serialize; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct ModelsList { + pub object: &'static str, + pub data: Vec, +} + +#[derive(Serialize)] +pub struct ModelEntry { + pub id: String, + pub object: &'static str, + pub owned_by: &'static str, +} + +pub async fn list_models(State(ctx): State>) -> Json { + let data = ctx + .config + .models + .iter() + .map(|m| ModelEntry { + id: m.id.clone(), + object: "model", + owned_by: "sglang", + }) + .collect(); + Json(ModelsList { + object: "list", + data, + }) +} + +#[cfg(test)] +mod tests { + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use http_body_util::BodyExt; + use tower::ServiceExt; + + use crate::config::PolicyKind; + + #[tokio::test] + async fn lists_configured_models() { + let mut ctx = crate::server::app_context::AppContext::stub(); + ctx.config.models = vec![ + crate::config::ModelConfig { + id: "qwen3".into(), + tokenizer_path: "x".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }, + crate::config::ModelConfig { + id: "deepseek".into(), + tokenizer_path: "y".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }, + ]; + let app = crate::server::app::build_router(std::sync::Arc::new(ctx)); + let res = app + .oneshot( + Request::builder() + .uri("/v1/models") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(v["object"], "list"); + let ids: Vec<&str> = v["data"] + .as_array() + .unwrap() + .iter() + .map(|m| m["id"].as_str().unwrap()) + .collect(); + assert_eq!(ids, vec!["qwen3", "deepseek"]); + assert_eq!(v["data"][0]["object"], "model"); + // Pin `owned_by` so a refactor that flips the hardcoded value to + // "openai" / "" / a typo would fail loudly here. OpenAI clients + // expect this field and some (e.g. langchain-openai) treat + // `owned_by != "system"` as a meaningful signal. + assert_eq!(v["data"][0]["owned_by"], "sglang"); + assert_eq!(v["data"][1]["owned_by"], "sglang"); + } +} diff --git a/experimental/sgl-router/src/server/routes/tokenize.rs b/experimental/sgl-router/src/server/routes/tokenize.rs new file mode 100644 index 000000000000..932f276d78b1 --- /dev/null +++ b/experimental/sgl-router/src/server/routes/tokenize.rs @@ -0,0 +1,362 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::server::app_context::AppContext; +use crate::server::error::ApiError; +use crate::tokenizer::adapter; +use axum::extract::State; +use axum::Json; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TokenizeRequest { + pub model: String, + pub prompt: String, +} + +#[derive(Serialize)] +#[cfg_attr(test, derive(Deserialize))] +pub struct TokenizeResponse { + pub model: String, + pub tokens: Vec, + pub count: usize, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DetokenizeRequest { + pub model: String, + pub tokens: Vec, + #[serde(default)] + pub skip_special_tokens: bool, +} + +#[derive(Serialize)] +#[cfg_attr(test, derive(Deserialize))] +pub struct DetokenizeResponse { + pub model: String, + pub text: String, +} + +pub async fn tokenize( + State(ctx): State>, + Json(req): Json, +) -> Result, ApiError> { + let tok = ctx + .tokenizers + .get(&req.model) + .ok_or_else(|| ApiError::ModelNotFound(req.model.clone()))?; + // Structured log on failure so an operator can correlate + // "every encode for model X errors" against the route, model id, and + // prompt size. The generic anyhow-chain log in ApiError::Internal still + // fires from IntoResponse — duplication is intentional: the route-level + // line carries `model` / `prompt_len`, the IntoResponse line carries + // the full anyhow chain. + let ids = adapter::encode(&tok, &req.prompt).map_err(|e| { + tracing::error!( + route = "/v1/tokenize", + model = %req.model, + prompt_len = req.prompt.len(), + error = ?e, + "tokenize.encode failed", + ); + ApiError::Internal(e) + })?; + Ok(Json(TokenizeResponse { + model: req.model, + count: ids.len(), + tokens: ids, + })) +} + +pub async fn detokenize( + State(ctx): State>, + Json(req): Json, +) -> Result, ApiError> { + let tok = ctx + .tokenizers + .get(&req.model) + .ok_or_else(|| ApiError::ModelNotFound(req.model.clone()))?; + let text = + adapter::decode_complete(&tok, &req.tokens, req.skip_special_tokens).map_err(|e| { + tracing::error!( + route = "/v1/detokenize", + model = %req.model, + n_tokens = req.tokens.len(), + skip_special = req.skip_special_tokens, + error = ?e, + "detokenize.decode_complete failed", + ); + ApiError::Internal(e) + })?; + Ok(Json(DetokenizeResponse { + model: req.model, + text, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use http_body_util::BodyExt; + use tower::ServiceExt; + + use crate::config::PolicyKind; + + fn ctx_with_tiny() -> Arc { + let cfg = crate::config::Config { + server: crate::config::ServerConfig { + host: "x".into(), + port: 0, + }, + observability: Default::default(), + models: vec![crate::config::ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: crate::config::DiscoveryConfig { + backend: crate::config::DiscoveryBackend::StaticUrls( + crate::config::StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }, + ), + }, + proxy: crate::config::ProxyConfig::default(), + active_load: crate::config::ActiveLoadConfig::default(), + }; + let registry = crate::tokenizer::TokenizerRegistry::load_from_config(&cfg).unwrap(); + let proxy = Arc::new( + crate::proxy::Proxy::new(std::time::Duration::from_secs(60)).expect("stub proxy"), + ); + let worker_registry = Arc::new(crate::workers::WorkerRegistry::default()); + let policies = Arc::new(crate::policies::PolicyRegistry::default()); + Arc::new(AppContext::new( + cfg, + Arc::new(registry), + proxy, + worker_registry, + policies, + )) + } + + #[tokio::test] + async fn tokenize_round_trip() { + let app = crate::server::app::build_router(ctx_with_tiny()); + let body = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", "prompt": "hello world" + })) + .unwrap(); + let res = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/tokenize") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let r: TokenizeResponse = serde_json::from_slice(&bytes).unwrap(); + assert!(r.count > 0); + + let body2 = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", "tokens": r.tokens, "skip_special_tokens": true + })) + .unwrap(); + let res2 = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/detokenize") + .header("content-type", "application/json") + .body(Body::from(body2)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res2.status(), StatusCode::OK); + let bytes2 = res2.into_body().collect().await.unwrap().to_bytes(); + let r2: DetokenizeResponse = serde_json::from_slice(&bytes2).unwrap(); + assert_eq!(r2.text, "hello world"); + } + + #[tokio::test] + async fn tokenize_request_does_not_advertise_add_special_tokens() { + // Regression: TokenizeRequest must not have an `add_special_tokens` field. + // Background: dynamo-tokenizers cannot honor it. Silently ignoring it + // would be a footgun for clients that set it. + let req: TokenizeRequest = + serde_json::from_str(r#"{"model": "tiny", "prompt": "hi"}"#).unwrap(); + let _ = req; // compiles → schema is correct minus that field + + // If someone sets it anyway, serde should reject with deny_unknown_fields. + let parsed: Result = serde_json::from_str( + r#"{"model": "tiny", "prompt": "hi", "add_special_tokens": true}"#, + ); + assert!( + parsed.is_err(), + "add_special_tokens should be rejected as unknown field" + ); + } + + /// Ported from SMG tests/api/parser_endpoints_test.rs (parse_function_call_missing_fields): + /// DetokenizeRequest has `deny_unknown_fields`; an extra field must yield 422 + /// Unprocessable Entity from axum's JSON extractor, not 200 with the field silently ignored. + /// Gap: the existing `tokenize_request_does_not_advertise_add_special_tokens` test only + /// exercises serde deserialization directly; this test exercises the HTTP layer. + #[tokio::test] + async fn detokenize_rejects_unknown_field() { + let app = crate::server::app::build_router(ctx_with_tiny()); + let body = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "tokens": [15496, 995], + "skip_special_tokens": false, + "add_special_tokens": true // unknown field — must be rejected + })) + .unwrap(); + let res = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/detokenize") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + res.status(), + StatusCode::UNPROCESSABLE_ENTITY, + "DetokenizeRequest must reject unknown fields via deny_unknown_fields" + ); + } + + /// Gap: `tokenize_round_trip` tests only `skip_special_tokens: true`. + /// When omitted, `#[serde(default)]` gives `false` — a different decode code-path. + /// This covers the default (omitted) and explicit-false routes end-to-end via HTTP. + #[tokio::test] + async fn detokenize_skip_special_tokens_false_default() { + let app = crate::server::app::build_router(ctx_with_tiny()); + + // First tokenize to get IDs. + let tok_body = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", "prompt": "hello world" + })) + .unwrap(); + let tok_res = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/tokenize") + .header("content-type", "application/json") + .body(Body::from(tok_body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(tok_res.status(), StatusCode::OK); + let tok_bytes = tok_res.into_body().collect().await.unwrap().to_bytes(); + let r: TokenizeResponse = serde_json::from_slice(&tok_bytes).unwrap(); + + // Detokenize with skip_special_tokens omitted (defaults to false). + let det_body_omitted = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "tokens": r.tokens + // skip_special_tokens intentionally absent — must default to false + })) + .unwrap(); + let det_res_omitted = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/detokenize") + .header("content-type", "application/json") + .body(Body::from(det_body_omitted)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(det_res_omitted.status(), StatusCode::OK); + let det_bytes_omitted = det_res_omitted + .into_body() + .collect() + .await + .unwrap() + .to_bytes(); + let d_omitted: DetokenizeResponse = serde_json::from_slice(&det_bytes_omitted).unwrap(); + assert_eq!( + d_omitted.text, "hello world", + "detokenize with skip_special_tokens omitted (default false) must round-trip" + ); + + // Also test explicit false — must be identical to omitted. + let det_body_explicit = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "tokens": r.tokens, + "skip_special_tokens": false + })) + .unwrap(); + let det_res_explicit = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/detokenize") + .header("content-type", "application/json") + .body(Body::from(det_body_explicit)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(det_res_explicit.status(), StatusCode::OK); + let det_bytes_explicit = det_res_explicit + .into_body() + .collect() + .await + .unwrap() + .to_bytes(); + let d_explicit: DetokenizeResponse = serde_json::from_slice(&det_bytes_explicit).unwrap(); + assert_eq!( + d_explicit.text, d_omitted.text, + "explicit skip_special_tokens=false must produce same result as omitted" + ); + } + + #[tokio::test] + async fn unknown_model_404() { + let app = crate::server::app::build_router(ctx_with_tiny()); + let body = serde_json::to_vec(&serde_json::json!({ + "model": "nope", "prompt": "x" + })) + .unwrap(); + let res = app + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/tokenize") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::NOT_FOUND); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "model_not_found" + ); + } +} diff --git a/experimental/sgl-router/src/tokenizer/adapter.rs b/experimental/sgl-router/src/tokenizer/adapter.rs new file mode 100644 index 000000000000..02fe72b6305e --- /dev/null +++ b/experimental/sgl-router/src/tokenizer/adapter.rs @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{Context, Result}; +use dynamo_tokenizers::{traits::DecodeResult, Tokenizer}; +use std::sync::Arc; + +pub fn load(path: &str) -> Result> { + Tokenizer::from_file(path) + .map(Arc::new) + .with_context(|| format!("load tokenizer from {path}")) +} + +pub fn encode(t: &Tokenizer, text: &str) -> Result> { + let enc = t.encode(text).context("encode")?; + Ok(enc.token_ids().to_vec()) +} + +/// Decode token ids to a complete UTF-8 string. +/// +/// Non-streaming callers (e.g. `/v1/detokenize`) get the full result either way: +/// - `DecodeResult::Complete(s)` — the token sequence ends on a codepoint boundary. +/// - `DecodeResult::Partial(s)` — the token sequence ends mid-codepoint; `s` ends +/// in U+FFFD. We return `s` as-is so the client sees the closest-possible string. +/// +/// Streaming callers should NOT use this; they should consume `DecodeResult` +/// directly and withhold the trailing U+FFFD until the next decode produces a +/// `Complete` result. +pub fn decode_complete(t: &Tokenizer, ids: &[u32], skip_special: bool) -> Result { + let res = t.decode(ids, skip_special).context("decode")?; + Ok(match res { + DecodeResult::Complete(s) => s, + DecodeResult::Partial(s) => { + tracing::debug!( + n_tokens = ids.len(), + trailing_bytes = s.len(), + "decode_complete: tokenizer returned Partial for non-streaming call" + ); + s + } + }) +} diff --git a/experimental/sgl-router/src/tokenizer/mod.rs b/experimental/sgl-router/src/tokenizer/mod.rs new file mode 100644 index 000000000000..36d3a5cee506 --- /dev/null +++ b/experimental/sgl-router/src/tokenizer/mod.rs @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +pub mod adapter; + +use anyhow::Result; +use dashmap::DashMap; +use dynamo_tokenizers::Tokenizer; +use std::sync::Arc; + +#[derive(Default)] +pub struct TokenizerRegistry { + inner: DashMap>, +} + +impl std::fmt::Debug for TokenizerRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TokenizerRegistry") + .field("models", &self.ids()) + .finish() + } +} + +impl TokenizerRegistry { + pub fn load_from_config(cfg: &crate::config::Config) -> Result { + let me = TokenizerRegistry::default(); + for m in &cfg.models { + let t = adapter::load(&m.tokenizer_path)?; + me.inner.insert(m.id.clone(), t); + } + Ok(me) + } + + pub fn get(&self, model_id: &str) -> Option> { + self.inner.get(model_id).map(|r| Arc::clone(&*r)) + } + + pub fn ids(&self) -> Vec { + self.inner.iter().map(|kv| kv.key().clone()).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use crate::config::PolicyKind; + + fn cfg() -> crate::config::Config { + crate::config::Config { + server: crate::config::ServerConfig { + host: "0".into(), + port: 0, + }, + observability: Default::default(), + models: vec![crate::config::ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: crate::config::DiscoveryConfig { + backend: crate::config::DiscoveryBackend::StaticUrls( + crate::config::StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }, + ), + }, + proxy: crate::config::ProxyConfig::default(), + active_load: crate::config::ActiveLoadConfig::default(), + } + } + + #[test] + fn loads_from_config() { + let r = TokenizerRegistry::load_from_config(&cfg()).unwrap(); + assert!(r.get("tiny").is_some()); + assert!(r.get("missing").is_none()); + } + + #[test] + fn shared_arc_per_model() { + let r = TokenizerRegistry::load_from_config(&cfg()).unwrap(); + let a = r.get("tiny").unwrap(); + let b = r.get("tiny").unwrap(); + assert!( + Arc::ptr_eq(&a, &b), + "registry should return shared Arc, not clones" + ); + } + + #[test] + fn decode_complete_preserves_round_trip() { + let r = TokenizerRegistry::load_from_config(&cfg()).unwrap(); + let t = r.get("tiny").unwrap(); + let ids = adapter::encode(&t, "hello world").unwrap(); + assert!(!ids.is_empty()); + let text = adapter::decode_complete(&t, &ids, true).unwrap(); + // tiny BPE fixture is byte-level and lossless for ASCII. + assert_eq!(text, "hello world"); + } + + /// Forces `decode_complete` through its `DecodeResult::Partial` branch. + /// + /// Strategy A: the fixture is a GPT-2 byte-level BPE. The 4-byte UTF-8 + /// emoji `😀` (`\xF0\x9F\x98\x80`) encodes into 2 byte-level BPE tokens + /// with this fixture: `[47249, 222]`. Decoding just the first token + /// yields a leading-bytes-only prefix that the HF adapter passes through + /// `String::from_utf8_lossy`, producing a trailing U+FFFD. dynamo's + /// `DecodeResult::from_decoded` then classifies that as `Partial`. + /// Pinning the literal token id keeps the test deterministic — if the + /// fixture or upstream BPE merges ever shift, this fails loudly rather + /// than silently dropping back into `Complete` and losing coverage. + #[test] + fn decode_complete_returns_string_on_partial_utf8() { + let r = TokenizerRegistry::load_from_config(&cfg()).unwrap(); + let t = r.get("tiny").unwrap(); + + // Sanity-check that the fixture still tokenises `😀` the way we + // expect; if upstream changes this we want a loud failure here. + let full = adapter::encode(&t, "😀").unwrap(); + assert_eq!( + full, + vec![47249, 222], + "fixture tokenisation drift: '😀' no longer encodes to [47249, 222]" + ); + + // Feed only the first token — its bytes are the leading 3 of a + // 4-byte UTF-8 codepoint, which is incomplete. + let s = adapter::decode_complete(&t, &full[..1], false).unwrap(); + + // We pin the exact output: the lossy decoder folds the 3 leading + // bytes into a single U+FFFD. Anything else (empty string, Err, or + // the original bytes) would be a regression. + assert_eq!(s, "\u{FFFD}"); + } + + /// Concurrent encode against one shared `Arc`. Pins that the + /// registry's `Arc` is `Send + Sync` and that + /// `dynamo_tokenizers::Tokenizer::encode` can be called concurrently + /// without interior mutability hazards. A regression that wraps + /// `Tokenizer` in `RefCell` / `!Sync` data would fail to compile; + /// a regression that introduces non-thread-safe internal caches + /// would surface as one of the tasks returning wrong ids (caught by + /// the per-task assertion against the sequentially-computed + /// reference). + /// + /// Uses a multi-thread runtime + `JoinSet` so the 10 tasks really do + /// run in parallel on distinct worker threads — a single-thread + /// runtime wouldn't exercise the `Sync` contract. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn tokenizer_supports_concurrent_encode() { + use tokio::task::JoinSet; + + let r = TokenizerRegistry::load_from_config(&cfg()).unwrap(); + let t = r.get("tiny").unwrap(); + + // Build the reference sequentially — what each task should return. + let inputs: Vec = (0..10).map(|i| format!("hello {i}")).collect(); + let expected: Vec> = inputs + .iter() + .map(|s| adapter::encode(&t, s).unwrap()) + .collect(); + + let mut set = JoinSet::new(); + for (i, text) in inputs.into_iter().enumerate() { + let shared = Arc::clone(&t); + set.spawn(async move { + let ids = adapter::encode(&shared, &text).expect("concurrent encode must not fail"); + (i, ids) + }); + } + + let mut got: Vec>> = vec![None; expected.len()]; + while let Some(joined) = set.join_next().await { + let (i, ids) = joined.expect("task panicked"); + got[i] = Some(ids); + } + + for (i, ids) in got.into_iter().enumerate() { + let ids = ids.unwrap_or_else(|| panic!("task {i} did not record a result")); + assert_eq!( + ids, expected[i], + "concurrent encode produced wrong tokens for task {i}; \ + sign of a non-thread-safe internal cache regression" + ); + } + } + + #[test] + fn missing_file_errors() { + let mut c = cfg(); + c.models[0].tokenizer_path = "/nonexistent.json".into(); + let err = TokenizerRegistry::load_from_config(&c).unwrap_err(); + assert!(err.to_string().to_lowercase().contains("tokenizer")); + } +} diff --git a/experimental/sgl-router/src/workers/introspect.rs b/experimental/sgl-router/src/workers/introspect.rs new file mode 100644 index 000000000000..fa829674b887 --- /dev/null +++ b/experimental/sgl-router/src/workers/introspect.rs @@ -0,0 +1,556 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Single-shot `/server_info` introspection for newly-discovered workers. +//! +//! Combines what used to be two separate round-trips (the worker +//! manager's `served_model_name` fetch and `KvEventIndex::add_worker`'s +//! `fetch_event_config`) into one HTTP request. The result is dispatched +//! by the manager: registry consumes `served_model_name`, the optional +//! `KvEventIndex` consumes the resolved `EventConfig`. +//! +//! # Failure semantics +//! +//! `fetch` is **infallible** — any error (network, non-2xx, JSON parse, +//! invalid worker URL) is logged at `warn!` and returns an empty +//! `ServerInfo` so the caller can register the worker with empty +//! `model_ids` and no kv-events attachment. Workers that need accuracy +//! around publisher availability use `kv_events::discovery::fetch_event_config` +//! directly (it returns `Result>`); the manager +//! intentionally doesn't. + +use std::time::Duration; + +use serde::Deserialize; +use tracing::warn; +use url::Url; + +use crate::policies::kv_events::EventConfig; + +/// Default timeout for `/server_info`. Conservative for a small JSON +/// payload served by SGLang's HTTP server. +const SERVER_INFO_TIMEOUT: Duration = Duration::from_secs(2); + +/// Retry budget for transient `/server_info` failures (connect/timeout/5xx). +/// 4xx + JSON-parse errors short-circuit — they're authoritative. +/// EndpointSlice can flip ready=true before the worker's HTTP server is +/// actually serving; without retry, that race lands a worker in the +/// registry with empty model_ids and chat dispatch fails with 502. +const FETCH_MAX_ATTEMPTS: u32 = 3; +const FETCH_BACKOFF_BASE: Duration = Duration::from_millis(100); + +/// Resolved per-worker bootstrap state. +/// +/// `served_model_name` populates the registry; `event_config` is handed +/// to `KvEventIndex::add_worker` (skipping its own fetch); +/// `disaggregation_role` lets the worker manager override the discovery +/// backend's PD classification (and fill in `WorkerSpec.bootstrap_port` +/// for prefill workers) — see `manager::register_one`. +#[derive(Debug, Clone, Default)] +pub struct ServerInfo { + pub served_model_name: Option, + pub event_config: Option, + pub disaggregation_role: Option, +} + +/// PD classification derived from a worker's `/server_info` response. +/// +/// `Some(_)` means the worker self-disclosed its role and we should trust +/// it over the discovery backend's classification. `None` (the +/// `ServerInfo::disaggregation_role` value, not a variant here) means the +/// worker didn't tell us — older SGLang, missing field, or a partial +/// response — and the backend's classification wins. See the resolution +/// table in `resolve_disaggregation_role`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DisaggregationRole { + Plain, + Prefill { bootstrap_port: u16 }, + Decode, +} + +/// Performs the single `/server_info` round-trip and projects the +/// response into both halves of `ServerInfo`. Cheap to clone — wraps a +/// `reqwest::Client` (which is internally `Arc`-backed). +#[derive(Clone)] +pub struct WorkerIntrospector { + client: reqwest::Client, +} + +impl WorkerIntrospector { + /// Build with a private `reqwest::Client` carrying the supplied + /// request timeout. Production callers pass `SERVER_INFO_TIMEOUT` + /// via `default()`; tests may pass shorter timeouts. + pub fn new(timeout: Duration) -> Self { + let client = reqwest::Client::builder() + .timeout(timeout) + .build() + .expect("introspector http client builds"); + Self { client } + } + + /// Reuse a caller-owned `reqwest::Client`. Useful in tests that want + /// to assert request shape via a fake HTTP transport, or to share a + /// connection pool across components. + pub fn with_client(client: reqwest::Client) -> Self { + Self { client } + } + + /// Fetch `/server_info` for the worker. Never returns an error: + /// any failure is logged at `warn!` and yields a default + /// `ServerInfo` with both halves `None`. Callers register the + /// worker with empty model IDs and no event subscription on the + /// failure path; future re-discovery will retry. + /// + /// Transient failures (network errors, 5xx) are retried up to + /// `FETCH_MAX_ATTEMPTS` times with exponential backoff. 4xx + /// responses and JSON-parse errors short-circuit immediately — + /// the worker answered authoritatively, retrying won't help. + pub async fn fetch(&self, worker_url: &str) -> ServerInfo { + let server_info_url = format!("{}/server_info", worker_url.trim_end_matches('/')); + let parsed = match Self::fetch_with_retry(&self.client, &server_info_url, worker_url).await + { + Some(p) => p, + None => return ServerInfo::default(), + }; + + let served_model_name = match parsed.served_model_name { + Some(name) if !name.is_empty() => Some(name), + Some(_) => { + warn!( + worker_url = %worker_url, + "introspect: /server_info has empty `served_model_name`; registering worker with empty model_ids" + ); + None + } + None => None, + }; + + let event_config = parsed + .kv_events + .map(|block| resolve_event_config(block, worker_url)); + + let disaggregation_role = resolve_disaggregation_role( + parsed.disaggregation_mode.as_deref(), + parsed.disaggregation_bootstrap_port, + worker_url, + ); + + ServerInfo { + served_model_name, + event_config, + disaggregation_role, + } + } + + /// Issue the `/server_info` GET with bounded retry on transient + /// errors. Returns `Some(body)` on success, `None` after exhausting + /// retries (the caller falls back to default `ServerInfo`). + async fn fetch_with_retry( + client: &reqwest::Client, + server_info_url: &str, + worker_url: &str, + ) -> Option { + let mut delay = FETCH_BACKOFF_BASE; + for attempt in 1..=FETCH_MAX_ATTEMPTS { + match client.get(server_info_url).send().await { + Err(e) => { + warn!( + worker_url = %worker_url, + attempt, + error = %e, + "introspect: /server_info request failed; will retry" + ); + } + Ok(resp) if resp.status().is_server_error() => { + warn!( + worker_url = %worker_url, + attempt, + status = %resp.status(), + "introspect: /server_info returned 5xx; will retry" + ); + } + Ok(resp) if !resp.status().is_success() => { + warn!( + worker_url = %worker_url, + status = %resp.status(), + "introspect: /server_info returned non-2xx; registering worker with empty model_ids" + ); + return None; + } + Ok(resp) => match resp.json::().await { + Ok(body) => return Some(body), + Err(e) => { + warn!( + worker_url = %worker_url, + error = %e, + "introspect: /server_info JSON parse failed; registering worker with empty model_ids" + ); + return None; + } + }, + } + if attempt < FETCH_MAX_ATTEMPTS { + tokio::time::sleep(delay).await; + delay *= 2; + } + } + warn!( + worker_url = %worker_url, + attempts = FETCH_MAX_ATTEMPTS, + "introspect: /server_info failed after retries; registering worker with empty model_ids" + ); + None + } +} + +/// Map the two `disaggregation_*` fields from `/server_info` into a +/// `DisaggregationRole`. Returns `None` when the worker hasn't told us +/// enough to be useful — the caller treats that as "defer to the +/// discovery backend's classification" instead of forcing Plain, which +/// preserves backwards compatibility with SGLang versions that predate +/// the field. +/// +/// Resolution table: +/// +/// | `disaggregation_mode` | `disaggregation_bootstrap_port` | Result | +/// |------------------------------|----------------------------------|-------------------------------------| +/// | `None` (older SGLang) | _any_ | `None` — defer to backend | +/// | `Some("null")` | _any_ | `Some(Plain)` | +/// | `Some("prefill")` | `Some(p)` | `Some(Prefill { bootstrap_port: p })` | +/// | `Some("prefill")` | `None` | warn + `None` — defer to backend | +/// | `Some("decode")` | _any_ | `Some(Decode)` | +/// | `Some(other)` | _any_ | warn + `None` | +fn resolve_disaggregation_role( + mode: Option<&str>, + bootstrap_port: Option, + worker_url: &str, +) -> Option { + match mode { + None => None, + Some("null") => Some(DisaggregationRole::Plain), + Some("prefill") => match bootstrap_port { + Some(p) => Some(DisaggregationRole::Prefill { bootstrap_port: p }), + None => { + warn!( + worker_url = %worker_url, + "introspect: /server_info reports disaggregation_mode=\"prefill\" but \ + disaggregation_bootstrap_port is missing; deferring to the discovery \ + backend's classification" + ); + None + } + }, + Some("decode") => Some(DisaggregationRole::Decode), + Some(other) => { + warn!( + worker_url = %worker_url, + disaggregation_mode = %other, + "introspect: /server_info has unknown disaggregation_mode value; \ + deferring to the discovery backend's classification" + ); + None + } + } +} + +impl Default for WorkerIntrospector { + fn default() -> Self { + Self::new(SERVER_INFO_TIMEOUT) + } +} + +/// Substitute a wildcard bind host (`*`, `0.0.0.0`, `::`, `[::]`) with +/// the host parsed from the worker URL — the gateway has to connect to +/// a routable address. An unparsable worker URL leaves the host +/// unchanged: the subsequent ZMQ connect will fail visibly with the +/// wildcard literal, which is the same observable failure mode that +/// would occur today if the bind/connect were skipped. +pub(crate) fn resolve_event_config(block: KvEventsBlock, worker_url: &str) -> EventConfig { + let host = if matches!( + block.endpoint_host.as_str(), + "*" | "0.0.0.0" | "::" | "[::]" + ) { + match Url::parse(worker_url) + .ok() + .and_then(|u| u.host_str().map(|s| s.to_owned())) + { + Some(h) => h, + None => { + warn!( + worker_url = %worker_url, + "introspect: cannot parse worker_url for wildcard substitution; keeping advertised host" + ); + block.endpoint_host + } + } + } else { + block.endpoint_host + }; + EventConfig { + host, + port_base: block.endpoint_port_base, + topic: block.topic, + block_size: block.block_size, + dp_size: block.dp_size, + } +} + +/// Projection of `/server_info` used by the introspector. Every field is +/// `#[serde(default)]` so a worker that exposes only some of them still +/// deserialises; downstream callers handle `None` as "absent". +#[derive(Debug, Default, Deserialize)] +struct ServerInfoBody { + #[serde(default)] + served_model_name: Option, + #[serde(default)] + kv_events: Option, + /// Carries the value of `ServerArgs.disaggregation_mode` + /// (`"null"` | `"prefill"` | `"decode"`). Absent on older SGLang + /// versions that predate the field. + #[serde(default)] + disaggregation_mode: Option, + /// `ServerArgs.disaggregation_bootstrap_port`. Meaningful only when + /// `disaggregation_mode == "prefill"`; the prefill server's + /// bootstrap server binds to exactly this port (no internal offset). + #[serde(default)] + disaggregation_bootstrap_port: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct KvEventsBlock { + // Forward-compatibility: the only publisher implementation + // supported on the gateway side is ZMQ. Keeping the field optional + // means a future SGLang that adds a non-ZMQ publisher string won't + // fail deserialize; the resulting subscriber will still try to open + // a ZMQ connection and fail visibly. + #[allow(dead_code)] + #[serde(default)] + publisher: Option, + pub endpoint_host: String, + pub endpoint_port_base: u16, + #[serde(default)] + pub topic: String, + pub block_size: u32, + pub dp_size: u32, +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{routing::get, Json, Router}; + use serde_json::{json, Value}; + use std::sync::Arc; + use tokio::net::TcpListener; + use tokio::sync::oneshot; + + async fn spawn_fake_worker(body: Value) -> (String, oneshot::Sender<()>) { + let body = Arc::new(body); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/server_info", + get(move || { + let body = body.clone(); + async move { Json((*body).clone()) } + }), + ); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = rx.await; + }) + .await; + }); + (format!("http://127.0.0.1:{port}"), tx) + } + + fn fast_introspector() -> WorkerIntrospector { + WorkerIntrospector::new(Duration::from_millis(500)) + } + + #[tokio::test] + async fn fetch_returns_both_served_model_name_and_event_config() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "Qwen3-0.6B", + "kv_events": { + "publisher": "zmq", + "endpoint_host": "10.1.2.3", + "endpoint_port_base": 6000, + "topic": "kv", + "block_size": 64, + "dp_size": 2, + } + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert_eq!(got.served_model_name.as_deref(), Some("Qwen3-0.6B")); + let cfg = got.event_config.expect("kv_events present"); + assert_eq!(cfg.host, "10.1.2.3"); + assert_eq!(cfg.port_base, 6000); + assert_eq!(cfg.topic, "kv"); + assert_eq!(cfg.block_size, 64); + assert_eq!(cfg.dp_size, 2); + } + + #[tokio::test] + async fn fetch_substitutes_wildcard_host() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "m", + "kv_events": { + "publisher": "zmq", + "endpoint_host": "*", + "endpoint_port_base": 5557, + "topic": "kv", + "block_size": 64, + "dp_size": 1, + } + })) + .await; + let got = fast_introspector().fetch(&url).await; + let cfg = got.event_config.expect("kv_events present"); + assert_eq!(cfg.host, "127.0.0.1"); + } + + #[tokio::test] + async fn fetch_returns_empty_on_connection_refused() { + // Port 1 is reserved; bind a temp listener to reserve a free + // port then drop it so the connect fails fast. + let temp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = temp.local_addr().unwrap().port(); + drop(temp); + let url = format!("http://127.0.0.1:{port}"); + let got = fast_introspector().fetch(&url).await; + assert!( + got.served_model_name.is_none(), + "served_model_name must be None on connection refused" + ); + assert!( + got.event_config.is_none(), + "event_config must be None on connection refused" + ); + } + + #[tokio::test] + async fn fetch_only_served_model_name_when_kv_events_absent() { + let (url, _shutdown) = spawn_fake_worker(json!({"served_model_name": "m"})).await; + let got = fast_introspector().fetch(&url).await; + assert_eq!(got.served_model_name.as_deref(), Some("m")); + assert!(got.event_config.is_none()); + } + + #[tokio::test] + async fn fetch_only_event_config_when_served_model_name_absent() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "kv_events": { + "publisher": "zmq", + "endpoint_host": "127.0.0.1", + "endpoint_port_base": 5557, + "topic": "", + "block_size": 64, + "dp_size": 1, + } + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert!(got.served_model_name.is_none()); + let cfg = got.event_config.expect("kv_events present"); + assert_eq!(cfg.port_base, 5557); + } + + /// `disaggregation_mode=prefill` + a bootstrap port → manager should + /// see the worker as a prefill peer with the supplied port. This is + /// the happy path that lets PD-on-K8s skip pod annotations entirely. + #[tokio::test] + async fn fetch_resolves_prefill_role_with_bootstrap_port() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "m", + "disaggregation_mode": "prefill", + "disaggregation_bootstrap_port": 8998, + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert_eq!( + got.disaggregation_role, + Some(DisaggregationRole::Prefill { + bootstrap_port: 8998 + }), + ); + } + + /// `disaggregation_mode=decode` → role is Decode regardless of any + /// bootstrap-port field value (decode workers don't bind one). + #[tokio::test] + async fn fetch_resolves_decode_role() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "m", + "disaggregation_mode": "decode", + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert_eq!(got.disaggregation_role, Some(DisaggregationRole::Decode)); + } + + /// `disaggregation_mode="null"` is SGLang's explicit "not + /// disaggregated" value — we trust it and force the worker to Plain + /// even if the discovery backend mistakenly classified it as + /// prefill/decode. + #[tokio::test] + async fn fetch_resolves_plain_role_when_mode_is_null() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "m", + "disaggregation_mode": "null", + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert_eq!(got.disaggregation_role, Some(DisaggregationRole::Plain)); + } + + /// Partial data (`prefill` mode with no bootstrap port) returns + /// `None` so the manager keeps the discovery backend's + /// classification. The alternative — forcing Plain — would silently + /// demote a misconfigured prefill worker to plain dispatch. + #[tokio::test] + async fn fetch_defers_to_backend_when_prefill_mode_lacks_bootstrap_port() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "m", + "disaggregation_mode": "prefill", + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert!( + got.disaggregation_role.is_none(), + "prefill with no bootstrap port must defer to backend, got {:?}", + got.disaggregation_role, + ); + } + + /// Older SGLang doesn't expose `disaggregation_mode`. The + /// introspector must not invent a classification — the discovery + /// backend's seed (K8s labels, static-urls Plain default) still + /// drives mode for these workers. + #[tokio::test] + async fn fetch_defers_to_backend_when_mode_field_is_absent() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "m", + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert!(got.disaggregation_role.is_none()); + } + + /// Unknown `disaggregation_mode` value (future SGLang adds a new + /// disaggregation flavor, network garbled the field, etc.) → defer + /// to backend rather than guessing. + #[tokio::test] + async fn fetch_defers_to_backend_when_mode_is_unrecognized() { + let (url, _shutdown) = spawn_fake_worker(json!({ + "served_model_name": "m", + "disaggregation_mode": "encode_only", + "disaggregation_bootstrap_port": 8998, + })) + .await; + let got = fast_introspector().fetch(&url).await; + assert!(got.disaggregation_role.is_none()); + } +} diff --git a/experimental/sgl-router/src/workers/manager.rs b/experimental/sgl-router/src/workers/manager.rs new file mode 100644 index 000000000000..4a583de663fc --- /dev/null +++ b/experimental/sgl-router/src/workers/manager.rs @@ -0,0 +1,765 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::config::Config; +use crate::discovery::{DiscoveryEvent, ModelId, WorkerId, WorkerMode, WorkerSpec}; +use crate::health::circuit_breaker::CircuitBreakerConfig; +use crate::policies::active_load::ActiveLoadRegistry; +use crate::policies::kv_events::KvEventIndex; +use crate::workers::introspect::{DisaggregationRole, WorkerIntrospector}; +use crate::workers::WorkerRegistry; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; + +/// Resolve the circuit-breaker config for all model IDs carried by a spec. +/// +/// Workers may serve multiple models; we use the config of the **first** model +/// that has an explicit CB config, falling back to `None` (default config). +fn cb_config_for_spec(spec: &WorkerSpec, cfg: &Config) -> Option { + for model_id in &spec.model_ids { + if let Some(mc) = cfg.models.iter().find(|m| m.id == model_id.0) { + if let Some(cbc) = &mc.circuit_breaker { + return Some(CircuitBreakerConfig { + threshold: cbc.threshold, + cool_down: Duration::from_secs(cbc.cool_down_secs), + }); + } + } + } + None +} + +pub async fn run(rx: mpsc::Receiver, registry: Arc) { + run_with_config(rx, registry, None, None, None).await; +} + +/// Run the worker manager, optionally honoring per-model circuit-breaker +/// configuration from `cfg`, an optional KV-event index that is notified +/// on every worker add / remove, and an optional active-load registry +/// that is asked to forget per-worker counters on `Removed`. +/// +/// When `kv_index` is `None` the cache-aware-zmq path is disabled +/// (selection falls through to the non-cache-aware policies); when +/// `active_load` is `None` the active-load bookkeeping is not pruned +/// on worker removal (leaks one `WorkerCounters` slot per departed +/// worker — fine for tests, but production passes `Some(...)`); when +/// `cfg` is `None` the default CB config is used for every worker +/// (threshold = 3). +/// +/// Uses the default HTTP client (2-second timeout) for `/server_info` +/// introspection. Tests that want a tighter timeout call +/// [`run_with_introspector`] directly. +pub async fn run_with_config( + rx: mpsc::Receiver, + registry: Arc, + cfg: Option>, + kv_index: Option>, + active_load: Option>, +) { + run_with_introspector( + rx, + registry, + cfg, + kv_index, + active_load, + Arc::new(WorkerIntrospector::default()), + ) + .await +} + +/// Internal entry point used by tests so they can supply a custom +/// [`WorkerIntrospector`] (e.g. shorter timeout, fake transport). +/// Production callers use [`run_with_config`]. +/// +/// # Concurrency model +/// +/// - **Added(spec):** spawned onto a `tokio::task` so multiple workers +/// can fetch `/server_info` and register concurrently. Without this, +/// a burst of N workers would serialize N × `SERVER_INFO_TIMEOUT` +/// worth of registration latency on the event loop. +/// - **Removed / ModeChanged:** processed sequentially on the event +/// loop, but first **await** any in-flight `Added` task for the same +/// id so the mutation observes the post-Added registry state. +/// Without this await, a `Removed` queued while `Added` is still +/// fetching would no-op (registry empty), then the deferred Added +/// write would leak the worker indefinitely. +pub async fn run_with_introspector( + mut rx: mpsc::Receiver, + registry: Arc, + cfg: Option>, + kv_index: Option>, + active_load: Option>, + introspector: Arc, +) { + // In-flight `Added` registrations, keyed by worker id. Subsequent + // `Removed` / `ModeChanged` events for the same id `await` the + // handle so they observe the registry write the spawned task is + // about to perform. Entries are removed on completion (Added's + // own task drops the slot before returning). + let mut pending: HashMap> = HashMap::new(); + + while let Some(event) = rx.recv().await { + // Opportunistically reap handles whose tasks have already + // completed so the map doesn't grow without bound under steady- + // state churn. This is O(map.len()) per event but the map only + // holds in-flight Added events (typically << total workers). + pending.retain(|_, h| !h.is_finished()); + + match event { + DiscoveryEvent::Added(spec) => { + tracing::info!("discovery: +worker {} ({:?})", spec.id, spec.mode); + let id = spec.id.clone(); + // If a previous Added for the same id is still in-flight, + // drain it first so the upsert observes a consistent + // pre-state (and so the new spawn doesn't race with the + // old). + if let Some(prev) = pending.remove(&id) { + let _ = prev.await; + } + let registry_t = registry.clone(); + let cfg_t = cfg.clone(); + let kv_index_t = kv_index.clone(); + let introspector_t = introspector.clone(); + let handle = tokio::spawn(async move { + register_one(spec, registry_t, cfg_t, kv_index_t, introspector_t).await; + }); + pending.insert(id, handle); + } + DiscoveryEvent::Removed { id } => { + tracing::info!("discovery: -worker {id}"); + if let Some(prev) = pending.remove(&id) { + // Wait for the matching Added to finish its registry + // write so the Removed observes (and clears) it. + let _ = prev.await; + } + // Look up the URL before dropping the entry so the + // KV-event index can clear its per-(url, dp_rank) state. + let worker_url = registry.get(&id).map(|w| w.url.clone()); + registry.remove(&id); + match (&kv_index, worker_url) { + (Some(idx), Some(url)) => { + idx.remove_worker(&url).await; + } + (Some(_), None) => { + // Registry didn't know this worker but kv-events + // is enabled — duplicate Removed or out-of-order + // event. KvEventIndex state for this id (if any) + // leaks until process shutdown; log so it's + // detectable. + tracing::warn!( + id = %id, + "discovery: Removed without a known URL; kv-events state (if any) not cleared", + ); + } + (None, _) => {} + } + // Drop the active-load per-worker counters slot. + // Idempotent on the registry side, so we call it + // unconditionally — a Removed for an unknown worker + // (duplicate event) is a no-op. In-flight guards + // pointing at this id are NOT invalidated; their drop + // still removes the per-request entry cleanly, but the + // per-worker counters slot will not be re-created + // (selectors no longer see the worker, so no new + // requests can register against it). + if let Some(al) = &active_load { + al.forget_worker(&id); + } + } + DiscoveryEvent::ModeChanged { id, mode } => { + if let Some(prev) = pending.remove(&id) { + // Same rationale as Removed: wait for the registry + // write so the mode flip lands on the new entry. + let _ = prev.await; + } + // Mutate mode in place — preserves active_requests counter + // (in-flight LoadGuards stay valid) and CircuitBreaker state + // (open/half-open survives PD role flips). + // + // workers_for_mode filters at query time via w.mode(), so no + // secondary index needs updating. + match registry.get(&id) { + Some(w) => { + tracing::info!("discovery: ~worker {id} mode→{mode:?}"); + w.set_mode(mode); + } + None => { + tracing::warn!( + id = %id, + mode = ?mode, + "discovery: ModeChanged for unknown worker — out-of-order event from backend", + ); + } + } + } + } + } + + // Drain any still-running registration tasks so callers `await`ing + // the manager handle (tests, shutdown paths) see all registry + // mutations land before the future resolves. + for (_, h) in pending.drain() { + let _ = h.await; + } +} + +/// Onboard a single worker: introspect once, then dispatch the result +/// to the registry and (if enabled) the KV-event index. Failure of any +/// step is logged inside the call chain; we still register the worker +/// with empty `model_ids` so the rest of the proxy plane treats it as +/// reachable. +async fn register_one( + mut spec: WorkerSpec, + registry: Arc, + cfg: Option>, + kv_index: Option>, + introspector: Arc, +) { + let worker_url = spec.url.clone(); + let info = introspector.fetch(&worker_url).await; + if let Some(name) = info.served_model_name { + spec.model_ids = vec![ModelId(name)]; + } + // Trust `/server_info` over the discovery backend when the worker + // self-disclosed its PD role: the server's own ServerArgs is the + // authoritative source for `disaggregation_mode` and + // `disaggregation_bootstrap_port`. The backend's mode (from K8s + // labels, static-urls seed, etc.) was a best-guess seed; if the + // server says it's actually a prefill peer on port 8998, that wins. + // `None` here means the worker didn't tell us — keep the backend's + // classification (older SGLang without the field, partial response, + // unknown mode value, etc.). + if let Some(role) = info.disaggregation_role { + let (new_mode, new_port) = match role { + DisaggregationRole::Plain => (WorkerMode::Plain, None), + DisaggregationRole::Prefill { bootstrap_port } => { + (WorkerMode::Prefill, Some(bootstrap_port)) + } + DisaggregationRole::Decode => (WorkerMode::Decode, None), + }; + if (new_mode, new_port) != (spec.mode, spec.bootstrap_port) { + tracing::info!( + worker_url = %worker_url, + backend_mode = ?spec.mode, + resolved_mode = ?new_mode, + backend_bootstrap_port = ?spec.bootstrap_port, + resolved_bootstrap_port = ?new_port, + "/server_info overrode discovery-backend classification", + ); + spec.mode = new_mode; + spec.bootstrap_port = new_port; + } + } + let cb = cfg.as_ref().and_then(|c| cb_config_for_spec(&spec, c)); + if let Err(e) = registry.add_with_cb(spec, cb) { + // Mixed PD + plain on the same model is rejected at registration + // time. Log loudly so the operator notices the conflicting + // worker — the alternative (silently dropping into either pool) + // makes the resolver surface the wrong 5xx code under partial + // outages. Skip the kv_index hook too: a worker we didn't add + // shouldn't drive cache-aware tree state. + tracing::error!( + worker_url = %worker_url, + error = %e, + "worker manager: refused to register worker due to mixed PD/plain configuration", + ); + return; + } + if let Some(idx) = kv_index { + // Pass the pre-resolved EventConfig so the KvEventIndex does + // not issue a second `/server_info` round-trip. + idx.add_worker(&worker_url, info.event_config).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{ + ActiveLoadConfig, CircuitBreakerConfig as RawCbConfig, DiscoveryBackend, DiscoveryConfig, + ModelConfig, PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, + }; + use crate::discovery::{WorkerId, WorkerMode}; + use axum::{routing::get, Json, Router}; + use serde_json::{json, Value}; + use std::num::NonZeroU32; + use tokio::net::TcpListener; + use tokio::sync::oneshot; + + fn cfg_with_model_cb(id: &str, threshold: u32, cool_down_secs: u64) -> Config { + Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: Default::default(), + models: vec![ModelConfig { + id: id.into(), + tokenizer_path: "/tmp/x".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: Some(RawCbConfig { + threshold: NonZeroU32::new(threshold).unwrap(), + cool_down_secs, + }), + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://test:30000".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + } + } + + #[test] + fn cb_config_for_spec_carries_threshold_and_cool_down() { + let cfg = cfg_with_model_cb("m", 5, 60); + let spec = WorkerSpec { + id: WorkerId("w".into()), + url: "http://x".into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("m".into())], + bootstrap_port: None, + }; + let cb = cb_config_for_spec(&spec, &cfg).expect("model has cb config"); + assert_eq!(cb.threshold.get(), 5); + assert_eq!(cb.cool_down, Duration::from_secs(60)); + } + + /// Helper: spawn a tiny fake worker that returns the supplied JSON body + /// on `GET /server_info`. Returns the worker URL + a shutdown channel. + async fn spawn_fake_server_info_worker(body: Value) -> (String, oneshot::Sender<()>) { + let body = Arc::new(body); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/server_info", + get(move || { + let body = body.clone(); + async move { Json((*body).clone()) } + }), + ); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = rx.await; + }) + .await; + }); + (format!("http://127.0.0.1:{port}"), tx) + } + + /// Reserve a TCP port and immediately drop the listener so subsequent + /// connection attempts during the test fail fast with + /// ConnectionRefused. + fn unused_port() -> u16 { + use std::net::TcpListener; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap().port() + } + + fn fast_introspector() -> Arc { + Arc::new(WorkerIntrospector::new(Duration::from_millis(500))) + } + + /// `/server_info` returns `served_model_name` => the registry entry + /// carries that as a single `ModelId`. + #[tokio::test] + async fn manager_resolves_model_id_from_server_info() { + let (worker_url, _shutdown) = + spawn_fake_server_info_worker(json!({"served_model_name": "Qwen3-0.6B"})).await; + + let registry = Arc::new(WorkerRegistry::default()); + let (tx, rx) = mpsc::channel::(8); + let manager_handle = tokio::spawn(run_with_introspector( + rx, + registry.clone(), + None, + None, + None, + fast_introspector(), + )); + + let spec = WorkerSpec { + id: WorkerId("w-1".into()), + url: worker_url, + mode: WorkerMode::Plain, + model_ids: Vec::new(), + bootstrap_port: None, + }; + tx.send(DiscoveryEvent::Added(spec.clone())).await.unwrap(); + + let registered = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Some(w) = registry.get(&spec.id) { + if w.model_ids.iter().any(|m| m.0 == "Qwen3-0.6B") { + return true; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!(registered.is_ok(), "manager did not resolve model id"); + + drop(tx); + let _ = manager_handle.await; + } + + /// Worker unreachable (connection refused) => registry still has the + /// worker, with `model_ids` empty. No panic; manager continues running. + #[tokio::test] + async fn manager_registers_with_empty_model_ids_when_server_info_unreachable() { + let port = unused_port(); + let worker_url = format!("http://127.0.0.1:{port}"); + + let registry = Arc::new(WorkerRegistry::default()); + let (tx, rx) = mpsc::channel::(8); + let manager_handle = tokio::spawn(run_with_introspector( + rx, + registry.clone(), + None, + None, + None, + fast_introspector(), + )); + + let spec = WorkerSpec { + id: WorkerId("w-2".into()), + url: worker_url, + mode: WorkerMode::Plain, + model_ids: Vec::new(), + bootstrap_port: None, + }; + tx.send(DiscoveryEvent::Added(spec.clone())).await.unwrap(); + + let registered = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Some(w) = registry.get(&spec.id) { + return w.model_ids.is_empty(); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + matches!(registered, Ok(true)), + "manager must register worker with empty model_ids when /server_info fails: {registered:?}" + ); + + drop(tx); + let _ = manager_handle.await; + } + + /// `/server_info` returns a JSON object without `served_model_name` + /// (or with the empty string): manager logs a warn and registers the + /// worker with empty `model_ids`. + #[tokio::test] + async fn manager_registers_with_empty_model_ids_when_served_model_name_missing() { + let (no_field_url, _no_field_shutdown) = + spawn_fake_server_info_worker(json!({"other_field": "value"})).await; + let (empty_url, _empty_shutdown) = + spawn_fake_server_info_worker(json!({"served_model_name": ""})).await; + + let registry = Arc::new(WorkerRegistry::default()); + let (tx, rx) = mpsc::channel::(8); + let manager_handle = tokio::spawn(run_with_introspector( + rx, + registry.clone(), + None, + None, + None, + fast_introspector(), + )); + + for (id, url) in [("w-no-field", no_field_url), ("w-empty", empty_url)] { + let spec = WorkerSpec { + id: WorkerId(id.into()), + url, + mode: WorkerMode::Plain, + model_ids: Vec::new(), + bootstrap_port: None, + }; + tx.send(DiscoveryEvent::Added(spec.clone())).await.unwrap(); + let registered = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Some(w) = registry.get(&spec.id) { + return w.model_ids.is_empty(); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + matches!(registered, Ok(true)), + "manager must register worker {id} with empty model_ids when served_model_name is missing/empty: {registered:?}" + ); + } + + drop(tx); + let _ = manager_handle.await; + } + + /// End-to-end wiring smoke test: spin up a fake worker, run the + /// manager with a real `KvEventIndex` against that worker URL, and + /// verify both `Added` and `Removed` propagate through to the + /// index's internal worker map. + /// + /// The fake worker advertises a `kv_events` block in `/server_info`, + /// so the manager → KvEventIndex → discovery → registry path is + /// exercised end-to-end. The ZMQ connect itself targets an unused + /// port and fails (port is closed), but the *index-level* state still + /// records the worker — which is exactly the invariant under test: + /// `add_worker` registers the worker URL in `KvEventIndex.workers` + /// even when the per-rank SUB connect fails. + #[tokio::test] + async fn manager_drives_kv_index_lifecycle() { + use tokio::time::timeout; + + // The fake worker advertises both `kv_events` for KvEventIndex AND + // `served_model_name` so the worker-manager HTTP introspection + // also resolves a model id. + let body = json!({ + "served_model_name": "m", + "kv_events": { + "publisher": "zmq", + "endpoint_host": "127.0.0.1", + "endpoint_port_base": 60000, + "topic": "", + "block_size": 64, + "dp_size": 1, + } + }); + let (worker_url, _shutdown) = spawn_fake_server_info_worker(body).await; + + let registry = Arc::new(WorkerRegistry::default()); + let kv_index = KvEventIndex::new(); + let (tx, rx) = mpsc::channel::(8); + let manager_handle = tokio::spawn(run_with_config( + rx, + registry.clone(), + None, + Some(kv_index.clone()), + None, + )); + + let spec = WorkerSpec { + id: WorkerId("w-1".into()), + url: worker_url.clone(), + mode: WorkerMode::Plain, + model_ids: Vec::new(), + bootstrap_port: None, + }; + tx.send(DiscoveryEvent::Added(spec.clone())).await.unwrap(); + // Wait until the manager has both registered the worker AND + // resolved /server_info — bound the wait so a hang surfaces. + let added = timeout(Duration::from_secs(2), async { + loop { + if registry.get(&spec.id).is_some() && kv_index.known_worker_count() == 1 { + return true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!(added.is_ok(), "manager failed to propagate Added"); + + tx.send(DiscoveryEvent::Removed { + id: spec.id.clone(), + }) + .await + .unwrap(); + let removed = timeout(Duration::from_secs(2), async { + loop { + if registry.get(&spec.id).is_none() && kv_index.known_worker_count() == 0 { + return true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!(removed.is_ok(), "manager failed to propagate Removed"); + + drop(tx); + let _ = manager_handle.await; + kv_index.shutdown().await; + } + + /// `Removed` for an unknown id with kv-events enabled must not panic. + /// The kv_index has no entry for that id either, so it must remain + /// empty after the no-op. + #[tokio::test] + async fn manager_removed_unknown_id_is_noop() { + use tokio::time::sleep; + + let registry = Arc::new(WorkerRegistry::default()); + let kv_index = KvEventIndex::new(); + let (tx, rx) = mpsc::channel::(8); + let manager_handle = tokio::spawn(run_with_config( + rx, + registry.clone(), + None, + Some(kv_index.clone()), + None, + )); + + tx.send(DiscoveryEvent::Removed { + id: WorkerId("never-added".into()), + }) + .await + .unwrap(); + // Let the manager process the event. + sleep(Duration::from_millis(50)).await; + + assert_eq!(kv_index.known_worker_count(), 0); + drop(tx); + let _ = manager_handle.await; + kv_index.shutdown().await; + } + + /// Task B: `DiscoveryEvent::Removed` calls + /// `ActiveLoadRegistry::forget_worker` so the per-worker counters + /// slot is reaped. Without this, a long-lived cluster with worker + /// churn would leak one `WorkerCounters` entry per departed worker. + #[tokio::test] + async fn manager_calls_active_load_forget_on_removed() { + use tokio::time::timeout; + + // Fake worker is needed so the introspection step succeeds and + // the Removed path observes a known URL — same shape as the + // existing `manager_drives_kv_index_lifecycle` test. + let (worker_url, _shutdown) = + spawn_fake_server_info_worker(json!({"served_model_name": "m"})).await; + + let registry = Arc::new(WorkerRegistry::default()); + let active_load = ActiveLoadRegistry::with_defaults(); + let (tx, rx) = mpsc::channel::(8); + let manager_handle = tokio::spawn(run_with_introspector( + rx, + registry.clone(), + None, + None, + Some(Arc::clone(&active_load)), + fast_introspector(), + )); + + let id = WorkerId("w-1".into()); + let spec = WorkerSpec { + id: id.clone(), + url: worker_url, + mode: WorkerMode::Plain, + model_ids: Vec::new(), + bootstrap_port: None, + }; + tx.send(DiscoveryEvent::Added(spec.clone())).await.unwrap(); + // Wait for the manager to land the registry write so the + // subsequent register/forget round trip exercises a live slot. + let added = timeout(Duration::from_secs(2), async { + loop { + if registry.get(&id).is_some() { + return true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!(added.is_ok(), "manager failed to register worker"); + + // Mint a guard to force the active-load registry to create a + // per-worker counters slot for this id. + let _g = active_load.register(id.clone(), "test://", 10, 1); + assert!(active_load.is_known(&id)); + + // Now drive the Removed event and assert the counters slot is + // gone. We tear down the guard last so the request entry is + // exercised on the post-forget path. + tx.send(DiscoveryEvent::Removed { id: id.clone() }) + .await + .unwrap(); + let removed = timeout(Duration::from_secs(2), async { + loop { + if !active_load.is_known(&id) && registry.get(&id).is_none() { + return true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + removed.is_ok(), + "manager must call active_load.forget_worker on Removed", + ); + + drop(tx); + let _ = manager_handle.await; + } + + /// Discovery backend emits a `Plain` worker with no bootstrap port, + /// but `/server_info` says `disaggregation_mode="prefill"` with + /// `disaggregation_bootstrap_port=8998`. The manager must trust + /// `/server_info` and register the worker as Prefill with the + /// disclosed port — this is the load-bearing assertion for + /// PD-on-K8s, where the K8s backend always emits Plain + None for + /// `bootstrap_port` and the manager has to recover the role from + /// the worker's self-disclosure. + #[tokio::test] + async fn manager_overrides_backend_classification_from_server_info() { + let (worker_url, _shutdown) = spawn_fake_server_info_worker(json!({ + "served_model_name": "m", + "disaggregation_mode": "prefill", + "disaggregation_bootstrap_port": 8998, + })) + .await; + + let registry = Arc::new(WorkerRegistry::default()); + let (tx, rx) = mpsc::channel::(8); + let manager_handle = tokio::spawn(run_with_introspector( + rx, + registry.clone(), + None, + None, + None, + fast_introspector(), + )); + + // Backend says Plain + None — the shape the K8s backend always + // emits today. + let spec = WorkerSpec { + id: WorkerId("w-prefill".into()), + url: worker_url, + mode: WorkerMode::Plain, + model_ids: Vec::new(), + bootstrap_port: None, + }; + tx.send(DiscoveryEvent::Added(spec.clone())).await.unwrap(); + + let resolved = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Some(w) = registry.get(&spec.id) { + if w.mode() == WorkerMode::Prefill && w.bootstrap_port() == Some(8998) { + return true; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + resolved.is_ok(), + "manager must apply /server_info disaggregation_role override; \ + expected mode=Prefill bootstrap_port=Some(8998), got {:?}", + registry + .get(&spec.id) + .map(|w| (w.mode(), w.bootstrap_port())), + ); + + drop(tx); + let _ = manager_handle.await; + } +} diff --git a/experimental/sgl-router/src/workers/mod.rs b/experimental/sgl-router/src/workers/mod.rs new file mode 100644 index 000000000000..68b8256a9543 --- /dev/null +++ b/experimental/sgl-router/src/workers/mod.rs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +pub mod introspect; +pub mod manager; +pub mod registry; +pub mod worker; + +pub use introspect::{ServerInfo, WorkerIntrospector}; +pub use registry::WorkerRegistry; +pub use worker::LoadGuard; +pub use worker::Worker; diff --git a/experimental/sgl-router/src/workers/registry.rs b/experimental/sgl-router/src/workers/registry.rs new file mode 100644 index 000000000000..11c9d360dd9e --- /dev/null +++ b/experimental/sgl-router/src/workers/registry.rs @@ -0,0 +1,550 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use crate::health::circuit_breaker::CircuitBreakerConfig; +use crate::workers::worker::Worker; +use dashmap::DashMap; +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; + +/// Reason a [`WorkerRegistry::add`] call refused the spec. +#[derive(Debug, Clone, thiserror::Error)] +pub enum AddWorkerError { + /// The spec's mode (plain vs prefill/decode) conflicts with workers + /// already registered for one of its `model_ids`. The router does + /// not support mixed PD + plain pools on a single model: the + /// resolver derives the PD-vs-plain shape from the registered + /// workers, and a mixed pool would silently degrade to whichever + /// shape happens to be healthy when the other is breaker-open, + /// surfacing the wrong error code to clients. + #[error( + "worker {worker:?} for model {model:?} would mix PD ({pd_mode}) with plain workers on \ + the same model — sgl-router does not support mixed pools. Use one of: only Plain \ + workers, or only Prefill+Decode workers." + )] + MixedPdAndPlain { + worker: WorkerId, + model: ModelId, + /// The role of the *incoming* worker that triggered the conflict + /// (the *existing* worker has the opposite role). + pd_mode: &'static str, + }, +} + +#[derive(Debug, Default)] +pub struct WorkerRegistry { + by_id: DashMap>, + by_model: DashMap>, + /// Serializes the validate→insert section of `add_with_cb` so the + /// `MixedPdAndPlain` check is atomic with the subsequent write. Two + /// concurrent registrations from `manager::register_one` for the + /// same model with conflicting modes could otherwise both observe + /// an empty pool and both insert, leaving the registry in a mixed + /// state — the exact corruption the check is meant to prevent. + /// Reads (`workers_for`, `get`, `len`, …) stay lock-free against + /// the underlying DashMaps; only writes through `add_with_cb` / + /// `remove` take this lock so contention is bounded by registry + /// mutation rate (worker-discovery events), not request rate. + write: Mutex<()>, +} + +impl WorkerRegistry { + pub fn add(&self, spec: WorkerSpec) -> Result<(), AddWorkerError> { + self.add_with_cb(spec, None) + } + + /// Add a worker, optionally supplying a circuit-breaker config. + /// Pass `None` to use the circuit-breaker default (threshold = 3). + /// + /// Re-adding an existing `WorkerId` is an upsert: the prior entry's + /// `by_model` memberships are cleared first so a model that the new + /// spec no longer serves stops resolving to this worker. Without the + /// pre-removal step a worker whose model set shrank would still appear + /// in `workers_for()` because `by_id.get(...)` would + /// return the new worker via the stale model→id index. + /// + /// Returns [`AddWorkerError::MixedPdAndPlain`] when adding the spec + /// would mix PD (prefill/decode) workers with plain workers on the + /// same model. The conflict is detected against the *existing* + /// registry state — re-adding the same worker id is fine (the prior + /// entry is removed first), and adding a worker whose own + /// `model_ids` are all unmixed is fine even if other models in the + /// process have a mix of modes. + /// + /// On rejection the registry is **not** mutated. If the rejected + /// spec carries an id that already has an entry, the prior entry + /// stays put — it's the caller's responsibility to decide whether + /// to evict it (and, importantly, to also clean up sidecar state + /// in `KvEventIndex` / `ActiveLoadRegistry` if so). Doing that + /// cleanup here would leak orphan state into those sidecars when + /// a caller actually wanted to keep the prior entry. + pub fn add_with_cb( + &self, + spec: WorkerSpec, + cb: Option, + ) -> Result<(), AddWorkerError> { + let incoming_mode = spec.mode; + // Hold the write lock for the entire validate→insert sequence. + // Without it, two concurrent callers for conflicting modes on + // the same model can both see an empty pool and both proceed + // to insert, producing the mixed PD+plain state the check + // exists to prevent. + // + // Mutex poisoning here means a previous writer panicked while + // holding the lock — and since the critical section spans + // `remove_locked` + several `by_model` updates + the final + // `by_id.insert`, a panic mid-section can leave the registry + // with a partial entry across the two DashMaps. Recovering via + // `PoisonError::into_inner` would silently continue against + // that half-written state; propagating the panic instead + // surfaces the corruption to `manager::register_one`'s task + // and ultimately trips `supervise_critical_tasks → mark_unready` + // so the pod stops taking traffic. That's the right outcome. + let _guard = self.write.lock().unwrap(); + // Validate against existing workers BEFORE we mutate. Re-adding + // the same id is an upsert; pretend the prior entry is gone for + // the purposes of the check (otherwise an upsert of an unmixed + // worker would self-conflict if its current entry already + // serves the model). + for model in &spec.model_ids { + for existing in self.workers_for(model) { + if existing.id == spec.id { + continue; + } + if modes_are_mixed(incoming_mode, existing.mode()) { + return Err(AddWorkerError::MixedPdAndPlain { + worker: spec.id, + model: model.clone(), + pd_mode: mode_name(incoming_mode), + }); + } + } + } + let w = Arc::new(Worker::with_cb_config(spec, cb)); + let id = w.id.clone(); + self.remove_locked(&id); + for m in &w.model_ids { + self.by_model + .entry(m.clone()) + .or_default() + .insert(id.clone()); + } + self.by_id.insert(id, w); + Ok(()) + } + + pub fn remove(&self, id: &WorkerId) { + // Mirror `add_with_cb`'s write-lock acquisition so removals + // don't race with concurrent adds (a stale `workers_for` snapshot + // could otherwise let an add succeed against a peer that's + // about to be removed, or vice versa). + let _guard = self.write.lock().unwrap(); + self.remove_locked(id); + } + + /// Internal removal that assumes the write lock is already held. + /// Use this from any path that has acquired `self.write`. + fn remove_locked(&self, id: &WorkerId) { + if let Some((_, w)) = self.by_id.remove(id) { + for m in &w.model_ids { + if let Some(mut set) = self.by_model.get_mut(m) { + set.remove(id); + } + } + } + } + + pub fn workers_for(&self, model: &ModelId) -> Vec> { + self.by_model + .get(model) + .map(|ids| { + ids.iter() + .filter_map(|i| self.by_id.get(i).map(|w| Arc::clone(&w))) + .collect() + }) + .unwrap_or_default() + } + + pub fn healthy_workers_for(&self, model: &ModelId) -> Vec> { + // Use `would_allow` (non-mutating) for filtering — `allow()` would + // claim a half-open probe slot for every enumerated candidate, + // starving the worker that the policy actually picks. The probe + // is claimed at dispatch time by `forward_*_to` in + // [`crate::proxy`]. + self.workers_for(model) + .into_iter() + .filter(|w| w.breaker.would_allow()) + .collect() + } + + pub fn workers_for_mode(&self, model: &ModelId, mode: WorkerMode) -> Vec> { + self.workers_for(model) + .into_iter() + .filter(|w| w.mode() == mode) + .collect() + } + + pub fn len(&self) -> usize { + self.by_id.len() + } + + pub fn is_empty(&self) -> bool { + self.by_id.is_empty() + } + + pub fn get(&self, id: &WorkerId) -> Option> { + self.by_id.get(id).map(|w| Arc::clone(&w)) + } +} + +/// `true` when the two modes can't coexist for the same model — i.e. +/// one is `Plain` and the other is `Prefill` or `Decode`. +fn modes_are_mixed(a: WorkerMode, b: WorkerMode) -> bool { + matches!( + (a, b), + (WorkerMode::Plain, WorkerMode::Prefill | WorkerMode::Decode) + | (WorkerMode::Prefill | WorkerMode::Decode, WorkerMode::Plain) + ) +} + +fn mode_name(m: WorkerMode) -> &'static str { + match m { + WorkerMode::Plain => "plain", + WorkerMode::Prefill => "prefill", + WorkerMode::Decode => "decode", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; + + fn spec(id: &str, mode: WorkerMode, models: &[&str]) -> WorkerSpec { + WorkerSpec { + id: WorkerId(id.into()), + url: format!("http://{id}:30000"), + mode, + model_ids: models.iter().map(|m| ModelId((*m).into())).collect(), + bootstrap_port: None, + } + } + + #[test] + fn add_then_query_by_model() { + let r = WorkerRegistry::default(); + let _ = r.add(spec("w1", WorkerMode::Plain, &["m1", "m2"])); + let _ = r.add(spec("w2", WorkerMode::Plain, &["m1"])); + let m1 = r.workers_for(&ModelId("m1".into())); + let m2 = r.workers_for(&ModelId("m2".into())); + let m_missing = r.workers_for(&ModelId("missing".into())); + assert_eq!(m1.len(), 2); + assert_eq!(m2.len(), 1); + assert!(m_missing.is_empty()); + } + + #[test] + fn remove_drops_from_all_models() { + let r = WorkerRegistry::default(); + let _ = r.add(spec("w1", WorkerMode::Plain, &["m1", "m2"])); + r.remove(&WorkerId("w1".into())); + assert!(r.workers_for(&ModelId("m1".into())).is_empty()); + assert!(r.workers_for(&ModelId("m2".into())).is_empty()); + } + + /// `healthy_workers_for` must drop workers whose breaker is Open. + /// An earlier version of this test asserted `healthy.len() == 2` + /// against two workers with untouched breakers — i.e., it pinned + /// only the no-op case (both Closed) and would have passed even if + /// `healthy_workers_for` ignored the breaker entirely and was a + /// thin alias for `workers_for`. Tripping one breaker and asserting + /// the surviving set excludes it is the actual contract. + #[test] + fn healthy_subset_filters_via_breaker() { + use crate::health::circuit_breaker::CircuitBreakerConfig; + use std::num::NonZeroU32; + use std::time::Duration; + + let r = WorkerRegistry::default(); + let _ = r.add_with_cb(spec("ok", WorkerMode::Plain, &["m"]), None); + // Give "bad" a threshold=1 breaker so a single record_failure + // flips it to Open. + let _ = r.add_with_cb( + spec("bad", WorkerMode::Plain, &["m"]), + Some(CircuitBreakerConfig { + threshold: NonZeroU32::new(1).unwrap(), + cool_down: Duration::from_secs(30), + }), + ); + let bad = r.get(&WorkerId("bad".into())).expect("bad worker present"); + bad.breaker.record_failure(); + assert!( + !bad.breaker.would_allow(), + "sanity: threshold=1 + one failure must Open the breaker", + ); + + let healthy = r.healthy_workers_for(&ModelId("m".into())); + assert_eq!( + healthy.len(), + 1, + "only the worker with a non-Open breaker should survive", + ); + assert_eq!(healthy[0].id, WorkerId("ok".into())); + } + + /// PD prefill/decode workers and plain workers cannot coexist on the + /// same model. The resolver bases its PD-vs-plain shape on registered + /// workers; mixing the two forces a fallback to whichever bucket + /// happens to be healthy when the other is breaker-open, surfacing + /// the wrong 5xx code (`no_healthy_workers` instead of + /// `no_prefill_workers_available`). Reject the conflicting add up + /// front so the operator sees the misconfiguration immediately. + #[test] + fn plain_then_pd_for_same_model_is_rejected() { + let r = WorkerRegistry::default(); + assert!(r.add(spec("plain", WorkerMode::Plain, &["m"])).is_ok()); + let err = r + .add(spec("p", WorkerMode::Prefill, &["m"])) + .expect_err("PD worker must be rejected when model already has Plain workers"); + let msg = err.to_string(); + assert!( + msg.contains("PD") && msg.contains("plain"), + "error must name both modes; got: {msg}" + ); + // Existing plain worker survives the rejection. + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Plain) + .len(), + 1, + ); + assert!(r + .workers_for_mode(&ModelId("m".into()), WorkerMode::Prefill) + .is_empty()); + } + + #[test] + fn pd_then_plain_for_same_model_is_rejected() { + let r = WorkerRegistry::default(); + assert!(r.add(spec("p", WorkerMode::Prefill, &["m"])).is_ok()); + assert!(r.add(spec("d", WorkerMode::Decode, &["m"])).is_ok()); + let err = r + .add(spec("plain", WorkerMode::Plain, &["m"])) + .expect_err("plain worker must be rejected when model already has PD workers"); + let msg = err.to_string(); + assert!( + msg.contains("PD") && msg.contains("plain"), + "error must name both modes; got: {msg}" + ); + } + + #[test] + fn plain_only_pool_admits_more_plain_workers() { + let r = WorkerRegistry::default(); + assert!(r.add(spec("a", WorkerMode::Plain, &["m"])).is_ok()); + assert!(r.add(spec("b", WorkerMode::Plain, &["m"])).is_ok()); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Plain) + .len(), + 2, + ); + } + + #[test] + fn pd_pool_admits_more_pd_workers_in_both_roles() { + let r = WorkerRegistry::default(); + assert!(r.add(spec("p1", WorkerMode::Prefill, &["m"])).is_ok()); + assert!(r.add(spec("p2", WorkerMode::Prefill, &["m"])).is_ok()); + assert!(r.add(spec("d1", WorkerMode::Decode, &["m"])).is_ok()); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Prefill) + .len(), + 2, + ); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Decode) + .len(), + 1, + ); + } + + /// Re-adding a worker with a shrunken `model_ids` must drop the worker + /// from the models it no longer serves. The earlier implementation + /// only updated `by_id`, leaving the stale `by_model` entries pointing + /// at the new worker. + #[test] + fn re_add_with_shrunken_model_set_drops_stale_indexes() { + let r = WorkerRegistry::default(); + let _ = r.add(spec("w1", WorkerMode::Plain, &["m1", "m2"])); + assert_eq!(r.workers_for(&ModelId("m2".into())).len(), 1); + + let _ = r.add(spec("w1", WorkerMode::Plain, &["m1"])); + assert_eq!( + r.workers_for(&ModelId("m2".into())).len(), + 0, + "w1 no longer serves m2 after re-add" + ); + assert_eq!( + r.workers_for(&ModelId("m1".into())).len(), + 1, + "w1 still serves m1" + ); + } + + /// Re-adding the same id with a different mode reflects in + /// `workers_for_mode`. + #[test] + fn re_add_with_different_mode_updates_mode_filter() { + let r = WorkerRegistry::default(); + let _ = r.add(spec("w1", WorkerMode::Prefill, &["m"])); + let _ = r.add(spec("w1", WorkerMode::Decode, &["m"])); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Prefill) + .len(), + 0, + ); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Decode) + .len(), + 1, + ); + } + + /// On a rejected upsert with `MixedPdAndPlain`, the registry is + /// **not** mutated — the prior entry for the rejected id stays + /// put. Eviction (with the matching `KvEventIndex` / + /// `ActiveLoadRegistry` cleanup) is the manager's responsibility; + /// doing it here would leak orphan state in those sidecars. + #[test] + fn upsert_rejected_with_mixed_modes_leaves_registry_unchanged() { + let r = WorkerRegistry::default(); + // Healthy PD pool on model m. + let _ = r.add(spec("p", WorkerMode::Prefill, &["m"])); + let _ = r.add(spec("d", WorkerMode::Decode, &["m"])); + // Re-add "p" with Plain mode — discovery has reported a role flip. + // The decode worker "d" is still on m, so validation rejects. + let err = r + .add(spec("p", WorkerMode::Plain, &["m"])) + .expect_err("plain upsert must be rejected when peer decode worker remains"); + assert!(err.to_string().contains("plain"), "got: {err}"); + // Prior "p" entry survives (still Prefill). The registry + // deliberately does NOT auto-evict on rejection — eviction + // (and the matching sidecar cleanup) is the caller's call. + let p = r + .get(&WorkerId("p".into())) + .expect("prior entry must remain — caller owns the cleanup"); + assert_eq!(p.mode(), WorkerMode::Prefill); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Prefill) + .len(), + 1, + ); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Decode) + .len(), + 1, + ); + } + + /// A *new* (not-yet-registered) worker rejected with `MixedPdAndPlain` + /// must not affect the pool. Combined with the upsert test above, + /// this pins that rejection never mutates registry state on its own. + #[test] + fn rejected_new_add_leaves_pool_untouched() { + let r = WorkerRegistry::default(); + let _ = r.add(spec("plain", WorkerMode::Plain, &["m"])); + let err = r + .add(spec("p", WorkerMode::Prefill, &["m"])) + .expect_err("PD worker must be rejected against existing plain pool"); + assert!(err.to_string().contains("plain"), "got: {err}"); + assert_eq!( + r.workers_for_mode(&ModelId("m".into()), WorkerMode::Plain) + .len(), + 1, + ); + assert!(r.get(&WorkerId("p".into())).is_none()); + } + + /// Concurrent registrations from `manager::register_one` race against + /// each other: each spawned task calls `add_with_cb` in parallel, and + /// the validate-then-insert sequence inside that method is **not** + /// atomic. Two threads adding workers of conflicting modes for the + /// same model can both pass the existing-workers check (each sees an + /// empty pool) and both proceed to insert, leaving the registry in a + /// mixed PD+plain state — exactly the corruption the + /// `MixedPdAndPlain` check is supposed to prevent. + /// + /// Invariant we pin: for every model, the resulting pool must be + /// EITHER all-Plain OR all-PD, never a mix. We don't care which + /// "winner" mode is selected — the racing manager already serialises + /// per-WorkerId so it's the cross-id case that needs atomicity here. + #[test] + fn concurrent_conflicting_modes_never_produce_mixed_pool() { + use std::sync::Arc; + use std::sync::Barrier; + use std::thread; + + // All threads target one shared model so every `add_with_cb` + // racer contends on the same `workers_for("m")` slot — that's + // what makes the read-validate-write window of one thread + // overlap with another's mutate. An earlier variant spread the + // load across 4 models and did not reliably reproduce the bug + // (per-slot contention was diluted to ~N/4 threads). 200 + // iterations × 16 threads triggers the race within the first + // few iterations on the author's machine; post-fix the + // invariant must hold across every iteration. + const N_THREADS: usize = 16; + const ITER: usize = 200; + + for iter in 0..ITER { + let r = Arc::new(WorkerRegistry::default()); + let barrier = Arc::new(Barrier::new(N_THREADS)); + let mut handles = Vec::with_capacity(N_THREADS); + for t in 0..N_THREADS { + let r = Arc::clone(&r); + let barrier = Arc::clone(&barrier); + // Half the threads register Plain workers, half register + // Prefill, all on the same model. With a non-atomic + // validate→write inside `add_with_cb`, a Plain and a + // Prefill thread both see an empty pool and both + // succeed. + let mode = if t % 2 == 0 { + WorkerMode::Plain + } else { + WorkerMode::Prefill + }; + let id = format!("iter{iter}-t{t}"); + handles.push(thread::spawn(move || { + barrier.wait(); + let _ = r.add(spec(&id, mode, &["m"])); + })); + } + for h in handles { + h.join().unwrap(); + } + + // Invariant check: model is single-mode. + let model = ModelId("m".into()); + let plain = r.workers_for_mode(&model, WorkerMode::Plain).len(); + let prefill = r.workers_for_mode(&model, WorkerMode::Prefill).len(); + let decode = r.workers_for_mode(&model, WorkerMode::Decode).len(); + let pd = prefill + decode; + assert!( + plain == 0 || pd == 0, + "iter {iter}: registry holds a mixed pool — \ + plain={plain}, prefill={prefill}, decode={decode}. \ + The MixedPdAndPlain check in `add_with_cb` is not atomic \ + across concurrent callers.", + ); + // Sanity: the first thread to take the lock must succeed + // (no peer exists yet). Defends against a degenerate "fix" + // that satisfies the single-mode invariant by silently + // rejecting every add. + assert!( + plain + pd >= 1, + "iter {iter}: no workers were registered — \ + the lock or mixed-mode check is starving every caller.", + ); + } + } +} diff --git a/experimental/sgl-router/src/workers/worker.rs b/experimental/sgl-router/src/workers/worker.rs new file mode 100644 index 000000000000..a33a29a8362e --- /dev/null +++ b/experimental/sgl-router/src/workers/worker.rs @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::discovery::{ModelId, WorkerId, WorkerMode}; +use crate::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; +use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; +use std::sync::Arc; + +/// Parse a host from a worker URL. Matches SMG's `worker_builder.rs` +/// fallback chain: parse as-is, retry with `http://` prefix if missing, +/// fall back to `"localhost"` if both fail. The fallback is defensive — +/// discovery code should never emit an unparsable URL — but a panic +/// here would crash the whole router on a single bad config entry. +fn parse_bootstrap_host(url: &str) -> String { + if let Ok(parsed) = url::Url::parse(url) { + if let Some(h) = parsed.host_str() { + return h.to_string(); + } + } + if !url.contains("://") { + if let Ok(parsed) = url::Url::parse(&format!("http://{url}")) { + if let Some(h) = parsed.host_str() { + return h.to_string(); + } + } + } + tracing::warn!( + worker_url = %url, + "Failed to parse worker URL for bootstrap_host; defaulting to 'localhost'" + ); + "localhost".to_string() +} + +/// RAII guard that increments `active_requests` on construction and +/// decrements on drop. Obtain via [`Worker::load_guard`]. +/// +/// `#[must_use]`: a statement-form call like `worker.load_guard();` would +/// drop the guard on the same line, so the counter would never see the +/// in-flight request. The compile-time warning catches that misuse. +#[must_use = "LoadGuard must be held for the request's lifetime; dropping it immediately decrements active_requests"] +pub struct LoadGuard { + counter: Arc, +} + +impl LoadGuard { + pub(crate) fn new(counter: Arc) -> Self { + counter.fetch_add(1, Ordering::Relaxed); + Self { counter } + } +} + +impl Drop for LoadGuard { + fn drop(&mut self) { + self.counter.fetch_sub(1, Ordering::Relaxed); + } +} + +impl WorkerMode { + fn as_u8(self) -> u8 { + match self { + WorkerMode::Plain => 0, + WorkerMode::Prefill => 1, + WorkerMode::Decode => 2, + } + } + + /// Inverse of [`Self::as_u8`]. The only writers of the underlying + /// `AtomicU8` are `as_u8`-derived values, so any out-of-range byte + /// indicates memory corruption or a stale store from an + /// incompatible build — fail loudly rather than silently mislabel + /// the worker as `Decode`. + fn from_u8(v: u8) -> Self { + match v { + 0 => WorkerMode::Plain, + 1 => WorkerMode::Prefill, + 2 => WorkerMode::Decode, + other => unreachable!("invalid WorkerMode discriminant {other}"), + } + } +} + +pub struct Worker { + pub id: WorkerId, + pub url: String, + /// Interior-mutable mode so `ModeChanged` can update in place without + /// dropping the Worker (which would reset `active_requests` + breaker). + mode: AtomicU8, + pub model_ids: Vec, + pub breaker: Arc, + pub active_requests: Arc, + /// Hostname parsed from `url` at construction time and cached. + /// Used as the `bootstrap_host` field on PD-disagg requests so the + /// prefill engine can match incoming KV-transfer requests from + /// decode peers. Falls back to `"localhost"` if the URL fails to + /// parse — a misconfigured worker will fail the prefill request + /// downstream rather than panic here. + bootstrap_host: String, + /// SGLang bootstrap server port for prefill workers (`None` for + /// decode and plain). Set via `--disaggregation-bootstrap-port` at + /// worker startup; carried from `WorkerSpec`. + bootstrap_port: Option, +} + +impl Worker { + pub fn new(spec: crate::discovery::WorkerSpec) -> Self { + Self::with_cb_config(spec, None) + } + + /// Construct a worker with an explicit circuit-breaker configuration. + /// Pass `None` to use the default config (threshold = 3, cool_down = 30 s). + pub fn with_cb_config( + spec: crate::discovery::WorkerSpec, + cb: Option, + ) -> Self { + let breaker = match cb { + Some(cfg) => Arc::new(CircuitBreaker::with_config(cfg)), + None => Arc::new(CircuitBreaker::new()), + }; + let bootstrap_host = parse_bootstrap_host(&spec.url); + Self { + id: spec.id, + url: spec.url, + mode: AtomicU8::new(spec.mode.as_u8()), + model_ids: spec.model_ids, + breaker, + active_requests: Arc::new(AtomicUsize::new(0)), + bootstrap_host, + bootstrap_port: spec.bootstrap_port, + } + } + + /// Hostname carried on PD-disagg request bodies as `bootstrap_host`. + pub fn bootstrap_host(&self) -> &str { + &self.bootstrap_host + } + + /// SGLang bootstrap server port. `None` for decode / plain workers. + pub fn bootstrap_port(&self) -> Option { + self.bootstrap_port + } + + /// Returns the current [`WorkerMode`] of this worker. + /// + /// Uses `Relaxed` ordering: mode changes are rare discovery events and do + /// not need to synchronise with any other memory access. + pub fn mode(&self) -> WorkerMode { + WorkerMode::from_u8(self.mode.load(Ordering::Relaxed)) + } + + /// Update the worker's mode in place. + /// + /// Preserves `active_requests` and `breaker` state — the same `Arc` + /// identity survives the mode transition. + pub fn set_mode(&self, m: WorkerMode) { + self.mode.store(m.as_u8(), Ordering::Relaxed); + } + + pub fn active_load(&self) -> usize { + self.active_requests.load(Ordering::Relaxed) + } + + /// Returns a RAII guard that increments `active_requests` now and + /// decrements when the guard is dropped. + pub fn load_guard(&self) -> LoadGuard { + LoadGuard::new(self.active_requests.clone()) + } +} + +impl std::fmt::Debug for Worker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Worker") + .field("id", &self.id) + .field("url", &self.url) + .field("mode", &self.mode()) + .field("active_load", &self.active_load()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; + + #[test] + fn load_guard_increments_and_decrements() { + let w = Worker::new(WorkerSpec { + id: WorkerId("w".into()), + url: "http://x".into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("m".into())], + bootstrap_port: None, + }); + assert_eq!(w.active_load(), 0); + let g = w.load_guard(); + assert_eq!(w.active_load(), 1); + let g2 = w.load_guard(); + assert_eq!(w.active_load(), 2); + drop(g); + assert_eq!(w.active_load(), 1); + drop(g2); + assert_eq!(w.active_load(), 0); + } + + #[test] + fn mode_accessor_round_trips_all_variants() { + for m in [WorkerMode::Plain, WorkerMode::Prefill, WorkerMode::Decode] { + let w = Worker::new(WorkerSpec { + id: WorkerId("w".into()), + url: "http://x".into(), + mode: m, + model_ids: vec![], + bootstrap_port: None, + }); + assert_eq!(w.mode(), m); + } + } + + #[test] + fn set_mode_updates_in_place() { + let w = Worker::new(WorkerSpec { + id: WorkerId("w".into()), + url: "http://x".into(), + mode: WorkerMode::Prefill, + model_ids: vec![], + bootstrap_port: None, + }); + assert_eq!(w.mode(), WorkerMode::Prefill); + w.set_mode(WorkerMode::Decode); + assert_eq!(w.mode(), WorkerMode::Decode); + w.set_mode(WorkerMode::Plain); + assert_eq!(w.mode(), WorkerMode::Plain); + } + + #[test] + fn bootstrap_port_returns_spec_value_for_prefill() { + let w = Worker::new(WorkerSpec { + id: WorkerId("p1".into()), + url: "http://10.0.0.1:30000".into(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("m".into())], + bootstrap_port: Some(8997), + }); + assert_eq!(w.bootstrap_port(), Some(8997)); + } + + #[test] + fn bootstrap_port_defaults_to_none() { + let w = Worker::new(WorkerSpec { + id: WorkerId("w".into()), + url: "http://10.0.0.1:30000".into(), + mode: WorkerMode::Plain, + model_ids: vec![], + bootstrap_port: None, + }); + assert_eq!(w.bootstrap_port(), None); + } + + #[test] + fn bootstrap_host_parses_ipv4_from_url() { + let w = Worker::new(WorkerSpec { + id: WorkerId("p1".into()), + url: "http://10.0.0.1:30000".into(), + mode: WorkerMode::Prefill, + model_ids: vec![], + bootstrap_port: Some(8997), + }); + assert_eq!(w.bootstrap_host(), "10.0.0.1"); + } + + #[test] + fn bootstrap_host_parses_dns_name_from_url() { + let w = Worker::new(WorkerSpec { + id: WorkerId("p1".into()), + url: "http://prefill-0.svc.cluster.local:30000".into(), + mode: WorkerMode::Prefill, + model_ids: vec![], + bootstrap_port: Some(8997), + }); + assert_eq!(w.bootstrap_host(), "prefill-0.svc.cluster.local"); + } + + #[test] + fn bootstrap_host_falls_back_to_localhost_for_unparsable_url() { + // An empty / invalid URL is not expected from discovery, but the + // accessor must return a usable string rather than panic — the + // prefill worker will reject the request body-side if the host + // really is unreachable. + let w = Worker::new(WorkerSpec { + id: WorkerId("p1".into()), + url: "not a url".into(), + mode: WorkerMode::Prefill, + model_ids: vec![], + bootstrap_port: Some(8997), + }); + assert_eq!(w.bootstrap_host(), "localhost"); + } +} diff --git a/experimental/sgl-router/tests/component/discovery/mod.rs b/experimental/sgl-router/tests/component/discovery/mod.rs new file mode 100644 index 000000000000..f09d04fce595 --- /dev/null +++ b/experimental/sgl-router/tests/component/discovery/mod.rs @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +mod static_urls; diff --git a/experimental/sgl-router/tests/component/discovery/static_urls.rs b/experimental/sgl-router/tests/component/discovery/static_urls.rs new file mode 100644 index 000000000000..8097eac70190 --- /dev/null +++ b/experimental/sgl-router/tests/component/discovery/static_urls.rs @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use sgl_router::config::StaticUrlsDiscoveryConfig; +use sgl_router::discovery::{DiscoveryEvent, WorkerMode}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; + +#[tokio::test] +async fn emits_one_added_per_url_with_plain_seed() { + let cfg = StaticUrlsDiscoveryConfig { + urls: vec!["http://x:30000".into(), "http://y:30000".into()], + }; + let (tx, mut rx) = mpsc::channel(16); + let _h = sgl_router::discovery::static_urls::spawn(cfg, tx) + .await + .unwrap(); + + let mut seen = std::collections::HashSet::new(); + for _ in 0..2 { + let event = tokio::time::timeout(Duration::from_secs(2), rx.recv()) + .await + .unwrap() + .unwrap(); + match event { + DiscoveryEvent::Added(spec) => { + // mode / model_ids / bootstrap_port are seeded as Plain/empty/None; + // the worker manager fills them from /server_info post-discovery. + assert_eq!(spec.mode, WorkerMode::Plain); + assert!(spec.model_ids.is_empty()); + assert_eq!(spec.bootstrap_port, None); + // The URL doubles as the worker id — strings already have to + // be unique (rejected at config-load otherwise). + assert_eq!(spec.id.0, spec.url); + seen.insert(spec.url); + } + other => panic!("unexpected event: {other:?}"), + } + } + assert_eq!( + seen, + ["http://x:30000".to_string(), "http://y:30000".to_string()].into(), + ); +} + +/// Single-URL list — the common dev deployment shape. The producer +/// emits exactly one event and then parks until the receiver is +/// dropped. Earlier versions exited as soon as fan-out completed, +/// which tripped `server::supervisor::supervise_critical_tasks` → +/// `mark_unready` → `/readyz` 503; the lib-side +/// `stays_alive_after_fanout_until_receiver_dropped` pins that +/// invariant in isolation, while this test pins the same contract +/// through the public `spawn` entry point used by the binary. +#[tokio::test] +async fn emits_one_event_and_parks_until_receiver_dropped() { + let cfg = StaticUrlsDiscoveryConfig { + urls: vec!["http://x:30000".into()], + }; + let (tx, mut rx) = mpsc::channel(16); + let h = sgl_router::discovery::static_urls::spawn(cfg, tx) + .await + .unwrap(); + let event = rx.recv().await.unwrap(); + assert!(matches!(event, DiscoveryEvent::Added(_))); + assert!(rx.try_recv().is_err(), "exactly one event expected"); + + // Drop the receiver → producer's `tx.closed()` resolves → task + // exits cleanly. + drop(rx); + tokio::time::timeout(Duration::from_secs(2), h) + .await + .expect("static_urls task should exit after receiver is dropped") + .expect("join handle should not panic"); +} + +/// Spin up a fake worker that advertises +/// `disaggregation_mode = "prefill"` + `disaggregation_bootstrap_port`, +/// pipe it through `spawn_discovery` (StaticUrls backend) into +/// `manager::run_with_config`, and assert the worker lands in the +/// registry with `WorkerMode::Prefill` + the disclosed port. +/// +/// This is the load-bearing end-to-end assertion for the refactor's +/// central claim — "prefill, decode, and plain workers can all appear +/// in the same `urls` list and end up classified correctly" — exercised +/// against the full discovery → introspect → registry pipeline rather +/// than just the in-isolation `register_one` unit test. +#[tokio::test] +async fn static_urls_pd_role_resolved_end_to_end() { + use axum::{routing::get, Json, Router}; + use serde_json::json; + use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ObservabilityConfig, + ProxyConfig, ServerConfig, + }; + use sgl_router::discovery::{spawn_discovery, WorkerId}; + use sgl_router::workers::{manager, WorkerRegistry}; + use tokio::net::TcpListener; + use tokio::sync::oneshot; + + // Fake worker advertising a prefill role + bootstrap port. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let url = format!("http://127.0.0.1:{port}"); + let app = Router::new().route( + "/server_info", + get(|| async { + Json(json!({ + "served_model_name": "tiny", + "disaggregation_mode": "prefill", + "disaggregation_bootstrap_port": 8998, + })) + }), + ); + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }) + .await; + }); + + let cfg = Config { + server: ServerConfig { + host: "127.0.0.1".into(), + port: 0, + }, + observability: ObservabilityConfig::default(), + models: vec![], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec![url.clone()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + }; + + let registry = Arc::new(WorkerRegistry::default()); + let (event_rx, _disc) = spawn_discovery(&cfg).await.unwrap(); + let _mgr = tokio::spawn(manager::run_with_config( + event_rx, + registry.clone(), + Some(Arc::new(cfg)), + None, + None, + )); + + let id = WorkerId(url); + let resolved = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Some(w) = registry.get(&id) { + if w.mode() == WorkerMode::Prefill && w.bootstrap_port() == Some(8998) { + return true; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + resolved.is_ok(), + "expected mode=Prefill bootstrap_port=Some(8998); got {:?}", + registry.get(&id).map(|w| (w.mode(), w.bootstrap_port())) + ); + + let _ = shutdown_tx.send(()); +} diff --git a/experimental/sgl-router/tests/component/health/circuit_breaker.rs b/experimental/sgl-router/tests/component/health/circuit_breaker.rs new file mode 100644 index 000000000000..7969918ac6e3 --- /dev/null +++ b/experimental/sgl-router/tests/component/health/circuit_breaker.rs @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; +use std::time::Duration; + +fn cb() -> CircuitBreaker { + CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(3).unwrap(), + cool_down: Duration::from_millis(100), + }) +} + +#[test] +fn starts_closed_and_allows() { + let b = cb(); + assert!(b.allow()); +} + +#[test] +fn three_failures_open_the_breaker() { + let b = cb(); + b.record_failure(); + b.record_failure(); + assert!(b.allow(), "still closed before threshold"); + b.record_failure(); + assert!(!b.allow(), "open after threshold reached"); +} + +#[test] +fn intermittent_success_resets_failure_count() { + let b = cb(); + b.record_failure(); + b.record_failure(); + b.record_success(); // resets + b.record_failure(); + b.record_failure(); + assert!(b.allow(), "should still be closed (2 failures since reset)"); +} + +#[tokio::test(start_paused = true)] +async fn open_breaker_recovers_via_half_open() { + let b = cb(); + b.record_failure(); + b.record_failure(); + b.record_failure(); + assert!(!b.allow()); + + // Wait past cool_down. + tokio::time::advance(Duration::from_millis(150)).await; + + // Half-open: allow one probe. + assert!(b.allow(), "half-open allows the probe"); + // While half-open, further allow() calls should reject (only one probe in flight). + assert!(!b.allow(), "half-open rejects second probe"); + + // Probe succeeded. + b.record_success(); + assert!(b.allow(), "closed after successful probe"); + assert!(b.allow(), "stays closed"); +} + +#[tokio::test(start_paused = true)] +async fn half_open_failure_reopens() { + let b = cb(); + b.record_failure(); + b.record_failure(); + b.record_failure(); + + tokio::time::advance(Duration::from_millis(150)).await; + assert!(b.allow(), "half-open admit"); + b.record_failure(); + // Back to Open. + assert!(!b.allow(), "back to open"); +} + +#[tokio::test(start_paused = true)] +async fn would_allow_is_non_mutating_past_cool_down() { + // `would_allow()` answers "would `allow()` return true right now?" without + // claiming a probe slot. Enumeration / filtering paths (e.g. + // `WorkerRegistry::healthy_workers_for`) call it to inspect breakers + // without disturbing state. + let b = cb(); + b.record_failure(); + b.record_failure(); + b.record_failure(); + assert!(!b.allow(), "open after threshold"); + + tokio::time::advance(Duration::from_millis(150)).await; + + // Repeated would_allow() returns true and leaves state untouched. + assert!(b.would_allow()); + assert!(b.would_allow()); + assert!(b.would_allow()); + + // The first allow() claims the half-open probe. + assert!(b.allow(), "allow() admits the probe"); + // The probe is in flight — subsequent allow() (and would_allow()) reject. + assert!(!b.allow(), "only one probe in flight"); + assert!(!b.would_allow(), "would_allow() agrees: no slot available"); +} + +#[tokio::test(start_paused = true)] +async fn enumeration_then_dispatch_preserves_probe() { + // Regression for the bug where `healthy_workers_for` filtered with + // mutating `allow()`. Once would_allow() is the filter, an enumeration + // pass over many workers must not steal the probe slot from the one + // worker that actually gets dispatched to. + let b = cb(); + b.record_failure(); + b.record_failure(); + b.record_failure(); + tokio::time::advance(Duration::from_millis(150)).await; + + // Imagine 3 workers; enumeration filters each with would_allow(). + for _ in 0..3 { + assert!(b.would_allow(), "filter sees the worker as available"); + } + + // Now the policy picks ONE worker and dispatch claims the probe. + assert!(b.allow(), "dispatch on the picked worker succeeds"); +} + +#[test] +fn would_allow_in_closed_state_is_true_and_non_mutating() { + let b = cb(); + for _ in 0..5 { + assert!(b.would_allow()); + } + // And allow() should still work afterwards. + assert!(b.allow()); +} + +#[tokio::test(start_paused = true)] +async fn open_breaker_recovery_is_not_delayed_by_continued_failures() { + // Regression: previously, record_failure on an already-Open breaker + // refreshed opened_at, so a failure storm pinned the breaker open + // forever. Now the cool_down is measured from first-open. + let b = CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(3).unwrap(), + cool_down: Duration::from_millis(100), + }); + // Open it. + b.record_failure(); + b.record_failure(); + b.record_failure(); + assert!(!b.allow(), "breaker should be open"); + + // Advance halfway through cool_down, then record more failures. + tokio::time::advance(Duration::from_millis(50)).await; + b.record_failure(); + b.record_failure(); + b.record_failure(); + + // Advance just past the original cool_down. + tokio::time::advance(Duration::from_millis(60)).await; + + // We're past the original cool_down → HalfOpen. + assert!( + b.allow(), + "breaker should be half-open after cool_down from first-open" + ); +} diff --git a/experimental/sgl-router/tests/component/health/mod.rs b/experimental/sgl-router/tests/component/health/mod.rs new file mode 100644 index 000000000000..65a441502899 --- /dev/null +++ b/experimental/sgl-router/tests/component/health/mod.rs @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +mod circuit_breaker; diff --git a/experimental/sgl-router/tests/component/main.rs b/experimental/sgl-router/tests/component/main.rs new file mode 100644 index 000000000000..d2a8461f9624 --- /dev/null +++ b/experimental/sgl-router/tests/component/main.rs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Component-scope integration tests. +//! +//! Each submodule exercises a single library component (policy, registry, +//! discovery, health, tokenizer) via the crate's public API. None of these +//! tests spin up the full HTTP router; for those see `tests/proxy/`. + +mod discovery; +mod health; +mod policies; +mod tokenizer; +mod workers; diff --git a/experimental/sgl-router/tests/component/policies/cache_aware_zmq.rs b/experimental/sgl-router/tests/component/policies/cache_aware_zmq.rs new file mode 100644 index 000000000000..ba5b0fd34ac9 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/cache_aware_zmq.rs @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! E2E test for the cache-aware-zmq policy. +//! +//! Drives a real ZMQ PUB socket → `KvEventIndex` subscriber pipeline → +//! `HashTree` → `CacheAwareZmqPolicy::select`. Verifies that an event +//! published by one worker's PUB causes subsequent selection to route +//! to that worker (cache-aware affinity). +//! +//! API constraint: the subscriber registry builds endpoints as +//! `tcp://{host}:{port_base + dp_rank}` where `port_base` is in the +//! per-worker `EventConfig`. Both mock workers below share +//! `127.0.0.1` as host, so both subscribe to the same PUB socket and +//! both end up indexed in the tree. The tiebreak (lowest active_load) +//! picks the worker we want; same shape as the SMG version of this +//! test. + +use std::sync::Arc; +use std::time::Duration; + +use zeromq::SocketSend; + +use sgl_router::config::CacheAwareConfig; +use sgl_router::config::{ActiveLoadConfig, ProxyConfig}; + +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::cache_aware_zmq::CacheAwareZmqPolicy; +use sgl_router::policies::kv_events::{compute_block_hashes, discovery::EventConfig, KvEventIndex}; +use sgl_router::policies::{Policy, SelectionContext}; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::Worker; + +use super::zmq_helpers::{ + build_multipart, encode_block_stored_event, encode_event_batch, make_pub_bound, +}; + +fn build_worker(url: &str, model: &str) -> Arc { + Arc::new(Worker::new(WorkerSpec { + id: WorkerId(url.into()), + url: url.into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId(model.into())], + bootstrap_port: None, + })) +} + +/// E2E: real PUB socket publishes a `BlockStored` for worker A's +/// hash chain. The `CacheAwareZmqPolicy`'s shared `KvEventIndex` +/// receives it, applies it to the tree, and the next `select` call +/// picks worker A. +/// +/// Both workers share `127.0.0.1` as host so both subscribers connect +/// to the same PUB and both get indexed under their KvWorkerIds — the +/// same shape as the SMG e2e test. We tie-break on min-load: worker B +/// is bumped above worker A so the matched-worker pick prefers A. +#[tokio::test] +async fn zmq_indexer_routes_to_publishing_worker_e2e() { + let model_id = ModelId("tiny".into()); + + // 1. Tokenizer registry — use the in-tree tiny fixture. + let cfg = sgl_router::config::Config { + server: sgl_router::config::ServerConfig { + host: "0".into(), + port: 0, + }, + observability: Default::default(), + models: vec![sgl_router::config::ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: sgl_router::config::PolicyKind::CacheAwareZmq, + circuit_breaker: None, + cache_aware: None, + }], + discovery: sgl_router::config::DiscoveryConfig { + backend: sgl_router::config::DiscoveryBackend::StaticUrls( + sgl_router::config::StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }, + ), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + }; + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + + // 2. Bind a real PUB socket on an OS-assigned port. + let (mut pub_a, port) = make_pub_bound().await; + + // 3. Compute the hash chain for the routing prompt. + let text = "hello world hello world hello world"; + let tok = tokenizers.get("tiny").unwrap(); + let token_ids = sgl_router::tokenizer::adapter::encode(&tok, text).unwrap(); + let block_size = 4u32; + let hashes = compute_block_hashes(&token_ids, block_size as usize); + assert!(!hashes.is_empty(), "tiny tokenizer must yield ≥1 block"); + + // 4. Build the KvEventIndex + policy. The policy holds an + // Arc that the index also owns; events the index + // receives mutate the same tree the policy reads. + let kv_index = KvEventIndex::new(); + // Mirror what `KvEventIndex::add_worker` would do in production: seed + // the oracle with the worker-reported page_size before any cache + // lookup happens. The integration path calls `add_worker` further + // down, but here we want the policy to know `block_size` immediately. + let block_size_oracle = kv_index.block_size_oracle(); + block_size_oracle.try_set(block_size).unwrap(); + let policy = CacheAwareZmqPolicy::new( + CacheAwareConfig { + cache_threshold: 0.0, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }, + kv_index.tree(), + Arc::clone(&tokenizers), + block_size_oracle, + ); + + // 5. Register two workers. They share `127.0.0.1` so both + // subscribers connect to the same PUB; preresolved EventConfig + // points at the bound port. + let url_a = "http://127.0.0.1:30000"; + let url_b = "http://127.0.0.1:30001"; + let preresolved = EventConfig { + host: "127.0.0.1".to_string(), + port_base: port, + topic: String::new(), + block_size, + dp_size: 1, + }; + kv_index.add_worker(url_a, Some(preresolved.clone())).await; + kv_index.add_worker(url_b, Some(preresolved)).await; + + // SUB sockets take a moment to handshake. The polling loop below + // soaks up any extra latency; this is just a publish-before-SUB + // guard. + tokio::time::sleep(Duration::from_millis(150)).await; + + // 6. Publish a BlockStored event for the routing prompt's chain. + let event_bytes = encode_block_stored_event(&hashes, None, &token_ids, block_size); + let payload = encode_event_batch(0.0, vec![event_bytes], Some(0)); + pub_a + .send(build_multipart(1, payload)) + .await + .expect("send block-stored event"); + + // 7. Bump worker B's load so the tie-break picks A among matched + // workers. The bump stays below balance_abs_threshold so the + // imbalance fast-path does not skip cache-aware selection. + // Bind the guards to a Vec held for the rest of the test scope + // so the counter stays > 0 through the polling loop. + let w_a = build_worker(url_a, "tiny"); + let w_b = build_worker(url_b, "tiny"); + let _b_load: Vec<_> = (0..3).map(|_| w_b.load_guard()).collect(); + let workers = vec![Arc::clone(&w_a), Arc::clone(&w_b)]; + + // 8. Drive select until the event has been applied. The pipeline is + // asynchronous (publish → SUB recv → mpsc → pump → tree); a + // polling loop is less flaky than a fixed sleep. + let body = serde_json::to_vec(&serde_json::json!({"prompt": text})).unwrap(); + let ctx = SelectionContext::new(&model_id, Some(&body)); + + let start = std::time::Instant::now(); + let mut chose_a = false; + while start.elapsed() < Duration::from_secs(3) { + if let Some(w) = policy.select(&workers, &ctx) { + if w.url == url_a { + chose_a = true; + break; + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!( + chose_a, + "policy did not route to publishing worker A within timeout", + ); + + // 9. Shutdown cleanly. + let r = tokio::time::timeout(Duration::from_secs(2), kv_index.shutdown()).await; + assert!(r.is_ok(), "kv_index shutdown should not hang"); +} diff --git a/experimental/sgl-router/tests/component/policies/kv_events_hash_parity.rs b/experimental/sgl-router/tests/component/policies/kv_events_hash_parity.rs new file mode 100644 index 000000000000..a0520d141849 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/kv_events_hash_parity.rs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Cross-implementation parity test for the KV-event block-hash algorithm. +//! +//! The Rust implementation at `src/policies/kv_events/hash.rs` must produce +//! the same i64 block hashes as SGLang's `radix_cache::RadixKey.hash_page` +//! followed by `hash_str_to_int64`. Hard-coded `cross_language_golden_*` +//! values inside `hash.rs` are correct but brittle: if either side's +//! algorithm changes, the comments don't get regenerated and the tests +//! pass with stale expectations. +//! +//! This test consumes a fixture produced by +//! `tests/scripts/generate_kv_events_hash_parity.py`, which replicates the +//! SGLang algorithm verbatim (see the script's docstring for authority +//! pointers). CI regenerates the fixture (see +//! `.github/workflows/pr-test-sgl-router.yml`) and diffs against the +//! committed file; this test asserts the Rust implementation matches +//! whatever fixture is checked in. + +use serde::Deserialize; +use sgl_router::policies::kv_events::compute_block_hashes; +use std::path::PathBuf; + +#[derive(Debug, Deserialize)] +struct ParityCase { + name: String, + tokens: Vec, + block_size: usize, + expected_i64_hashes: Vec, +} + +fn fixture_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join("kv_events_hash_parity.json") +} + +fn load_cases() -> Vec { + let path = fixture_path(); + let bytes = std::fs::read(&path) + .unwrap_or_else(|e| panic!("read parity fixture {}: {e}", path.display())); + serde_json::from_slice(&bytes) + .unwrap_or_else(|e| panic!("decode parity fixture {}: {e}", path.display())) +} + +#[test] +fn fixture_is_non_empty() { + let cases = load_cases(); + assert!( + !cases.is_empty(), + "kv_events_hash_parity.json is empty — run \ + tests/scripts/generate_kv_events_hash_parity.py", + ); +} + +/// Drives every case in the fixture through `compute_block_hashes` and +/// asserts equality with the Python-derived expectation. +#[test] +fn rust_block_hashes_match_python_radix_cache() { + for case in load_cases() { + // block_size of 0 is rejected by `compute_block_hashes` with a + // panic; the Python generator also rejects it. The fixture + // doesn't include a 0 case, so unwrap is safe. + let block_size = std::num::NonZeroUsize::new(case.block_size) + .unwrap_or_else(|| panic!("case {} has block_size=0 which is invalid", case.name)); + let got = compute_block_hashes(&case.tokens, block_size.get()); + assert_eq!( + got, case.expected_i64_hashes, + "case {}: tokens={:?} block_size={} — Rust produced {:?}, fixture says {:?}", + case.name, case.tokens, case.block_size, got, case.expected_i64_hashes, + ); + } +} diff --git a/experimental/sgl-router/tests/component/policies/kv_events_tree_concurrent.rs b/experimental/sgl-router/tests/component/policies/kv_events_tree_concurrent.rs new file mode 100644 index 000000000000..b0afee7cb770 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/kv_events_tree_concurrent.rs @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Concurrent-mutation stress test for `HashTree`. +//! +//! The 19 inline tests in `policies::kv_events::tree` are all +//! single-threaded. Under production load, multiple worker subscribers +//! drive `insert` / `remove` / `clear_worker` against the same tree from +//! tokio worker threads while the chat handler simultaneously calls +//! `match_prefix` from many concurrent requests. +//! +//! The tree is documented as taking a write-lock for mutations and a +//! read-lock for `match_prefix`; this test exercises that contract under +//! heavy contention to catch: +//! +//! * Deadlocks between the reverse index and the arena's RwLock. +//! * Logical races where a removed worker still appears in the reverse +//! index (or vice versa). +//! * Panics from a node arena being mutated mid-read. +//! +//! After the storm settles, the tree must be self-consistent: every +//! worker that was fully cleared must be absent from every node's worker +//! set, and `node_count()` must converge to zero. + +use std::sync::Arc; +use std::thread; + +use sgl_router::policies::kv_events::{HashTree, KvWorkerId}; + +fn worker(i: usize) -> KvWorkerId { + KvWorkerId { + url: format!("http://w{i}:30000"), + dp_rank: 0, + } +} + +/// 8 mutator threads × 200 ops + 4 reader threads × 500 match queries. +/// Each mutator inserts a chain, queries it, then clears the worker; the +/// invariant is that after every thread joins, the tree is empty (every +/// worker was cleared) and no thread panicked. +#[test] +fn tree_survives_concurrent_inserts_removes_and_matches() { + let tree = Arc::new(HashTree::new()); + + let mut handles = Vec::new(); + + for tid in 0..8 { + let tree = tree.clone(); + handles.push(thread::spawn(move || { + let w = worker(tid); + for round in 0..200_u64 { + // Each round uses a fresh chain so different mutators + // don't trample each other's nodes — we want contention + // on the lock, not contention on the keys (those are + // covered by the single-threaded reinsert/remove tests). + let chain: Vec = (0..4) + .map(|i| ((tid as i64) << 32) | ((round as i64) << 8) | i as i64) + .collect(); + tree.insert(&w, None, &chain); + + let m = tree.match_prefix(None, &chain); + assert!( + m.matched_blocks <= chain.len(), + "match must never exceed query length", + ); + + // Half the rounds use remove(&chain); the rest use + // clear_worker — both must leave a consistent tree. + if round % 2 == 0 { + tree.remove(&w, &chain); + } else { + tree.clear_worker(&w); + } + } + // Final blanket clear in case the last iteration used `remove` + // on only part of the chain. + tree.clear_worker(&w); + })); + } + + for tid in 0..4 { + let tree = tree.clone(); + handles.push(thread::spawn(move || { + for round in 0..500_u64 { + let probe: Vec = (0..3) + .map(|i| ((tid as i64) << 40) | ((round as i64) << 8) | i as i64) + .collect(); + // Readers must never block-walk and must never panic. + let _ = tree.match_prefix(None, &probe); + } + })); + } + + for h in handles { + h.join() + .expect("worker thread panicked under concurrent load"); + } + + assert_eq!( + tree.node_count(), + 0, + "tree must be empty after every worker was cleared; \ + residual nodes indicate a missed clear_worker path", + ); + + // The arena and the reverse index must agree: zero non-root nodes + // means zero `by_hash` entries. A bug that prunes the arena but not + // the reverse index would leak memory and corrupt future inserts; + // this assertion turns that into an immediate test failure. + assert_eq!( + tree.reverse_index_size(), + 0, + "by_hash reverse index must be empty when no non-root nodes remain", + ); +} + +/// A mutator races `clear_worker` against a reader that is mid-`match_prefix` +/// on a deep chain. The reader must never see a partially-mutated tree +/// (no panic, no double-counted workers in the result set). +#[test] +fn match_prefix_is_consistent_with_concurrent_clear() { + let tree = Arc::new(HashTree::new()); + let w = worker(0); + let chain: Vec = (0..32).map(|i| 1_000 + i).collect(); + + // Pre-populate so the reader has something to walk. + tree.insert(&w, None, &chain); + + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + + let mutator = { + let tree = tree.clone(); + let stop = stop.clone(); + let w = w.clone(); + let chain = chain.clone(); + thread::spawn(move || { + let mut round = 0u64; + while !stop.load(std::sync::atomic::Ordering::Relaxed) { + if round.is_multiple_of(2) { + tree.clear_worker(&w); + } else { + tree.insert(&w, None, &chain); + } + round += 1; + } + }) + }; + + for _ in 0..2_000 { + let m = tree.match_prefix(None, &chain); + // Either the worker was present (matched_blocks == chain.len(), + // workers set contains w) or it was cleared mid-walk (matched_blocks + // == 0 OR matched_blocks > 0 with empty workers if the chain is + // partially present). Whichever — the result must be internally + // consistent. + if m.matched_blocks == chain.len() { + assert!( + m.workers.contains(&w), + "full match must include worker; got {:?}", + m.workers, + ); + } + } + + stop.store(true, std::sync::atomic::Ordering::Relaxed); + mutator.join().unwrap(); +} diff --git a/experimental/sgl-router/tests/component/policies/kv_events_two_subscribers.rs b/experimental/sgl-router/tests/component/policies/kv_events_two_subscribers.rs new file mode 100644 index 000000000000..e603978fc733 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/kv_events_two_subscribers.rs @@ -0,0 +1,305 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Two independent `KvEventIndex` instances subscribed to the same PUB +//! socket — the in-process surrogate for "two router replicas watching +//! the same SGLang worker's KV publisher." +//! +//! Why this matters: sgl-router v1 explicitly omits multi-replica state +//! sync (deferred to v2 in the slim-design spec). Independent ZMQ +//! subscription is the **only** mechanism by which two routers arrive at +//! a consistent cache-aware view today. If a future change accidentally +//! degraded that property — e.g. a worker that only allows one subscriber, +//! a switch from PUB/SUB to PUSH/PULL, or a teardown bug that drops +//! events to one of N subscribers — this test fails loudly. +//! +//! Property pinned: after publishing N `BlockStored` events, both trees +//! report the same `match_prefix(matched_blocks, workers)` for the +//! published key, and an unpublished key remains absent from both. + +use std::sync::Arc; +use std::time::Duration; + +use zeromq::SocketSend; + +use sgl_router::policies::kv_events::discovery::EventConfig; +use sgl_router::policies::kv_events::{compute_block_hashes, KvEventIndex, KvWorkerId}; + +use super::zmq_helpers::{ + build_multipart, encode_block_stored_event, encode_event_batch, make_pub_bound, +}; + +#[tokio::test] +async fn two_independent_subscribers_converge_to_same_tree_state() { + // 1. One PUB socket — the worker. Both router surrogates connect to it. + let (mut publisher, port) = make_pub_bound().await; + let worker_url = "http://127.0.0.1:30000"; + let block_size = 4u32; + let cfg = EventConfig { + host: "127.0.0.1".into(), + port_base: port, + topic: String::new(), + block_size, + dp_size: 1, + }; + + // 2. Two independent router-process surrogates, each with its own + // `KvEventIndex` (own tree, own subscriber, own pump task). Both + // call `add_worker` with the same preresolved `EventConfig` — the + // same shape production wires through `WorkerManager`. + let router_a = KvEventIndex::new(); + let router_b = KvEventIndex::new(); + router_a.add_worker(worker_url, Some(cfg.clone())).await; + router_b.add_worker(worker_url, Some(cfg.clone())).await; + + // SUB-side handshake settle. Publishing before the subscribers + // finish their initial connect loses messages in PUB/SUB semantics; + // the polling loop below would then never converge. + tokio::time::sleep(Duration::from_millis(200)).await; + + // 3. Publish a deterministic, multi-block event chain. + let tokens: Vec = (0..16).collect(); + let hashes = compute_block_hashes(&tokens, block_size as usize); + assert!( + hashes.len() >= 3, + "test needs ≥3 blocks; got {}", + hashes.len() + ); + let event_bytes = encode_block_stored_event(&hashes, None, &tokens, block_size); + let payload = encode_event_batch(0.0, vec![event_bytes], Some(0)); + publisher + .send(build_multipart(1, payload)) + .await + .expect("publish BlockStored"); + + // 4. Poll both trees until both report the FULL chain matched. The + // SUB→mpsc→pump→tree pipeline is async; loopback delivery is + // reliable but not instantaneous. + let target = hashes.len(); + let key = KvWorkerId { + url: worker_url.into(), + dp_rank: 0, + }; + let start = std::time::Instant::now(); + loop { + let ma = router_a.tree().match_prefix(None, &hashes); + let mb = router_b.tree().match_prefix(None, &hashes); + let converged = ma.matched_blocks == target + && mb.matched_blocks == target + && ma.workers.contains(&key) + && mb.workers.contains(&key); + if converged { + // Both trees agree on count AND on the worker that holds the + // prefix. This is what the cache-aware-zmq policy reads to + // pick a worker; both routers picking the same key here + // means they would route the same prompt to the same worker. + assert_eq!( + ma.matched_blocks, mb.matched_blocks, + "subscribers disagreed on matched_blocks", + ); + assert_eq!( + ma.workers, mb.workers, + "subscribers disagreed on worker set", + ); + break; + } + if start.elapsed() > Duration::from_secs(3) { + panic!( + "subscribers did not converge within 3s: \ + router_a={{matched={}, workers={:?}}}, \ + router_b={{matched={}, workers={:?}}}, target={target}", + ma.matched_blocks, ma.workers, mb.matched_blocks, mb.workers, + ); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + + // 5. Negative leg: a key that was never published must not appear in + // either tree. Guards against a future bug where one subscriber + // accidentally inherits another's state (shared static, etc.). + let unseen: Vec = vec![999_999_999_001, 999_999_999_002, 999_999_999_003]; + let na = router_a.tree().match_prefix(None, &unseen); + let nb = router_b.tree().match_prefix(None, &unseen); + assert_eq!(na.matched_blocks, 0, "router_a leaked unpublished key"); + assert_eq!(nb.matched_blocks, 0, "router_b leaked unpublished key"); + + // 6. Both shutdowns must complete cleanly — no hang from the second + // subscriber holding a reference to a shared resource. The first + // drains under a generous ceiling (worker thread joins, mpsc + // receiver drop); the second has nothing left to wait on and + // must complete promptly. A slow second shutdown indicates the + // two subscribers were sharing a resource that serialized them. + let r = tokio::time::timeout(Duration::from_secs(2), Arc::clone(&router_a).shutdown()).await; + assert!(r.is_ok(), "router_a shutdown hung"); + + let t = std::time::Instant::now(); + let r = tokio::time::timeout(Duration::from_secs(2), Arc::clone(&router_b).shutdown()).await; + assert!(r.is_ok(), "router_b shutdown hung"); + let elapsed = t.elapsed(); + assert!( + elapsed < Duration::from_millis(100), + "router_b shutdown after router_a drained took {elapsed:?}; \ + expected <100ms (no shared-resource contention)", + ); +} + +/// Two PUB sockets (two workers) + two `KvEventIndex` instances (two +/// routers), each subscribed to **both** publishers. This is the real +/// v1 HA shape: each router replica fans out subscriptions across the +/// worker pool and merges every publisher's `BlockStored` stream into +/// its own tree. The companion 1-PUB test above only verifies broadcast +/// fan-out; this test verifies the per-worker attribution stays correct +/// when events arrive from multiple sources concurrently. +/// +/// Property pinned: after publishing prefix `X` on `pub_x` and prefix +/// `Y` on `pub_y`, both trees report +/// * `match_prefix(X) = {full, workers={worker_x}}` +/// * `match_prefix(Y) = {full, workers={worker_y}}` +/// with no cross-attribution (worker_x must NOT appear in match(Y)). +/// A regression that wires both subscribers to the same internal +/// channel — or that mis-keys events by their arrival socket rather +/// than their announced worker URL — would surface here as cross- +/// contamination of the worker sets. +#[tokio::test] +async fn two_subscribers_merge_events_from_two_publishers() { + let (mut pub_x, port_x) = make_pub_bound().await; + let (mut pub_y, port_y) = make_pub_bound().await; + let worker_x = "http://127.0.0.1:30001"; + let worker_y = "http://127.0.0.1:30002"; + let block_size = 4u32; + let cfg_x = EventConfig { + host: "127.0.0.1".into(), + port_base: port_x, + topic: String::new(), + block_size, + dp_size: 1, + }; + let cfg_y = EventConfig { + host: "127.0.0.1".into(), + port_base: port_y, + topic: String::new(), + block_size, + dp_size: 1, + }; + + // Both routers subscribe to BOTH workers — the production fan-out. + let router_a = KvEventIndex::new(); + let router_b = KvEventIndex::new(); + router_a.add_worker(worker_x, Some(cfg_x.clone())).await; + router_a.add_worker(worker_y, Some(cfg_y.clone())).await; + router_b.add_worker(worker_x, Some(cfg_x.clone())).await; + router_b.add_worker(worker_y, Some(cfg_y.clone())).await; + + // Four SUB→PUB handshakes need to settle before publishing; missed + // SUBSCRIBE frames lose messages forever in PUB/SUB semantics. + tokio::time::sleep(Duration::from_millis(200)).await; + + // Two non-overlapping token streams → two distinct hash chains. The + // gap between them (0..16 vs 1000..1016) keeps `compute_block_hashes` + // outputs disjoint so a cross-attribution bug can't be masked by + // hash collision. + let tokens_x: Vec = (0..16).collect(); + let tokens_y: Vec = (1000..1016).collect(); + let hashes_x = compute_block_hashes(&tokens_x, block_size as usize); + let hashes_y = compute_block_hashes(&tokens_y, block_size as usize); + assert!(hashes_x.len() >= 3 && hashes_y.len() >= 3); + + let payload_x = encode_event_batch( + 0.0, + vec![encode_block_stored_event( + &hashes_x, None, &tokens_x, block_size, + )], + Some(0), + ); + let payload_y = encode_event_batch( + 0.0, + vec![encode_block_stored_event( + &hashes_y, None, &tokens_y, block_size, + )], + Some(0), + ); + pub_x + .send(build_multipart(1, payload_x)) + .await + .expect("publish on pub_x"); + pub_y + .send(build_multipart(1, payload_y)) + .await + .expect("publish on pub_y"); + + let key_x = KvWorkerId { + url: worker_x.into(), + dp_rank: 0, + }; + let key_y = KvWorkerId { + url: worker_y.into(), + dp_rank: 0, + }; + let target_x = hashes_x.len(); + let target_y = hashes_y.len(); + + let start = std::time::Instant::now(); + loop { + let ax = router_a.tree().match_prefix(None, &hashes_x); + let ay = router_a.tree().match_prefix(None, &hashes_y); + let bx = router_b.tree().match_prefix(None, &hashes_x); + let by = router_b.tree().match_prefix(None, &hashes_y); + let converged = ax.matched_blocks == target_x + && ay.matched_blocks == target_y + && bx.matched_blocks == target_x + && by.matched_blocks == target_y + && ax.workers.contains(&key_x) + && ay.workers.contains(&key_y) + && bx.workers.contains(&key_x) + && by.workers.contains(&key_y); + if converged { + // Negative attribution: prefix X must not be attributed to + // worker_y in either tree, and vice versa. A regression that + // keyed events by arriving socket rather than announced + // worker URL would set BOTH worker keys on each prefix. + assert!( + !ax.workers.contains(&key_y), + "router_a cross-attributed worker_y to prefix X: {:?}", + ax.workers, + ); + assert!( + !ay.workers.contains(&key_x), + "router_a cross-attributed worker_x to prefix Y: {:?}", + ay.workers, + ); + assert!( + !bx.workers.contains(&key_y), + "router_b cross-attributed worker_y to prefix X: {:?}", + bx.workers, + ); + assert!( + !by.workers.contains(&key_x), + "router_b cross-attributed worker_x to prefix Y: {:?}", + by.workers, + ); + break; + } + if start.elapsed() > Duration::from_secs(3) { + panic!( + "trees did not converge within 3s:\n \ + router_a: X={{matched={}, workers={:?}}}, Y={{matched={}, workers={:?}}}\n \ + router_b: X={{matched={}, workers={:?}}}, Y={{matched={}, workers={:?}}}\n \ + targets: X={target_x}, Y={target_y}", + ax.matched_blocks, + ax.workers, + ay.matched_blocks, + ay.workers, + bx.matched_blocks, + bx.workers, + by.matched_blocks, + by.workers, + ); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + + let r = tokio::time::timeout(Duration::from_secs(2), Arc::clone(&router_a).shutdown()).await; + assert!(r.is_ok(), "router_a shutdown hung"); + let r = tokio::time::timeout(Duration::from_secs(2), Arc::clone(&router_b).shutdown()).await; + assert!(r.is_ok(), "router_b shutdown hung"); +} diff --git a/experimental/sgl-router/tests/component/policies/mod.rs b/experimental/sgl-router/tests/component/policies/mod.rs new file mode 100644 index 000000000000..6837a9b64405 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/mod.rs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +mod zmq_helpers; + +mod cache_aware_zmq; +mod kv_events_hash_parity; +mod kv_events_tree_concurrent; +mod kv_events_two_subscribers; +mod power_of_two; +mod round_robin; diff --git a/experimental/sgl-router/tests/component/policies/power_of_two.rs b/experimental/sgl-router/tests/component/policies/power_of_two.rs new file mode 100644 index 000000000000..21178e372a00 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/power_of_two.rs @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::power_of_two::PowerOfTwoChoicesPolicy; +use sgl_router::policies::{Policy, SelectionContext}; +use sgl_router::workers::Worker; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +fn worker(id: &str) -> Arc { + Arc::new(Worker::new(WorkerSpec { + id: WorkerId(id.into()), + url: format!("http://{id}"), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("m".into())], + bootstrap_port: None, + })) +} + +#[test] +fn selects_lower_load() { + let a = worker("a"); + let b = worker("b"); + a.active_requests.store(10, Ordering::Relaxed); + b.active_requests.store(2, Ordering::Relaxed); + let p = PowerOfTwoChoicesPolicy::new(); + let ws = vec![a.clone(), b.clone()]; + let model_id = ModelId("m".into()); + let ctx = SelectionContext::new(&model_id, None); + let chosen = p.select(&ws, &ctx).unwrap(); + assert_eq!(chosen.id.0, "b"); +} + +#[test] +fn distribution_skews_to_lower_load() { + // With 3 workers and one heavily loaded, the loaded one should win + // significantly less than 1/3 of selections. + let workers = vec![worker("a"), worker("b"), worker("c")]; + workers[2].active_requests.store(100, Ordering::Relaxed); // c is loaded + + let p = PowerOfTwoChoicesPolicy::new(); + let model_id = ModelId("m".into()); + let ctx = SelectionContext::new(&model_id, None); + let mut counts = std::collections::HashMap::new(); + for _ in 0..1000 { + let w = p.select(&workers, &ctx).unwrap(); + *counts.entry(w.id.0.clone()).or_insert(0) += 1; + } + let c_picks = *counts.get("c").unwrap_or(&0); + assert!( + c_picks < 200, + "loaded worker should be picked < 20% of the time, got {c_picks}" + ); +} + +#[test] +fn empty_returns_none() { + let p = PowerOfTwoChoicesPolicy::new(); + let ws: Vec> = vec![]; + let model_id = ModelId("m".into()); + let ctx = SelectionContext::new(&model_id, None); + assert!(p.select(&ws, &ctx).is_none()); +} + +#[test] +fn single_worker_returns_it() { + let p = PowerOfTwoChoicesPolicy::new(); + let ws = vec![worker("only")]; + let model_id = ModelId("m".into()); + let ctx = SelectionContext::new(&model_id, None); + assert_eq!(p.select(&ws, &ctx).unwrap().id.0, "only"); +} diff --git a/experimental/sgl-router/tests/component/policies/round_robin.rs b/experimental/sgl-router/tests/component/policies/round_robin.rs new file mode 100644 index 000000000000..c28762bd4960 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/round_robin.rs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::round_robin::RoundRobinPolicy; +use sgl_router::policies::{Policy, SelectionContext}; +use sgl_router::workers::Worker; +use std::sync::Arc; + +fn worker(id: &str) -> Arc { + Arc::new(Worker::new(WorkerSpec { + id: WorkerId(id.into()), + url: format!("http://{id}"), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("m".into())], + bootstrap_port: None, + })) +} + +#[test] +fn cycles_through_workers() { + let p = RoundRobinPolicy::new(); + let ws = vec![worker("a"), worker("b"), worker("c")]; + let model_id = ModelId("m".into()); + let ctx = SelectionContext::new(&model_id, None); + let picks: Vec<_> = (0..6) + .filter_map(|_| p.select(&ws, &ctx)) + .map(|w| w.id.0.clone()) + .collect(); + assert_eq!(picks, vec!["a", "b", "c", "a", "b", "c"]); +} + +#[test] +fn empty_pool_returns_none() { + let p = RoundRobinPolicy::new(); + let ws: Vec> = vec![]; + let model_id = ModelId("m".into()); + let ctx = SelectionContext::new(&model_id, None); + assert!(p.select(&ws, &ctx).is_none()); +} + +#[test] +fn distribution_across_100_calls() { + let p = RoundRobinPolicy::new(); + let ws = vec![worker("a"), worker("b"), worker("c")]; + let model_id = ModelId("m".into()); + let ctx = SelectionContext::new(&model_id, None); + let mut counts = std::collections::HashMap::new(); + for _ in 0..99 { + let w = p.select(&ws, &ctx).unwrap(); + *counts.entry(w.id.0.clone()).or_insert(0) += 1; + } + assert_eq!(counts["a"], 33); + assert_eq!(counts["b"], 33); + assert_eq!(counts["c"], 33); +} diff --git a/experimental/sgl-router/tests/component/policies/zmq_helpers.rs b/experimental/sgl-router/tests/component/policies/zmq_helpers.rs new file mode 100644 index 000000000000..ec36eb2f3aa1 --- /dev/null +++ b/experimental/sgl-router/tests/component/policies/zmq_helpers.rs @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared ZMQ wire-format helpers for the `policies::kv_events` component +//! tests. Encodes events in the same msgspec layout SGLang emits, builds +//! the two-frame `[seq, payload]` ZMQ message a real publisher sends, and +//! binds a loopback PUB socket on an OS-assigned port. + +#![allow(dead_code)] + +use bytes::Bytes; +use rmp::encode as mp; +use zeromq::{Endpoint, PubSocket, Socket, ZmqMessage}; + +/// Bind a PUB socket to an OS-assigned 127.0.0.1 port. Returns +/// `(socket, port)`. +pub async fn make_pub_bound() -> (PubSocket, u16) { + let mut sock = PubSocket::new(); + let endpoint = sock + .bind("tcp://127.0.0.1:0") + .await + .expect("bind PUB socket"); + let port = match endpoint { + Endpoint::Tcp(_, p) => p, + other => panic!("unexpected endpoint: {other:?}"), + }; + (sock, port) +} + +/// Encode a single `BlockStored` event in the wire format msgspec +/// emits. Layout: `["BlockStored", block_hashes, parent, token_ids, +/// block_size, lora_id, medium]`. +pub fn encode_block_stored_event( + block_hashes: &[i64], + parent: Option, + token_ids: &[u32], + block_size: u32, +) -> Vec { + let mut buf = Vec::new(); + mp::write_array_len(&mut buf, 7).unwrap(); + mp::write_str(&mut buf, "BlockStored").unwrap(); + mp::write_array_len(&mut buf, block_hashes.len() as u32).unwrap(); + for v in block_hashes { + mp::write_sint(&mut buf, *v).unwrap(); + } + match parent { + Some(v) => { + mp::write_sint(&mut buf, v).unwrap(); + } + None => mp::write_nil(&mut buf).unwrap(), + } + mp::write_array_len(&mut buf, token_ids.len() as u32).unwrap(); + for v in token_ids { + mp::write_uint(&mut buf, *v as u64).unwrap(); + } + mp::write_uint(&mut buf, block_size as u64).unwrap(); + mp::write_nil(&mut buf).unwrap(); // lora_id + mp::write_str(&mut buf, "GPU").unwrap(); + buf +} + +/// Wrap one or more pre-encoded events into a KVEventBatch with +/// timestamp + optional dp-rank. +pub fn encode_event_batch(ts: f64, events: Vec>, attn_dp_rank: Option) -> Vec { + let mut buf = Vec::new(); + mp::write_array_len(&mut buf, 3).unwrap(); + mp::write_f64(&mut buf, ts).unwrap(); + mp::write_array_len(&mut buf, events.len() as u32).unwrap(); + for ev in events { + buf.extend_from_slice(&ev); + } + match attn_dp_rank { + Some(v) => { + mp::write_uint(&mut buf, v as u64).unwrap(); + } + None => mp::write_nil(&mut buf).unwrap(), + } + buf +} + +/// Build the two-frame ZMQ message a real KV publisher sends: +/// `[seq (big-endian i64), payload]`. +pub fn build_multipart(seq: i64, payload: Vec) -> ZmqMessage { + let mut msg = ZmqMessage::from(Bytes::new()); + msg.push_back(Bytes::copy_from_slice(&seq.to_be_bytes())); + msg.push_back(Bytes::from(payload)); + msg +} diff --git a/experimental/sgl-router/tests/component/tokenizer/mod.rs b/experimental/sgl-router/tests/component/tokenizer/mod.rs new file mode 100644 index 000000000000..09c530d860e9 --- /dev/null +++ b/experimental/sgl-router/tests/component/tokenizer/mod.rs @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +mod parity; diff --git a/experimental/sgl-router/tests/component/tokenizer/parity.rs b/experimental/sgl-router/tests/component/tokenizer/parity.rs new file mode 100644 index 000000000000..274f6cc5ed39 --- /dev/null +++ b/experimental/sgl-router/tests/component/tokenizer/parity.rs @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Bit-parity check: dynamo-tokenizers must produce the same token_ids as +//! SGLang's reference (transformers.AutoTokenizer) for every (model, shape) +//! fixture. Any drift is a regression. +//! +//! ## Running +//! +//! `cargo test --release --test component tokenizer::parity` runs the test. +//! +//! Each fixture cell needs the model's `tokenizer.json` on disk; the test +//! looks in the local HuggingFace cache (`HF_HOME` or `~/.cache/huggingface`). +//! Cells whose snapshot isn't cached are skipped (with a warning); cells +//! whose snapshot IS cached are asserted bit-identical. +//! +//! Locally, when no fixtures can be checked (fresh cache) the test emits a +//! warning and passes — useful for contributors without the model snapshots. +//! In CI (`SGLANG_IS_IN_CI=true`) the same condition is a hard failure: a +//! parity matrix that validates nothing is worse than no test at all, since +//! it gives a false sense of coverage. The e2e HTTP tokenize test remains +//! the authoritative live-model parity gate, but this matrix must actually +//! run against cached snapshots when present in CI. +//! +//! ## Regenerating fixtures +//! +//! Run `tests/scripts/generate_parity_fixtures.py` after changing a prompt +//! shape or adding a model, then commit the new JSON. + +use serde::Deserialize; +use std::path::PathBuf; + +#[derive(Deserialize)] +struct Fixture { + model_id: String, + shape: String, + prompt_text: String, + expected_token_ids: Vec, + #[allow(dead_code)] + skip_special_tokens: bool, +} + +fn fixture_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/tokenizer_parity") +} + +/// Resolve a model's tokenizer.json file from the local HF cache. +/// +/// Strategy: +/// 1. Check HF_HOME env var, or default to ~/.cache/huggingface +/// 2. Look for models--/snapshots//tokenizer.json +/// 3. Return None if not found — the test cell is skipped. +fn resolve_tokenizer_path(model_id: &str) -> Option { + let hf_home = std::env::var("HF_HOME") + .ok() + .map(PathBuf::from) + .or_else(|| dirs::home_dir().map(|h| h.join(".cache/huggingface")))?; + let safe = model_id.replace('/', "--"); + let candidate = hf_home.join("hub").join(format!("models--{safe}")); + if !candidate.exists() { + return None; + } + let snapshots = candidate.join("snapshots"); + let snap = std::fs::read_dir(&snapshots).ok()?.next()?.ok()?.path(); + let tj = snap.join("tokenizer.json"); + tj.exists().then_some(tj) +} + +/// Parity matrix: dynamo-tokenizers vs. transformers.AutoTokenizer. +/// +/// Skips cells whose tokenizer.json isn't in the local HF cache. See +/// module-level docs. +#[test] +fn parity_matrix() { + let mut checked = 0; + let mut skipped = vec![]; + for model_dir in std::fs::read_dir(fixture_root()).unwrap() { + let model_dir = model_dir.unwrap().path(); + if !model_dir.is_dir() { + continue; + } + for shape_file in std::fs::read_dir(&model_dir).unwrap() { + let p = shape_file.unwrap().path(); + if p.extension().and_then(|s| s.to_str()) != Some("json") { + continue; + } + let raw = std::fs::read_to_string(&p).unwrap(); + let f: Fixture = + serde_json::from_str(&raw).unwrap_or_else(|e| panic!("parse {}: {e}", p.display())); + let Some(tp) = resolve_tokenizer_path(&f.model_id) else { + skipped.push((f.model_id.clone(), f.shape.clone())); + continue; + }; + let tok = sgl_router::tokenizer::adapter::load(tp.to_str().unwrap()).unwrap(); + let ids = sgl_router::tokenizer::adapter::encode(&tok, &f.prompt_text).unwrap(); + assert_eq!( + ids, f.expected_token_ids, + "DRIFT on {}/{}", + f.model_id, f.shape + ); + checked += 1; + } + } + let expected = std::fs::read_dir(fixture_root()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.path().is_dir()) + .map(|e| { + std::fs::read_dir(e.path()) + .unwrap() + .filter_map(|f| f.ok()) + .filter(|f| f.path().extension().and_then(|s| s.to_str()) == Some("json")) + .count() + }) + .sum::(); + assert_eq!( + checked + skipped.len(), + expected, + "expected {expected} fixtures, found {}", + checked + skipped.len() + ); + if checked == 0 { + let families: Vec = skipped + .iter() + .map(|(m, _)| m.clone()) + .collect::>() + .into_iter() + .collect(); + let msg = format!( + "parity_matrix: no fixtures could be checked — HF cache empty? skipped {} cells \ + across model families: [{}]. The e2e HTTP tokenize test remains the \ + authoritative live-model parity gate.", + skipped.len(), + families.join(", "), + ); + if std::env::var("SGLANG_IS_IN_CI").as_deref() == Ok("true") { + panic!( + "{msg}\n\nThis is a hard failure in CI: a parity test that validates zero \ + cells provides no coverage. Either pre-populate the HF cache for these \ + model families on the runner, or remove the parity test." + ); + } + eprintln!("{msg}"); + } else { + eprintln!( + "parity: {checked} cells passed, {} skipped (no HF snapshot)", + skipped.len() + ); + } +} diff --git a/experimental/sgl-router/tests/component/workers/concurrent_state.rs b/experimental/sgl-router/tests/component/workers/concurrent_state.rs new file mode 100644 index 000000000000..3cf6eeaf63e3 --- /dev/null +++ b/experimental/sgl-router/tests/component/workers/concurrent_state.rs @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Concurrent-state invariants for the worker/registry/breaker layer. +//! +//! These tests stress the lock-free / single-Mutex paths that production +//! traffic exercises in parallel: many requests calling `breaker.allow()`, +//! many discovery events racing with workers_for() reads, and LoadGuard +//! lifecycles under panics. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; +use sgl_router::workers::{Worker, WorkerRegistry}; + +/// HalfOpen state must admit at most one probe at a time even under high +/// concurrency. N threads race `allow()` when the breaker is HalfOpen; the +/// invariant is that exactly one observes `true` (the probe holder); the +/// rest see `false` because `probe_in_flight` is already set. +#[tokio::test(start_paused = true)] +async fn breaker_half_open_admits_only_one_probe_concurrently() { + let cb = Arc::new(CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(1).unwrap(), + cool_down: Duration::from_millis(50), + })); + + // Trip into Open. + cb.record_failure(); + assert!(!cb.allow(), "must be Open immediately after a failure"); + + // Advance the paused clock past cool_down so the next `allow()` will + // attempt the Open → HalfOpen transition. + tokio::time::advance(Duration::from_millis(60)).await; + + let admitted = Arc::new(AtomicUsize::new(0)); + let mut handles = Vec::new(); + for _ in 0..32 { + let cb = cb.clone(); + let admitted = admitted.clone(); + handles.push(tokio::spawn(async move { + if cb.allow() { + admitted.fetch_add(1, Ordering::Relaxed); + } + })); + } + for h in handles { + h.await.unwrap(); + } + assert_eq!( + admitted.load(Ordering::Relaxed), + 1, + "exactly one probe must be admitted in HalfOpen", + ); +} + +/// Concurrent `add_with_cb` (upsert) and `remove` from many threads on the +/// same WorkerId must not panic, must not deadlock, and must leave a +/// consistent index — `workers_for(model)` may return 0 or 1 worker, but +/// must never resolve to a worker that has been removed. +#[test] +fn registry_concurrent_add_remove_keeps_indexes_consistent() { + let r = Arc::new(WorkerRegistry::default()); + let model = ModelId("m".into()); + + let mut handles = Vec::new(); + for i in 0..8 { + let r = r.clone(); + let model = model.clone(); + handles.push(std::thread::spawn(move || { + for _ in 0..200 { + let _ = r.add(WorkerSpec { + id: WorkerId(format!("w{i}")), + url: format!("http://w{i}:30000"), + mode: WorkerMode::Plain, + model_ids: vec![model.clone()], + bootstrap_port: None, + }); + let snapshot = r.workers_for(&model); + for w in &snapshot { + // Cross-index invariant: an entry surfaced via + // `by_model[m]` must come from a Worker whose own + // `model_ids` includes `m`. An earlier version of + // this assertion checked `w.id.0.starts_with('w')`, + // which is a tautology — every id is `w0..w7` by + // construction — and a regression where `by_model` + // pointed at the wrong Worker (e.g., a stale entry + // left after an upsert that should have cleared its + // by_model membership for the dropped model) would + // pass silently. We can't `re-get by_id and ptr_eq` + // because a concurrent remove can drop the by_id + // entry between the two reads — `Arc` keeps the + // Worker alive on our side but the index map is + // gone. The model-membership claim, however, is a + // property of the Arc itself and stays stable. + assert!( + w.model_ids.contains(&model), + "cross-index drift: by_model[{model:?}] surfaced \ + {:?} whose own model_ids = {:?}", + w.id, + w.model_ids, + ); + } + r.remove(&WorkerId(format!("w{i}"))); + } + })); + } + for h in handles { + h.join().unwrap(); + } + + // After every thread finishes, every removed worker must really be gone. + assert!( + r.workers_for(&model).is_empty(), + "registry must be empty after all threads finished their add/remove cycles", + ); +} + +/// `LoadGuard` must decrement the counter during a panic-unwind, not just +/// on a normal scope exit. Rust's RAII contract via `Drop` covers this, +/// but a future refactor (e.g. adding a manual decrement on a non-panic +/// path) could silently regress it. This test pins the invariant. +#[test] +fn load_guard_decrements_on_panic_unwind() { + let w = Arc::new(Worker::new(WorkerSpec { + id: WorkerId("w".into()), + url: "http://x:30000".into(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("m".into())], + bootstrap_port: None, + })); + assert_eq!(w.active_load(), 0); + + let w_inner = w.clone(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _g = w_inner.load_guard(); + assert_eq!(w_inner.active_load(), 1); + panic!("synthetic panic to exercise Drop on unwind"); + })); + assert!(result.is_err(), "the closure must have panicked"); + assert_eq!( + w.active_load(), + 0, + "LoadGuard's Drop must decrement even when the holder panics", + ); +} diff --git a/experimental/sgl-router/tests/component/workers/manager.rs b/experimental/sgl-router/tests/component/workers/manager.rs new file mode 100644 index 000000000000..45b32fc2889f --- /dev/null +++ b/experimental/sgl-router/tests/component/workers/manager.rs @@ -0,0 +1,511 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use axum::{routing::get, Json, Router}; +use serde_json::{json, Value}; +use sgl_router::discovery::{DiscoveryEvent, ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::workers::{manager, WorkerRegistry}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::net::TcpListener; +use tokio::sync::{mpsc, oneshot}; + +/// Spin up a tiny fake worker that returns `body` on `GET /server_info`. +/// Returns the worker base URL and a shutdown channel. +async fn spawn_fake_worker(body: Value) -> (String, oneshot::Sender<()>) { + let body = Arc::new(body); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/server_info", + get(move || { + let body = body.clone(); + async move { Json((*body).clone()) } + }), + ); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = rx.await; + }) + .await; + }); + (format!("http://127.0.0.1:{port}"), tx) +} + +fn spec_for(id: &str, url: &str, mode: WorkerMode) -> WorkerSpec { + // model_ids are intentionally empty: the manager resolves them via + // /server_info introspection. Pre-populating here would lie about + // what discovery backends actually emit. + WorkerSpec { + id: WorkerId(id.into()), + url: url.into(), + mode, + model_ids: Vec::new(), + bootstrap_port: None, + } +} + +#[tokio::test] +async fn manager_processes_added_then_removed() { + let (url_a, _s_a) = spawn_fake_worker(json!({"served_model_name": "m"})).await; + let (url_b, _s_b) = spawn_fake_worker(json!({"served_model_name": "m"})).await; + + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + tx.send(DiscoveryEvent::Added(spec_for( + "w1", + &url_a, + WorkerMode::Plain, + ))) + .await + .unwrap(); + tx.send(DiscoveryEvent::Added(spec_for( + "w2", + &url_b, + WorkerMode::Plain, + ))) + .await + .unwrap(); + + // Give the manager time to drain. + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(registry.workers_for(&ModelId("m".into())).len(), 2); + + tx.send(DiscoveryEvent::Removed { + id: WorkerId("w1".into()), + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(registry.workers_for(&ModelId("m".into())).len(), 1); + + drop(tx); + h.await.unwrap(); +} + +#[tokio::test] +async fn manager_handles_mode_changed() { + let (url, _s) = spawn_fake_worker(json!({"served_model_name": "m"})).await; + + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + tx.send(DiscoveryEvent::Added(spec_for( + "w1", + &url, + WorkerMode::Prefill, + ))) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!( + registry + .workers_for_mode(&ModelId("m".into()), WorkerMode::Prefill) + .len(), + 1 + ); + + tx.send(DiscoveryEvent::ModeChanged { + id: WorkerId("w1".into()), + mode: WorkerMode::Decode, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + registry + .workers_for_mode(&ModelId("m".into()), WorkerMode::Prefill) + .len(), + 0 + ); + assert_eq!( + registry + .workers_for_mode(&ModelId("m".into()), WorkerMode::Decode) + .len(), + 1 + ); + + drop(tx); + h.await.unwrap(); +} + +#[tokio::test] +async fn mode_changed_preserves_active_requests_and_breaker() { + let (url, _s) = spawn_fake_worker(json!({"served_model_name": "m"})).await; + + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + tx.send(DiscoveryEvent::Added(spec_for( + "w1", + &url, + WorkerMode::Prefill, + ))) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + + // Grab a handle, bump active_requests, and open the breaker. + let w = registry.get(&WorkerId("w1".into())).unwrap(); + w.active_requests.fetch_add(5, Ordering::Relaxed); + // Default threshold is 3 — record 10 failures to guarantee Open state. + for _ in 0..10 { + w.breaker.record_failure(); + } + let breaker_open_before = !w.breaker.allow(); + assert!( + breaker_open_before, + "breaker should be open after 10 failures" + ); + + // Flip mode via ModeChanged. + tx.send(DiscoveryEvent::ModeChanged { + id: WorkerId("w1".into()), + mode: WorkerMode::Decode, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // Re-fetch the Worker handle from the registry. + let w_after = registry.get(&WorkerId("w1".into())).unwrap(); + + assert_eq!( + w_after.mode(), + WorkerMode::Decode, + "mode should have flipped to Decode" + ); + assert_eq!( + w_after.active_requests.load(Ordering::Relaxed), + 5, + "active_requests should be preserved across mode change" + ); + assert!( + !w_after.breaker.allow(), + "breaker open state should be preserved across mode change" + ); + + // Critical: the Arc identity must be the same — mutation in place. + assert!( + Arc::ptr_eq(&w, &w_after), + "Worker handle should be the SAME Arc, not a fresh replacement" + ); + + drop(tx); + h.await.unwrap(); +} + +/// An out-of-order `ModeChanged` for a worker the registry does not know +/// about (e.g. a buggy discovery backend reordered `Removed` and +/// `ModeChanged`) must not panic, must not silently log INFO claiming the +/// mode flip happened, and must leave the registry untouched. +#[tokio::test] +async fn manager_handles_orphan_mode_changed_without_panic() { + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + tx.send(DiscoveryEvent::ModeChanged { + id: WorkerId("ghost".into()), + mode: WorkerMode::Decode, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + assert!( + registry.get(&WorkerId("ghost".into())).is_none(), + "an orphan ModeChanged must not create a phantom worker", + ); + assert_eq!( + registry.workers_for(&ModelId("m".into())).len(), + 0, + "registry must be empty after an orphan event", + ); + + drop(tx); + h.await.unwrap(); +} + +/// A `Removed` for an unknown id is a no-op — registry stays empty, manager +/// keeps running. +#[tokio::test] +async fn manager_handles_orphan_removed_without_panic() { + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + tx.send(DiscoveryEvent::Removed { + id: WorkerId("ghost".into()), + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(registry.is_empty()); + + drop(tx); + h.await.unwrap(); +} + +/// Duplicate `Added` for the same id is an upsert — the registry ends up +/// with exactly one worker. The model resolved by /server_info wins on +/// re-add (a different worker may advertise a different served model). +#[tokio::test] +async fn manager_handles_duplicate_added_as_upsert() { + let (url_first, _s_first) = spawn_fake_worker(json!({"served_model_name": "m1"})).await; + let (url_second, _s_second) = spawn_fake_worker(json!({"served_model_name": "m1"})).await; + + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + tx.send(DiscoveryEvent::Added(spec_for( + "w1", + &url_first, + WorkerMode::Plain, + ))) + .await + .unwrap(); + tx.send(DiscoveryEvent::Added(spec_for( + "w1", + &url_second, + WorkerMode::Plain, + ))) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + + assert_eq!( + registry.workers_for(&ModelId("m1".into())).len(), + 1, + "w1 still serves m1 after the second Added", + ); + + drop(tx); + h.await.unwrap(); +} + +/// Spawn a fake worker whose `/server_info` returns `body` only after +/// sleeping for `delay`. Returns the worker URL and a shutdown channel. +async fn spawn_slow_worker(body: Value, delay: Duration) -> (String, oneshot::Sender<()>) { + let body = Arc::new(body); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/server_info", + get(move || { + let body = body.clone(); + async move { + tokio::time::sleep(delay).await; + Json((*body).clone()) + } + }), + ); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = rx.await; + }) + .await; + }); + (format!("http://127.0.0.1:{port}"), tx) +} + +/// Spawn a fake worker that counts each `GET /server_info` hit in the +/// returned `AtomicUsize`. Used to assert the manager makes exactly +/// one round-trip per worker. +async fn spawn_counting_worker(body: Value) -> (String, Arc, oneshot::Sender<()>) { + let body = Arc::new(body); + let counter = Arc::new(AtomicUsize::new(0)); + let counter_clone = counter.clone(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/server_info", + get(move || { + let body = body.clone(); + let counter = counter_clone.clone(); + async move { + counter.fetch_add(1, Ordering::SeqCst); + Json((*body).clone()) + } + }), + ); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + let _ = axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = rx.await; + }) + .await; + }); + (format!("http://127.0.0.1:{port}"), counter, tx) +} + +/// Registration must run in parallel across multiple `Added` events. +/// Each fake worker delays its `/server_info` by 200ms; with sequential +/// processing the manager would take ≥1000ms for 5 workers. We allow +/// up to 600ms (3x the per-fetch delay) as a generous bound that still +/// rejects the sequential implementation. +#[tokio::test] +async fn added_events_run_in_parallel() { + let delay = Duration::from_millis(200); + let n = 5; + let mut workers = Vec::new(); + for _ in 0..n { + workers.push(spawn_slow_worker(json!({"served_model_name": "m"}), delay).await); + } + + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + let start = Instant::now(); + for (i, (url, _s)) in workers.iter().enumerate() { + tx.send(DiscoveryEvent::Added(spec_for( + &format!("w{i}"), + url, + WorkerMode::Plain, + ))) + .await + .unwrap(); + } + let registered = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if registry.workers_for(&ModelId("m".into())).len() == n { + return true; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + let elapsed = start.elapsed(); + assert!(registered.is_ok(), "manager failed to register {n} workers"); + assert!( + elapsed < Duration::from_millis(600), + "registration of {n} workers took {elapsed:?}; sequential per-worker /server_info \ + fetches would take ≥1000ms — parallel spawn is required" + ); + + drop(tx); + h.await.unwrap(); +} + +/// A `Removed` issued while the matching `Added` is still mid-fetch +/// must await the in-flight registration handle before removing. +/// Without that ordering the removal runs first (registry has nothing +/// to remove), then the Added's deferred registry write leaks the +/// worker. +#[tokio::test] +async fn removed_awaits_pending_added() { + let (url, _s) = spawn_slow_worker( + json!({"served_model_name": "m"}), + Duration::from_millis(300), + ) + .await; + + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let h = tokio::spawn(manager::run(rx, registry.clone())); + + tx.send(DiscoveryEvent::Added(spec_for( + "w-slow", + &url, + WorkerMode::Plain, + ))) + .await + .unwrap(); + tx.send(DiscoveryEvent::Removed { + id: WorkerId("w-slow".into()), + }) + .await + .unwrap(); + + // Wait long enough for the Added's /server_info to complete (300ms), + // then assert the worker is gone. If Removed ran before Added's + // registry write, the post-fetch write would leak the entry. + tokio::time::sleep(Duration::from_millis(600)).await; + assert!( + registry.get(&WorkerId("w-slow".into())).is_none(), + "Removed must await the in-flight Added; otherwise the deferred \ + registry write leaks the worker" + ); + + drop(tx); + h.await.unwrap(); +} + +/// The manager must make exactly ONE `/server_info` request per worker. +/// Before this fix the worker manager fetched `served_model_name` and +/// `KvEventIndex::add_worker` fetched the `kv_events` block +/// independently — 2N round-trips for N workers. +#[tokio::test] +async fn manager_emits_single_server_info_fetch_per_worker() { + use sgl_router::policies::kv_events::KvEventIndex; + + let body = json!({ + "served_model_name": "m", + "kv_events": { + "publisher": "zmq", + "endpoint_host": "127.0.0.1", + "endpoint_port_base": 60100, + "topic": "", + "block_size": 64, + "dp_size": 1, + } + }); + let (url, counter, _s) = spawn_counting_worker(body).await; + + let (tx, rx) = mpsc::channel(16); + let registry = Arc::new(WorkerRegistry::default()); + let kv_index = KvEventIndex::new(); + let h = tokio::spawn(manager::run_with_config( + rx, + registry.clone(), + None, + Some(kv_index.clone()), + None, + )); + + tx.send(DiscoveryEvent::Added(spec_for( + "w1", + &url, + WorkerMode::Plain, + ))) + .await + .unwrap(); + + // Wait for both the registry and kv-events index to reflect the worker. + let ready = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if registry.get(&WorkerId("w1".into())).is_some() && kv_index.known_worker_count() == 1 + { + return true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + ready.is_ok(), + "manager did not finish onboarding the worker" + ); + + let hits = counter.load(Ordering::SeqCst); + assert_eq!( + hits, 1, + "manager must fetch /server_info exactly once per worker (got {hits})" + ); + + drop(tx); + h.await.unwrap(); + kv_index.shutdown().await; +} diff --git a/experimental/sgl-router/tests/component/workers/mod.rs b/experimental/sgl-router/tests/component/workers/mod.rs new file mode 100644 index 000000000000..e17217e97f8d --- /dev/null +++ b/experimental/sgl-router/tests/component/workers/mod.rs @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +mod concurrent_state; +mod manager; diff --git a/experimental/sgl-router/tests/e2e/chat_completions/test_two_router_convergence.py b/experimental/sgl-router/tests/e2e/chat_completions/test_two_router_convergence.py new file mode 100644 index 000000000000..34fe9707574f --- /dev/null +++ b/experimental/sgl-router/tests/e2e/chat_completions/test_two_router_convergence.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Content-based cross-router routing test for cache-aware-zmq. + +Two routers + two SGLang workers + one shared model. Each router runs an +independent ``cache_aware_zmq`` policy whose ``KvEventIndex`` subscribes +to **both** workers' KV publishers. + +The test warms each worker with a DIFFERENT prefix DIRECTLY (bypassing +both routers), then sends those prefixes through each router and +asserts that routing follows the prefix CONTENT: ``PREFIX_X`` lands on +the worker holding X, ``PREFIX_Y`` lands on the worker holding Y, on +both routers. + +# Why content-based, not convergence + +An earlier version of this test asserted that both routers converged on +the *same dominant worker* after a one-prefix warmup. That property +sounds like it pins the ZMQ-fan-out contract, but it doesn't: when the +KV-event path is broken (subscribers never opened, e.g. a worker's +``/server_info`` lacks the ``kv_events`` block), ``cache_aware_zmq`` +silently degrades to **min-load** — which, with sequential requests +holding ``active_load`` at zero, picks the same worker deterministically +on every call within a router. Both routers' min-load picks happened to +agree often enough (about half the time, modulo HashSet seed) to make +the convergence assertion pass even when no event ever flowed. + +Content-based routing is uniquely sensitive to the KV-event path. Two +disjoint prefixes warmed on two different workers can only be routed +correctly if the router knows *which worker holds which content* — the +only mechanism that supplies that information is the ``BlockStored`` +event stream. Under min-load fallback, both prefixes route to the same +default worker on each router, so the ``PREFIX_Y → worker_y`` assertion +fails regardless of which worker min-load defaults to. +""" + +from __future__ import annotations + +import re +import time + +import httpx +import pytest +from infra.gateway import Gateway +from infra.model_pool import PASSTHROUGH_CHAT_TEMPLATE_PATH, spawn_worker +from infra.model_specs import get_model_spec + +# Disjoint prefixes — share no common opening text, so block 0 hashes +# differ from the first block onward and each worker's HashTree +# contribution is uniquely identifying. +# +# Length matters: each prefix must span ≥2 SGLang blocks at the default +# block_size of 64 tokens so the worker actually emits BlockStored +# events. Below that, the publisher stays quiet and we'd be testing +# min-load by accident — the exact failure mode this test exists to +# rule out. +_PREFIX_X_BODY = ( + "Apricot bouquet cinnamon dewdrop elderflower fennel garlic " + "hibiscus indigo jasmine kumquat lavender mint nutmeg oregano " + "paprika quince rosemary saffron tarragon. " +) +PREFIX_X = (_PREFIX_X_BODY * 8).strip() + +_PREFIX_Y_BODY = ( + "Zephyr yellow xylophone wombat vortex umbrella thistle saffron " + "quartz peppermint orchid nightshade marigold lemongrass kale " + "juniper iris hyacinth gardenia foxglove. " +) +PREFIX_Y = (_PREFIX_Y_BODY * 8).strip() + + +_REQ_TOTAL_RE = re.compile( + r"^sgl_router_requests_total\{([^}]*)\}\s+(\d+(?:\.\d+)?)\s*$" +) +_LABEL_RE = re.compile(r'(\w+)="([^"]*)"') + + +def _success_counts_by_worker(router_url: str) -> dict[str, int]: + """Scrape ``/metrics`` and return ``{worker_url: success_count}``.""" + r = httpx.get(f"{router_url}/metrics", timeout=5.0) + r.raise_for_status() + counts: dict[str, int] = {} + for line in r.text.splitlines(): + m = _REQ_TOTAL_RE.match(line) + if not m: + continue + labels = dict(_LABEL_RE.findall(m.group(1))) + if labels.get("outcome") != "success": + continue + worker = labels.get("worker_url") + if not worker: + continue + try: + counts[worker] = counts.get(worker, 0) + int(float(m.group(2))) + except ValueError: + continue + return counts + + +def _send_chat(url: str, model_id: str, prompt: str) -> int: + """POST one chat completion; return the HTTP status.""" + r = httpx.post( + f"{url}/v1/chat/completions", + json={ + "model": model_id, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 4, + "stream": False, + }, + timeout=60.0, + ) + return r.status_code + + +def _direct_warm(worker_url: str, model_id: str, prefix: str) -> None: + """Send one ``/v1/chat/completions`` request with ``prefix`` DIRECTLY to a worker. + + The KV-event publisher emits ``BlockStored`` as the request's + prompt blocks commit to that worker's cache; routers subscribed to + the publisher receive the event and add ``(block_hash → worker)`` + entries to their ``HashTree``. The test then exercises those + entries by routing through the router. + + Direct-warming (rather than going through a router) is the load- + bearing detail: routing through a router would itself choose which + worker to populate, so the two workers' HashTree state would no + longer be uniquely identifying. + + Token alignment with the router — ``cache_aware_zmq`` hashes + ``messages[*].content`` RAW (``cache_aware_zmq.rs::extract_prompt_text``) + using ``add_special_tokens=false``. By default SGLang's chat + endpoint would wrap ``prefix`` in the model's chat template before + tokenizing — adding role tags, end-of-turn markers, and a + generation prompt — and the resulting block hashes would never + match what the router computes from raw content. + + The test launches each worker with ``--chat-template + ``: a Jinja template that emits + only ``messages[*].content`` (the same shape the router extracts), + and which combines with Transformers' ``apply_chat_template( + tokenize=True, add_special_tokens=False)`` to produce the same + token stream the router will compute. So warm and route hash the + same blocks via the same endpoint. + """ + r = httpx.post( + f"{worker_url}/v1/chat/completions", + json={ + "model": model_id, + "messages": [{"role": "user", "content": prefix}], + "max_tokens": 4, + "stream": False, + }, + timeout=60.0, + ) + assert ( + r.status_code == 200 + ), f"direct warm to {worker_url} failed: HTTP {r.status_code} {r.text!r}" + + +def _route_through(router_url: str, model_id: str, prompt: str) -> str: + """Send one request through ``router_url``; return which worker handled it. + + Computed by diffing the per-worker success-counter on ``/metrics`` + around the call. Asserts exactly one worker absorbed the request + (no partial counts, no cancellation race). + """ + before = _success_counts_by_worker(router_url) + code = _send_chat(router_url, model_id, prompt) + assert code == 200, f"request to {router_url} failed: HTTP {code}" + after = _success_counts_by_worker(router_url) + deltas = {w: after.get(w, 0) - before.get(w, 0) for w in set(after) | set(before)} + winners = [w for w, d in deltas.items() if d > 0] + assert ( + len(winners) == 1 + ), f"expected exactly one worker delta on {router_url}, got {deltas}" + return winners[0] + + +@pytest.mark.real_gpu +@pytest.mark.slow +def test_two_routers_route_by_prefix_content( + router_binary, # noqa: ARG001 — fixture forces release-binary presence + gpu_allocator, +): + """Each router must route by prefix CONTENT, agreeing across routers. + + With each worker direct-warmed by a different disjoint prefix, the + only way a router can route ``PREFIX_X → worker_x`` AND + ``PREFIX_Y → worker_y`` is by consulting a HashTree populated from + the BlockStored events the workers emit. Min-load fallback (the + failure mode when no SUB socket opened) is content-blind and would + route both prefixes to whichever worker its tiebreaker prefers. + """ + spec = get_model_spec("qwen3-0.6b") + gpus = gpu_allocator.acquire(2) + # Passthrough chat template — see _direct_warm for the rationale. Both + # workers must run with the same template; otherwise their KV blocks + # would hash template-wrapped tokens while the router hashes raw + # content, and every lookup would miss the tree. + worker_chat_template_args = ["--chat-template", PASSTHROUGH_CHAT_TEMPLATE_PATH] + try: + with ( + spawn_worker( + "qwen3-0.6b", + gpu_ids=[gpus[0]], + enable_kv_events=True, + extra_args=worker_chat_template_args, + ) as worker_x, + spawn_worker( + "qwen3-0.6b", + gpu_ids=[gpus[1]], + enable_kv_events=True, + extra_args=worker_chat_template_args, + ) as worker_y, + Gateway() as router_a, + Gateway() as router_b, + ): + worker_urls = [worker_x.url, worker_y.url] + for gw in (router_a, router_b): + gw.start_regular( + model_id=spec["model"], + tokenizer_path=spec["model"], + worker_urls=worker_urls, + policy="cache_aware_zmq", + timeout=120.0, + ) + + # 1. Direct-warm each worker with its own prefix. Must happen + # AFTER both routers have started — ZMQ PUB/SUB doesn't + # replay messages emitted before SUB attaches, so any + # BlockStored event predating subscription is lost and + # the HashTree never sees it. + _direct_warm(worker_x.url, spec["model"], PREFIX_X) + _direct_warm(worker_y.url, spec["model"], PREFIX_Y) + + # 2. Drain the SUB mpsc + pump-apply path. Sub-second under + # loopback ZMQ; 2 s leaves comfortable headroom. + time.sleep(2.0) + + # 3. Content-routing assertion (×4): each prefix must land + # on the worker that holds it, on either router. + # + # The four assertions below are independently strong: + # min-load fallback routes both prefixes on a given + # router to a single default worker, so for ANY broken- + # fan-out scenario at least one of the four fails. + for router, label in ((router_a, "A"), (router_b, "B")): + landed = _route_through(router.base_url, spec["model"], PREFIX_X) + assert landed == worker_x.url, ( + f"router {label}: PREFIX_X must route to worker_x " + f"({worker_x.url}); landed on {landed}. " + f"Likely cause: HashTree is empty — KV-event " + f"subscriber never opened, or BlockStored events " + f"never reached the pump." + ) + landed = _route_through(router.base_url, spec["model"], PREFIX_Y) + assert landed == worker_y.url, ( + f"router {label}: PREFIX_Y must route to worker_y " + f"({worker_y.url}); landed on {landed}. " + f"Likely cause: HashTree is empty — KV-event " + f"subscriber never opened, or BlockStored events " + f"never reached the pump." + ) + finally: + gpu_allocator.release(gpus) diff --git a/experimental/sgl-router/tests/e2e/chat_completions/test_validation.py b/experimental/sgl-router/tests/e2e/chat_completions/test_validation.py new file mode 100644 index 000000000000..9b1983ac719a --- /dev/null +++ b/experimental/sgl-router/tests/e2e/chat_completions/test_validation.py @@ -0,0 +1,98 @@ +"""Basic chat-completions correctness — ported from SMG's +``e2e_test/chat_completions/test_validation.py``, narrowed to the +subset that exercises sgl-router (not SMG's per-message validators). + +The shape: + - single-worker regular-mode router + - non-streaming + streaming chat completion + - assistant message non-empty, role correct, finish_reason set + +These are the smoke tests that run first; if they pass, the heavier +multi-worker acceptance tests are worth running. +""" + +from __future__ import annotations + +import httpx +import pytest +from infra.gateway import Gateway +from infra.model_pool import spawn_worker +from infra.model_specs import get_model_spec + + +@pytest.mark.real_gpu +def test_chat_non_streaming_returns_assistant_message( + router_binary, # noqa: ARG001 + gpu_allocator, +): + gpu = gpu_allocator.acquire(1) + try: + with spawn_worker("qwen3-0.6b", gpu_ids=gpu) as worker: + spec = get_model_spec("qwen3-0.6b") + with Gateway() as gw: + gw.start_regular( + model_id=spec["model"], + tokenizer_path=spec["model"], + worker_urls=[worker.url], + timeout=120.0, + ) + resp = httpx.post( + f"{gw.base_url}/v1/chat/completions", + json={ + "model": spec["model"], + "messages": [{"role": "user", "content": "Say hi."}], + "max_tokens": 16, + "stream": False, + }, + timeout=60.0, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + choice = body["choices"][0] + assert choice["message"]["role"] == "assistant" + assert choice["message"][ + "content" + ], f"empty assistant content: {choice!r}" + assert choice.get("finish_reason"), choice + finally: + gpu_allocator.release(gpu) + + +@pytest.mark.real_gpu +def test_chat_streaming_emits_sse_chunks_with_done( + router_binary, # noqa: ARG001 + gpu_allocator, +): + gpu = gpu_allocator.acquire(1) + try: + with spawn_worker("qwen3-0.6b", gpu_ids=gpu) as worker: + spec = get_model_spec("qwen3-0.6b") + with Gateway() as gw: + gw.start_regular( + model_id=spec["model"], + tokenizer_path=spec["model"], + worker_urls=[worker.url], + timeout=120.0, + ) + chunks: list[str] = [] + with httpx.stream( + "POST", + f"{gw.base_url}/v1/chat/completions", + json={ + "model": spec["model"], + "messages": [{"role": "user", "content": "Say hi."}], + "max_tokens": 16, + "stream": True, + }, + timeout=60.0, + ) as resp: + assert resp.status_code == 200, resp.read().decode() + for line in resp.iter_lines(): + if line.startswith("data:"): + chunks.append(line.strip()) + assert len(chunks) >= 2, f"expected >=2 SSE chunks, got: {chunks}" + assert any( + "[DONE]" in c for c in chunks + ), f"no [DONE] terminator in stream: {chunks}" + finally: + gpu_allocator.release(gpu) diff --git a/experimental/sgl-router/tests/e2e/conftest.py b/experimental/sgl-router/tests/e2e/conftest.py new file mode 100644 index 000000000000..ae5fb2728f7a --- /dev/null +++ b/experimental/sgl-router/tests/e2e/conftest.py @@ -0,0 +1,327 @@ +"""Pytest fixtures for ``experimental/sgl-router/tests/e2e/``. + +Two flavors of fixtures coexist here: + + 1. **Session-scoped smoke fixtures** (``sglang_server`` + ``router``) — + launch ONE SGLang worker + ONE router on fixed ports for the whole + test session. Used by the lightweight ``test_chat_smoke.py`` / + ``test_tokenize_smoke.py`` files. These are the cheap "did the + binary start at all" sanity tests. + + 2. **Per-test multi-worker fixtures** (``router_binary`` + + ``gpu_allocator``) — just enough infra for the acceptance tests in + ``chat_completions/`` to bring up their own multi-worker + topologies. Backed by the ``infra.gateway.Gateway`` and + ``infra.model_pool.spawn_worker`` helpers. + +Both sets share the same release binary; ``SGL_ROUTER_BINARY`` env var +overrides the path for both. +""" + +from __future__ import annotations + +import logging +import os +import signal +import subprocess +import sys +import tempfile +import threading +import time +from collections.abc import Iterator +from pathlib import Path + +import httpx +import pytest + +logger = logging.getLogger(__name__) + +# Make `from infra import gateway, model_pool, model_specs` resolve from +# tests under tests/e2e/ without requiring a sibling `__init__.py` chain. +# Mirrors SMG's e2e_test/conftest.py sys.path setup. +_E2E_DIR = Path(__file__).resolve().parent +if str(_E2E_DIR) not in sys.path: + sys.path.insert(0, str(_E2E_DIR)) + +MODEL = "Qwen/Qwen3-0.6B" +SGLANG_PORT = 30000 +ROUTER_PORT = 8090 + +# Path to the release binary. This file lives at +# `experimental/sgl-router/tests/e2e/conftest.py`, so: +# parent = tests/e2e/ +# parent.parent = tests/ +# parent.parent.parent = experimental/sgl-router/ ← cargo workspace root +# A previous version used `parent.parent / "target"`, which pointed at +# `experimental/sgl-router/tests/target/` and silently broke every +# fixture that tries to launch the router binary (CI's +# `cargo build --release` lands the artifact at +# `experimental/sgl-router/target/release/sgl-router`, not under +# `tests/`). +_SGL_ROUTER_ROOT = Path(__file__).parent.parent.parent +_BINARY = ( + Path(os.environ.get("CARGO_TARGET_DIR", str(_SGL_ROUTER_ROOT / "target"))) + / "release" + / "sgl-router" +) + + +def _wait_http(url: str, timeout: int = 120) -> None: + """Poll *url* until it returns 2xx or raises RuntimeError on timeout.""" + deadline = time.time() + timeout + last_exc: Exception | None = None + while time.time() < deadline: + try: + resp = httpx.get(url, timeout=5) + if resp.status_code < 300: + return + except Exception as exc: # noqa: BLE001 + last_exc = exc + time.sleep(5) + raise RuntimeError( + f"Timed out waiting for {url} after {timeout}s (last error: {last_exc})" + ) + + +@pytest.fixture(scope="session") +def sglang_server(): + """Launch a real SGLang server on port 30000 and wait until healthy.""" + # Stream the server's stdout/stderr to a file rather than capturing + # to subprocess.PIPE. The launch_server startup log is verbose (model + # download, JIT warmup, NCCL init); once a PIPE'd output fills its + # ~64 KB OS buffer with nothing reading it, the SGLang process + # blocks on stdout write and never reaches "Server started" — the + # health probe then times out at 300 s and we have no visibility + # into *why*. A real log file fixes both (no buffer pressure, and + # the file is dumped on failure for triage). + log_path = Path(tempfile.gettempdir()) / f"sglang-server-{SGLANG_PORT}.log" + log_handle = open(log_path, "w", buffering=1) # line-buffered + proc = subprocess.Popen( + [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + MODEL, + "--port", + str(SGLANG_PORT), + "--tp", + "1", + ], + stdout=log_handle, + stderr=subprocess.STDOUT, + ) + + try: + _wait_http(f"http://localhost:{SGLANG_PORT}/health", timeout=300) + except Exception: + # Dump the server log so the operator can see why startup failed + # (model download error, port conflict, OOM, JIT crash, etc.). + proc.send_signal(signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + log_handle.flush() + log_handle.close() + try: + tail = log_path.read_text(errors="replace").splitlines()[-200:] + except OSError: + tail = ["(server log unreadable)"] + logger.error( + "sglang_server fixture failed; last 200 log lines from %s:\n%s", + log_path, + "\n".join(tail), + ) + raise + + yield f"http://localhost:{SGLANG_PORT}" + + proc.send_signal(signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + log_handle.flush() + log_handle.close() + + +def _find_tokenizer_path(model: str) -> str: + """Locate the tokenizer.json for *model* from the local HF Hub cache. + + Falls back to the model string itself (a valid HF Hub repo identifier + that dynamo-tokenizers can resolve at runtime) when the cache is absent. + """ + try: + from huggingface_hub import try_to_load_from_cache # type: ignore[import] + + path = try_to_load_from_cache(model, "tokenizer.json") + if path and Path(path).is_file(): + return str(path) + except Exception: # noqa: BLE001 + pass + # Let dynamo-tokenizers resolve the repo identifier directly. + return model + + +def build_smoke_router_config( + *, + host: str, + port: int, + model: str, + tokenizer_path: str, + sglang_url: str, +) -> str: + """Build the TOML the smoke `router` fixture writes to disk. + + Returns ``main_config_text`` carrying ``[server]``, ``[[models]]``, + and ``[discovery] backend = "static_urls"`` with the worker URL + inline. The Rust ``Config`` struct requires a ``[discovery]`` + section (``DiscoveryConfig`` has no ``#[serde(default)]``) and has + no top-level ``workers`` field. The previous ``static_file`` + backend was replaced by ``static_urls`` (which holds the URL list + inline rather than via a side-car file). + """ + return f"""\ +[server] +host = "{host}" +port = {port} + +[[models]] +id = "{model}" +tokenizer_path = "{tokenizer_path}" + +[discovery] +backend = "static_urls" + +[discovery.static_urls] +urls = ["{sglang_url}"] +""" + + +@pytest.fixture(scope="session") +def router(sglang_server): # noqa: ARG001 (sglang_server must start first) + """Launch sgl-router on port 8090 pointed at the SGLang worker.""" + tok_path = _find_tokenizer_path(MODEL) + cfg_handle = tempfile.NamedTemporaryFile(mode="w", suffix=".toml", delete=False) + cfg_path = Path(cfg_handle.name) + main_text = build_smoke_router_config( + host="0.0.0.0", + port=ROUTER_PORT, + model=MODEL, + tokenizer_path=tok_path, + sglang_url=f"http://localhost:{SGLANG_PORT}", + ) + cfg_handle.write(main_text) + cfg_handle.close() + + try: + proc = subprocess.Popen( + [str(_BINARY), "--config", str(cfg_path)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + try: + _wait_http(f"http://localhost:{ROUTER_PORT}/readyz", timeout=60) + except Exception: + proc.send_signal(signal.SIGTERM) + proc.wait(timeout=30) + raise + + yield f"http://localhost:{ROUTER_PORT}" + + proc.send_signal(signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + finally: + cfg_path.unlink(missing_ok=True) + + +# --------------------------------------------------------------------------- +# Per-test multi-worker acceptance fixtures +# --------------------------------------------------------------------------- + + +def _detect_gpu_count() -> int: + """Count visible GPUs via ``nvidia-smi``. Returns 0 when no NVIDIA GPU + is available (CI on CPU-only runners, dev laptops, etc.). + """ + try: + out = subprocess.check_output( + ["nvidia-smi", "--query-gpu=index", "--format=csv,noheader"], + stderr=subprocess.DEVNULL, + timeout=5.0, + ) + except (FileNotFoundError, subprocess.SubprocessError): + return 0 + return len([ln for ln in out.decode().splitlines() if ln.strip()]) + + +class GPUAllocator: + """Single-process GPU index allocator. Test-scoped; not safe for + cross-process use (pytest-xdist) — each worker would race over the + full GPU set. Acceptance tests run serially, so this is fine. + """ + + def __init__(self, total: int): + self.total = total + self._free: list[int] = list(range(total)) + self._lock = threading.Lock() + + def acquire(self, n: int = 1) -> list[int]: + with self._lock: + if n > len(self._free): + raise pytest.skip.Exception( + f"requested {n} GPUs, only {len(self._free)}/{self.total} free" + ) + picked = self._free[:n] + self._free = self._free[n:] + return picked + + def release(self, ids: list[int]) -> None: + with self._lock: + self._free.extend(ids) + self._free.sort() + + +@pytest.fixture(scope="session") +def router_binary() -> Path: + """Locate the release ``sgl-router`` binary or skip the session. + + Used by the multi-worker acceptance tests (which spawn their own + Gateway per test instead of using the session-scoped ``router`` + fixture). + """ + env_path = os.environ.get("SGL_ROUTER_BINARY") + candidates: list[Path] = [] + if env_path: + candidates.append(Path(env_path)) + candidates.append(_BINARY) + for c in candidates: + if c.exists(): + return c + pytest.skip( + "sgl-router release binary not found at any of: " + + ", ".join(str(c) for c in candidates) + + ". Build with `cargo build --release` in experimental/sgl-router/." + ) + + +@pytest.fixture(scope="session") +def gpu_allocator() -> Iterator[GPUAllocator]: + """Session-scoped GPU index allocator. Skips the entire session when + no GPUs are visible — acceptance tests under chat_completions/ are + real-GPU. + """ + n = _detect_gpu_count() + if n == 0: + pytest.skip( + "no NVIDIA GPUs visible to nvidia-smi; acceptance tests are GPU-only" + ) + yield GPUAllocator(n) diff --git a/experimental/sgl-router/tests/e2e/infra/__init__.py b/experimental/sgl-router/tests/e2e/infra/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/experimental/sgl-router/tests/e2e/infra/gateway.py b/experimental/sgl-router/tests/e2e/infra/gateway.py new file mode 100644 index 000000000000..389267cff549 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/infra/gateway.py @@ -0,0 +1,404 @@ +"""Minimal sgl-router Gateway class — adapted from SMG's e2e_test/infra/gateway.py. + +Differences from SMG: + - SMG drives a Python launcher (`python3 -m sglang_router.launch_router`) + with worker URLs on the CLI. + - sgl-router uses a Rust binary (`experimental/sgl-router/target/release/sgl-router`) + with a TOML config file. Worker discovery is config-file-based; this + Gateway writes a TOML to a tempfile and execs the binary with + `--config `. + +Supported lifecycles: + - Regular mode: one model, N worker URLs, single policy. + - PD mode: one model, prefill_workers + decode_workers (lists of URLs), + discovery emits separate `WorkerMode::Prefill` / `WorkerMode::Decode` + entries. The router resolves PD pool isolation at request time. + +Use as a context manager: + + with Gateway() as gw: + gw.start_regular(model_path="...", worker_urls=[...]) + resp = httpx.post(f"{gw.base_url}/v1/chat/completions", json=...) + +or pytest fixture style (see e2e_test/conftest.py). +""" + +from __future__ import annotations + +import logging +import os +import signal +import socket +import subprocess +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import httpx + +logger = logging.getLogger(__name__) + +# Repo-relative path to the release binary. Set ``SGL_ROUTER_BINARY`` to +# override (e.g. a debug build, or a non-default ``CARGO_TARGET_DIR``). +# This file is at `experimental/sgl-router/tests/e2e/infra/gateway.py`, +# so four `.parent` hops to reach the sgl-router workspace root +# (infra → e2e → tests → sgl-router). Cargo lands the binary at +# `experimental/sgl-router/target/release/sgl-router`. A previous +# version used three hops and pointed at `tests/target/`, which +# would have broken any test that actually launches the router via +# this helper. +DEFAULT_BINARY = ( + Path(__file__).resolve().parent.parent.parent.parent + / "target" + / "release" + / "sgl-router" +) + + +def _get_open_port() -> int: + """Reserve an ephemeral TCP port in [20000, 55535]. + + The router itself doesn't have the ``port + 10000`` gRPC-derivation + constraint that SGLang's launch_server does, but we cap the range + anyway so the e2e helpers behave consistently across components. + """ + for _ in range(50): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + if 20000 <= port <= 55535: + return port + raise RuntimeError( + "could not allocate an ephemeral port in [20000, 55535] after 50 tries" + ) + + +def _resolve_tokenizer_path(tokenizer_path: str) -> str: + """Resolve a HuggingFace repo ID to a local ``tokenizer.json`` path. + + sgl-router's tokenizer loader treats the input as a filesystem path and + inspects its extension; a bare HF id like ``Qwen/Qwen3-0.6B`` looks + like a file with extension ``.6B`` and is rejected. When the HF Hub + cache already has the tokenizer, point the loader at the on-disk + ``tokenizer.json`` directly. Pass paths/URLs through unchanged. + """ + p = Path(tokenizer_path) + if p.exists(): + return str(p) + try: + from huggingface_hub import try_to_load_from_cache # type: ignore[import] + + cached = try_to_load_from_cache(tokenizer_path, "tokenizer.json") + if cached and Path(cached).is_file(): + return str(cached) + except Exception: # noqa: BLE001 + pass + return tokenizer_path + + +@dataclass +class WorkerInfo: + """Worker visible to the gateway via ``/v1/models``-style introspection. + + Mirrors SMG's WorkerInfo shape so test code reads the same. sgl-router + does not currently surface a `/v1/workers` admin API — this is a + placeholder for a future admin surface; current tests scrape + `/metrics` for per-worker observability instead. + """ + + id: str + url: str + model: str | None = None + status: str = "unknown" + metadata: dict[str, Any] = field(default_factory=dict) + + +class Gateway: + """Lifecycle-managed sgl-router instance for e2e tests. + + Not thread-safe; assume one Gateway per test (or per fixture scope). + """ + + def __init__( + self, + host: str = "127.0.0.1", + port: int | None = None, + binary: Path | None = None, + proxy_request_timeout_secs: int | None = None, + stale_request_timeout_secs: int | None = None, + ): + self.host = host + self.port = port or _get_open_port() + self.base_url = f"http://{self.host}:{self.port}" + # Resolve binary from env override, explicit arg, or repo default. + env_binary = os.environ.get("SGL_ROUTER_BINARY") + if binary is not None: + self.binary = Path(binary) + elif env_binary: + self.binary = Path(env_binary) + else: + self.binary = DEFAULT_BINARY + + # Test-side overrides for the router's tunables. Both default to + # `None`, in which case the router uses its production defaults + # (60 s proxy timeout, 300 s stale-request timeout). Tests set + # these short so per-request failures and stale-request expiry + # surface within the test's wall-time budget. + self.proxy_request_timeout_secs = proxy_request_timeout_secs + self.stale_request_timeout_secs = stale_request_timeout_secs + + self.process: subprocess.Popen | None = None + self._config_path: Path | None = None + self._started: bool = False + # Track child workers we spawned so __exit__ can tear them down. + self._owned_workers: list[subprocess.Popen] = [] + + # ----- context manager ------------------------------------------------- + + def __enter__(self) -> "Gateway": + return self + + def __exit__(self, *exc) -> None: + self.shutdown() + + # ----- start ---------------------------------------------------------- + + def start_regular( + self, + *, + model_id: str, + tokenizer_path: str, + worker_urls: list[str], + policy: str = "round_robin", + extra_models: list[dict] | None = None, + timeout: float = 60.0, + ) -> None: + """Start the router in regular (non-PD) mode. + + Args: + model_id: The model identifier the router will dispatch under. + tokenizer_path: Path or HF ID for the tokenizer the router uses + for cache-aware tokenization. + worker_urls: URLs of already-running ``sglang.launch_server`` + instances. The router uses ``static_urls`` discovery; + each worker's mode (plain) and any disaggregation + metadata are learned from ``/server_info``. + policy: Policy kind — ``round_robin``, ``random``, ``power_of_two``, + or ``cache_aware_zmq``. + timeout: How long to wait for ``/readyz`` before giving up. + """ + self._launch( + self._build_config( + model_id=model_id, + tokenizer_path=tokenizer_path, + urls=list(worker_urls), + policy=policy, + extra_models=extra_models or [], + ), + timeout=timeout, + ) + + def start_pd( + self, + *, + model_id: str, + tokenizer_path: str, + prefill_urls: list[str], + decode_urls: list[str], + policy: str = "round_robin", + timeout: float = 60.0, + ) -> None: + """Start the router in PD-disaggregated mode. + + All prefill + decode URLs go into one ``static_urls`` list. The + router seeds each worker as ``WorkerMode::Plain`` and the + manager's ``/server_info`` introspect step overrides mode + + ``bootstrap_port`` from the worker's self-disclosure. Workers + must have been launched with ``--disaggregation-mode`` and + ``--disaggregation-bootstrap-port`` for the PD role to be + picked up (see ``model_pool.spawn_worker``); modern SGLang is + assumed. + """ + self._launch( + self._build_config( + model_id=model_id, + tokenizer_path=tokenizer_path, + urls=list(prefill_urls) + list(decode_urls), + policy=policy, + extra_models=[], + ), + timeout=timeout, + ) + + # ----- shutdown -------------------------------------------------------- + + def shutdown(self) -> None: + """SIGTERM the router; SIGKILL after 30s. Idempotent.""" + if self.process is not None and self.process.poll() is None: + try: + self.process.send_signal(signal.SIGTERM) + try: + self.process.wait(timeout=30) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + except ProcessLookupError: + pass + self.process = None + if self._config_path and self._config_path.exists(): + self._config_path.unlink(missing_ok=True) + self._config_path = None + self._started = False + # Tear down any owned upstream workers. + for w in self._owned_workers: + if w.poll() is None: + try: + w.send_signal(signal.SIGTERM) + try: + w.wait(timeout=30) + except subprocess.TimeoutExpired: + w.kill() + w.wait() + except ProcessLookupError: + pass + self._owned_workers.clear() + + # ----- HTTP introspection helpers ------------------------------------- + + def healthy(self, timeout: float = 5.0) -> bool: + try: + resp = httpx.get(f"{self.base_url}/healthz", timeout=timeout) + return resp.status_code == 200 + except (httpx.RequestError, httpx.TimeoutException): + return False + + def ready(self, timeout: float = 5.0) -> bool: + try: + resp = httpx.get(f"{self.base_url}/readyz", timeout=timeout) + return resp.status_code == 200 + except (httpx.RequestError, httpx.TimeoutException): + return False + + def metrics_text(self, timeout: float = 5.0) -> str | None: + try: + resp = httpx.get(f"{self.base_url}/metrics", timeout=timeout) + if resp.status_code == 200: + return resp.text + return None + except (httpx.RequestError, httpx.TimeoutException): + return None + + # ----- internals ------------------------------------------------------ + + def _build_config( + self, + *, + model_id: str, + tokenizer_path: str, + urls: list[str], + policy: str, + extra_models: list[dict], + ) -> str: + resolved_tokenizer = _resolve_tokenizer_path(tokenizer_path) + + extra_model_toml = "" + for em in extra_models: + extra_model_toml += ( + f'\n[[models]]\nid = "{em["id"]}"\n' + f'tokenizer_path = "{_resolve_tokenizer_path(em["tokenizer_path"])}"\n' + f'policy = "{em.get("policy", policy)}"\n' + ) + + # Optional tunables — only emit the [proxy] and [active_load] + # sections if a test has overridden them, so production defaults + # apply otherwise. + proxy_section = "" + if self.proxy_request_timeout_secs is not None: + proxy_section = ( + f"\n[proxy]\nrequest_timeout_secs = {self.proxy_request_timeout_secs}\n" + ) + active_load_section = "" + if self.stale_request_timeout_secs is not None: + active_load_section = ( + f"\n[active_load]\nstale_request_timeout_secs = " + f"{self.stale_request_timeout_secs}\n" + ) + + urls_toml = ", ".join(f'"{u}"' for u in urls) + + return f"""\ +[server] +host = "{self.host}" +port = {self.port} + +[[models]] +id = "{model_id}" +tokenizer_path = "{resolved_tokenizer}" +policy = "{policy}" +{extra_model_toml} + +[discovery] +backend = "static_urls" + +[discovery.static_urls] +urls = [{urls_toml}] +{proxy_section}{active_load_section}""" + + def _launch(self, config_text: str, *, timeout: float) -> None: + if not self.binary.exists(): + raise RuntimeError( + f"sgl-router binary not found at {self.binary}. " + "Build it first: `cd experimental/sgl-router && cargo build --release` " + "or set SGL_ROUTER_BINARY to the binary path." + ) + # Write the main config. + fd, path = tempfile.mkstemp(suffix=".toml", prefix="sgl-router-") + os.close(fd) + self._config_path = Path(path) + self._config_path.write_text(config_text, encoding="utf-8") + logger.info("sgl-router config: %s", self._config_path) + logger.debug("sgl-router config text:\n%s", config_text) + + self.process = subprocess.Popen( + [str(self.binary), "--config", str(self._config_path)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + try: + self._wait_ready(timeout=timeout) + except Exception: + self.shutdown() + raise + self._started = True + + def _wait_ready(self, *, timeout: float) -> None: + deadline = time.time() + timeout + last_exc: Exception | None = None + while time.time() < deadline: + if self.process is not None and self.process.poll() is not None: + # Process exited early — surface stdout/stderr. + out = b"" + try: + if self.process.stdout is not None: + out = self.process.stdout.read() or b"" + except Exception: # noqa: BLE001 + pass + raise RuntimeError( + f"sgl-router exited during startup with code " + f"{self.process.returncode}. output:\n{out.decode(errors='replace')}", + ) + try: + resp = httpx.get(f"{self.base_url}/readyz", timeout=2.0) + if resp.status_code == 200: + return + except (httpx.RequestError, httpx.TimeoutException) as exc: + last_exc = exc + time.sleep(0.5) + raise TimeoutError( + f"sgl-router did not become ready at {self.base_url} within {timeout}s " + f"(last error: {last_exc})" + ) diff --git a/experimental/sgl-router/tests/e2e/infra/model_pool.py b/experimental/sgl-router/tests/e2e/infra/model_pool.py new file mode 100644 index 000000000000..2386bcbbc119 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/infra/model_pool.py @@ -0,0 +1,228 @@ +"""Minimal SGLang worker spawner for sgl-router e2e tests. + +Adapted from SMG's e2e_test/infra/model_pool.py — the 1200-line original +manages a pool of long-lived workers across many tests; here we only +need a thin wrapper around ``sglang.launch_server`` that: + + - allocates GPU(s) for the worker (via ``CUDA_VISIBLE_DEVICES``), + - spawns ``python3 -m sglang.launch_server`` with the right args, + - waits for ``/health`` to come up, + - optionally injects ``--kv-events-config`` so the worker exposes + the ``kv_events`` block on ``/server_info``. + +A test owns a ``ModelInstance`` for its duration; teardown shuts the +worker down. No cross-test pooling — the acceptance tests are slow +enough already (model load dominates) that pooling complexity wasn't +worth porting. +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import socket +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path + +import httpx + +from .model_specs import get_model_spec + +logger = logging.getLogger(__name__) + +# Passthrough Jinja chat template that emits ONLY `messages[*].content` +# joined with `\n` — matching the router's cache_aware_zmq prompt +# extraction. A worker launched with +# ``--chat-template `` tokenizes the +# raw content string, so its KV-block hashes align with what the +# router computes from the same chat-completions request. Test-only. +PASSTHROUGH_CHAT_TEMPLATE_PATH = str( + Path(__file__).parent / "passthrough_chat_template.jinja" +) + + +def _get_open_port() -> int: + """Allocate an ephemeral TCP port in the range [20000, 55535]. + + SGLang derives its internal gRPC port as ``http_port + 10000``; if the + kernel hands us an ephemeral port above 55535, that derivation overflows + 65535 and ``ServerArgs.__post_init__`` rejects it. Retrying a bounded + number of times keeps us safely below the ceiling without hand-rolling + a port registry. + """ + for _ in range(50): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + if 20000 <= port <= 55535: + return port + raise RuntimeError( + "could not allocate an ephemeral port in [20000, 55535] after 50 tries; " + "SGLang derives its internal gRPC port as http_port + 10000 and " + "rejects values above 65535" + ) + + +@dataclass +class ModelInstance: + """A running ``sglang.launch_server`` process. + + Use as a context manager: + + with spawn_worker("qwen3-0.6b", gpu_ids=[0]) as inst: + httpx.post(f"{inst.url}/generate", ...) + """ + + url: str + port: int + process: subprocess.Popen + model_id: str + gpu_ids: list[int] = field(default_factory=list) + kv_events_endpoint: str | None = None + + def __enter__(self) -> "ModelInstance": + return self + + def __exit__(self, *exc) -> None: + self.shutdown() + + def shutdown(self) -> None: + if self.process is not None and self.process.poll() is None: + try: + self.process.send_signal(signal.SIGTERM) + try: + self.process.wait(timeout=60) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + except ProcessLookupError: + pass + + +def spawn_worker( + model_id: str, + *, + gpu_ids: list[int], + port: int | None = None, + enable_kv_events: bool = False, + kv_events_port: int | None = None, + disagg_mode: str | None = None, + bootstrap_port: int | None = None, + extra_args: list[str] | None = None, + timeout: float = 600.0, +) -> ModelInstance: + """Spawn a single ``sglang.launch_server`` and wait for ``/health``. + + Args: + model_id: Key into :data:`model_specs.MODEL_SPECS`. + gpu_ids: Concrete GPU indices to bind via ``CUDA_VISIBLE_DEVICES``. + port: HTTP port; auto-assigned if None. + enable_kv_events: If True, inject ``--kv-events-config`` with a + ZMQ publisher so the router's introspection picks up the + kv_events block from ``/server_info`` (Patch 1). + kv_events_port: ZMQ publisher port. Auto-assigned if None and + ``enable_kv_events`` is True. + disagg_mode: "prefill" or "decode" for PD-disagg launches; passed + through as ``--disaggregation-mode``. + bootstrap_port: PD-disagg bootstrap port (prefill side only). + extra_args: Additional CLI args appended verbatim. + timeout: Health-check timeout. Cold-start on a fresh GPU can be + slow; default is 10 minutes. + """ + spec = get_model_spec(model_id) + port = port or _get_open_port() + base_url = f"http://127.0.0.1:{port}" + + cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + spec["model"], + "--port", + str(port), + "--host", + "127.0.0.1", + "--tp", + str(spec.get("tp", 1)), + ] + cmd.extend(spec.get("worker_args", []) or []) + + kv_events_endpoint: str | None = None + if enable_kv_events: + kv_port = kv_events_port or _get_open_port() + kv_events_endpoint = f"tcp://*:{kv_port}" + kv_cfg = { + "publisher": "zmq", + "endpoint": kv_events_endpoint, + "topic": "kv", + } + cmd.extend(["--kv-events-config", json.dumps(kv_cfg)]) + + if disagg_mode is not None: + cmd.extend(["--disaggregation-mode", disagg_mode]) + if bootstrap_port is not None: + cmd.extend(["--disaggregation-bootstrap-port", str(bootstrap_port)]) + + if extra_args: + cmd.extend(extra_args) + + env = os.environ.copy() + env["CUDA_VISIBLE_DEVICES"] = ",".join(str(g) for g in gpu_ids) + logger.info( + "spawning sglang worker: model=%s port=%d gpus=%s disagg=%s", + model_id, + port, + gpu_ids, + disagg_mode, + ) + + proc = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + inst = ModelInstance( + url=base_url, + port=port, + process=proc, + model_id=model_id, + gpu_ids=list(gpu_ids), + kv_events_endpoint=kv_events_endpoint, + ) + + # Wait for /health. Cold-start on H200 with weights uncached can take + # ~5 minutes; CI configurations should pre-warm. + deadline = time.time() + timeout + while time.time() < deadline: + if proc.poll() is not None: + out = b"" + try: + if proc.stdout is not None: + out = proc.stdout.read() or b"" + except Exception: # noqa: BLE001 + pass + raise RuntimeError( + f"sglang worker exited during startup with code {proc.returncode}; " + f"cmd: {' '.join(cmd)}\noutput:\n{out.decode(errors='replace')}", + ) + try: + resp = httpx.get(f"{base_url}/health", timeout=2.0) + if resp.status_code == 200: + logger.info("sglang worker ready at %s", base_url) + return inst + except (httpx.RequestError, httpx.TimeoutException): + pass + time.sleep(2.0) + + inst.shutdown() + raise TimeoutError( + f"sglang worker did not become healthy at {base_url} within {timeout}s", + ) diff --git a/experimental/sgl-router/tests/e2e/infra/model_specs.py b/experimental/sgl-router/tests/e2e/infra/model_specs.py new file mode 100644 index 000000000000..3363c8463741 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/infra/model_specs.py @@ -0,0 +1,77 @@ +"""Model specifications for sgl-router e2e tests. + +Adapted from SMG's e2e_test/infra/model_specs.py. The same dict-of-dicts +shape (so test code reads the same) but the entries are narrower — +sgl-router tests today target small/medium models only; the larger +function-calling / reasoning models from SMG are out of scope. + +Each entry: + - model: HuggingFace path or local path (env-resolved) + - memory_gb: estimated single-GPU footprint + - tp: tensor-parallel size (= GPUs needed) + - features: feature tags for filtering + - worker_args: optional extra `sglang.launch_server` flags +""" + +from __future__ import annotations + +import os + +# Local-cache root for CI / cluster nodes that pre-download HF weights. +# Mirrors the SMG `ROUTER_LOCAL_MODEL_PATH` env var. +ROUTER_LOCAL_MODEL_PATH = os.environ.get("ROUTER_LOCAL_MODEL_PATH", "") + + +def _resolve_model_path(hf_path: str) -> str: + """Prefer a local copy of the model when one exists under + ``ROUTER_LOCAL_MODEL_PATH``; otherwise fall back to the HuggingFace ID. + """ + if ROUTER_LOCAL_MODEL_PATH: + local_path = os.path.join(ROUTER_LOCAL_MODEL_PATH, hf_path) + if os.path.exists(local_path): + return local_path + return hf_path + + +MODEL_SPECS: dict[str, dict] = { + # Fast-start tiny model for convergence / decode-affinity / stale-request + # tests. Single GPU, ~2 GB weights, sub-30s start on a warm cache. + "qwen3-0.6b": { + "model": _resolve_model_path("Qwen/Qwen3-0.6B"), + "memory_gb": 4, + "tp": 1, + "features": ["chat", "streaming"], + }, + # Standard small chat model — matches SMG's `llama-1b` entry. + "llama-1b": { + "model": _resolve_model_path("meta-llama/Llama-3.2-1B-Instruct"), + "memory_gb": 4, + "tp": 1, + "features": ["chat", "streaming"], + }, + # Primary 8B chat model — matches SMG's `llama-8b`. + "llama-8b": { + "model": _resolve_model_path("meta-llama/Llama-3.1-8B-Instruct"), + "memory_gb": 16, + "tp": 1, + "features": ["chat", "streaming"], + }, +} + + +def get_model_spec(model_id: str) -> dict: + """Return the spec dict for ``model_id``; KeyError if absent.""" + if model_id not in MODEL_SPECS: + raise KeyError( + f"Unknown model: {model_id}. Available: {list(MODEL_SPECS.keys())}" + ) + return MODEL_SPECS[model_id] + + +def get_models_with_feature(feature: str) -> list[str]: + """Filter model IDs by feature tag (e.g. ``streaming``, ``chat``).""" + return [ + model_id + for model_id, spec in MODEL_SPECS.items() + if feature in spec.get("features", []) + ] diff --git a/experimental/sgl-router/tests/e2e/infra/passthrough_chat_template.jinja b/experimental/sgl-router/tests/e2e/infra/passthrough_chat_template.jinja new file mode 100644 index 000000000000..8d542dd66541 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/infra/passthrough_chat_template.jinja @@ -0,0 +1,13 @@ +{#- + Passthrough chat template for cache-aware-zmq e2e tests. + + Emits ONLY `messages[*].content` joined with `\n` — no role markers, + no special tokens, no generation prompt. This is the SAME shape the + router's cache_aware_zmq policy produces in `extract_prompt_text`, + so a worker launched with `--chat-template ` tokenizes the + same string the router will tokenize for routing — making block + hashes align across worker KV cache and router HashTree. + + Use only for tests; not appropriate for any real chat workload. +-#} +{{- messages | map(attribute='content') | join('\n') -}} diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker b/experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker new file mode 100644 index 000000000000..8cff23e91f21 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker @@ -0,0 +1,6 @@ +FROM python:3.12-slim +WORKDIR /app +RUN pip install --no-cache-dir fastapi uvicorn +COPY fake_worker.py . +EXPOSE 30000 +CMD ["python", "fake_worker.py"] diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.router b/experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.router new file mode 100644 index 000000000000..fe8a870f8d07 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.router @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1.6 +# Build sgl-router binary for k8s integration E2E. +# Context root: repo root (one level above experimental/sgl-router/). + +# Matches rust-toolchain.toml's pinned channel, avoiding an in-build rustup channel-sync. +FROM rust:1.90-bookworm AS builder + +# Pin to the exact toolchain pre-installed in the base image so rustup +# doesn't try to sync the channel manifest when it sees rust-toolchain.toml's +# `channel = "1.90"`. +ENV RUSTUP_TOOLCHAIN=1.90.0 + +# libssl-dev + pkg-config ship with rust:1.90-bookworm already; no apt-get needed. + +WORKDIR /build + +# Copy just the sgl-router crate (context is the repo root) +COPY experimental/sgl-router /build/experimental/sgl-router + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/build/experimental/sgl-router/target \ + cd /build/experimental/sgl-router \ + && cargo build --release --bin sgl-router \ + && cp target/release/sgl-router /usr/local/bin/sgl-router + +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/local/bin/sgl-router /usr/local/bin/sgl-router + +# Tiny tokenizer fixture used by the E2E config +COPY experimental/sgl-router/tests/fixtures/tiny_tokenizer.json /etc/tokenizer/tiny.json + +EXPOSE 8090 + +ENTRYPOINT ["sgl-router"] diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/conftest.py b/experimental/sgl-router/tests/e2e/k8s_integration/conftest.py new file mode 100644 index 000000000000..e2795ac6e5cb --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/conftest.py @@ -0,0 +1,251 @@ +"""Pytest configuration for sgl-router K8s integration tests. + +These tests require: + - A kind cluster named 'sgl-router-kind' + - The sgl-router:e2e and sgl-router-fake-worker:e2e images loaded into kind + - kubectl configured to use the kind-sgl-router-kind context + +Setup: ./tests/e2e/k8s_integration/setup.sh +Teardown: ./tests/e2e/k8s_integration/setup.sh teardown +""" + +from __future__ import annotations + +import logging +import socket +import subprocess +import time + +import httpx +import pytest + +logger = logging.getLogger(__name__) + +NAMESPACE = "sgl-router-test" +CLUSTER_NAME = "sgl-router-kind" +KUBECTL_CONTEXT = f"kind-{CLUSTER_NAME}" + +# sgl-router discovery reconciliation: if the watcher misses an event the +# reconciler fires within ~60s. Tests that exercise removal wait up to 90s. +RECONCILIATION_WAIT_SECS = 90 + +# Errors safe to retry while polling (transport-level only — HTTP 4xx/5xx +# are intentionally NOT included so real regressions surface immediately). +_TRANSIENT_ERRORS = ( + httpx.TransportError, + httpx.TimeoutException, + ConnectionError, + OSError, +) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "slow: marks tests that wait for multiple reconciliation cycles " + "(deselect with '-m \"not slow\"')", + ) + + +def _kubectl( + *args: str, + check: bool = True, + capture: bool = True, +) -> subprocess.CompletedProcess: + cmd = ["kubectl", "--context", KUBECTL_CONTEXT, *args] + logger.debug("Running: %s", " ".join(cmd)) + return subprocess.run(cmd, capture_output=capture, text=True, check=check) + + +def _apply_from_stdin(yaml_content: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["kubectl", "--context", KUBECTL_CONTEXT, "apply", "-f", "-"], + input=yaml_content, + capture_output=True, + text=True, + check=True, + ) + + +def _wait_for_deployment_ready( + name: str, + namespace: str = NAMESPACE, + timeout: int = 180, +) -> None: + _kubectl( + "rollout", + "status", + f"deployment/{name}", + "-n", + namespace, + f"--timeout={timeout}s", + ) + + +def _wait_for_pod_ready( + name: str, + namespace: str = NAMESPACE, + timeout: int = 120, +) -> None: + _kubectl( + "wait", + "--for=condition=Ready", + f"pod/{name}", + "-n", + namespace, + f"--timeout={timeout}s", + ) + + +def _wait_for_port(port: int, proc: subprocess.Popen, timeout: int = 15) -> None: + """Poll until a TCP connection to localhost:port succeeds.""" + deadline = time.time() + timeout + while time.time() < deadline: + if proc.poll() is not None: + stderr = proc.stderr.read().decode() if proc.stderr else "" + raise RuntimeError(f"port-forward process exited early: {stderr}") + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + return + except OSError: + time.sleep(0.5) + raise TimeoutError(f"Port {port} not ready after {timeout}s") + + +def _port_forward_start( + namespace: str, + service: str, + local_port: int, + remote_port: int, +) -> subprocess.Popen: + """Start kubectl port-forward and wait until the port is reachable.""" + cmd = [ + "kubectl", + "--context", + KUBECTL_CONTEXT, + "port-forward", + f"svc/{service}", + f"{local_port}:{remote_port}", + "-n", + namespace, + ] + logger.info("Starting port-forward: %s", " ".join(cmd)) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _wait_for_port(local_port, proc) + return proc + + +def _cleanup_port_forward(name: str, pf: subprocess.Popen) -> None: + try: + pf.terminate() + pf.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.warning( + "Port-forward %s did not exit on SIGTERM after 10s; killing", name + ) + pf.kill() + try: + pf.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.warning("Port-forward %s still running after SIGKILL", name) + except Exception as exc: + logger.warning("Error cleaning up %s port-forward: %s", name, exc) + + rc = pf.returncode + stderr = pf.stderr.read().decode() if pf.stderr else "" + if rc != -15: + suffix = f": {stderr.strip()}" if stderr.strip() else "" + logger.warning("Port-forward %s exited rc=%s%s", name, rc, suffix) + else: + logger.debug("Port-forward %s exited cleanly (rc=%s)", name, rc) + + +def _poll_until( + predicate, + description: str, + timeout: int, + interval: float = 5, +) -> bool: + """Poll predicate until True, or raise TimeoutError. + + Only transient network errors are retried; HTTP status errors and + programming errors propagate immediately. + """ + deadline = time.time() + timeout + last_error = None + attempts = 0 + while time.time() < deadline: + try: + attempts += 1 + if predicate(): + logger.info( + "Condition met: %s (after %d attempts)", description, attempts + ) + return True + except _TRANSIENT_ERRORS as exc: + last_error = exc + logger.debug("Transient error on attempt %d: %s", attempts, exc) + time.sleep(interval) + msg = f"Timeout waiting for: {description} (after {timeout}s, {attempts} attempts)" + if last_error: + msg += f" — last error: {last_error}" + raise TimeoutError(msg) + + +def _get_router_url(router_base: str) -> str: + return router_base + + +def _router_is_healthy(router_base: str) -> bool: + try: + r = httpx.get(f"{router_base}/healthz", timeout=3.0) + return r.status_code == 200 + except Exception: + return False + + +@pytest.fixture(scope="session") +def k8s_cluster(): + """Assert the kind cluster exists and kubectl context is reachable.""" + result = subprocess.run( + ["kind", "get", "clusters"], + capture_output=True, + text=True, + check=True, + ) + if CLUSTER_NAME not in result.stdout.splitlines(): + pytest.skip( + f"kind cluster '{CLUSTER_NAME}' not found — run " + f"./tests/e2e/k8s_integration/setup.sh first" + ) + _kubectl("cluster-info") + return True + + +@pytest.fixture(scope="function") +def router_port_forward(k8s_cluster): + """Per-test port-forward to sgl-router service. + + Function-scoped because some tests (notably + test_lifecycle.TestRouterRestart) force-delete the router pod; + a session-scoped port-forward would be bound to the deleted pod's + network namespace and stay dead for all subsequent tests in the + suite. Per-test setup costs ~1-2s. + """ + _wait_for_deployment_ready("sgl-router") + pf = _port_forward_start(NAMESPACE, "sgl-router", 8090, 8090) + try: + _poll_until( + lambda: _router_is_healthy("http://127.0.0.1:8090"), + "sgl-router /healthz returns 200", + timeout=30, + interval=1, + ) + yield "http://127.0.0.1:8090" + finally: + _cleanup_port_forward("sgl-router", pf) + + +@pytest.fixture(scope="function") +def router_url(router_port_forward): + return router_port_forward diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/fake_worker.py b/experimental/sgl-router/tests/e2e/k8s_integration/fake_worker.py new file mode 100644 index 000000000000..658556bacf2a --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/fake_worker.py @@ -0,0 +1,73 @@ +"""Minimal fake SGLang worker for kind E2E integration testing. + +Responds to: + GET /health -> {"status": "ok"} + GET /server_info -> {"served_model_name": MODEL_ID} + GET /v1/models -> list with a single MODEL_ID model entry + POST /v1/chat/completions -> echoes the last user message back +""" + +from __future__ import annotations + +import os + +import uvicorn +from fastapi import FastAPI, Request + +app = FastAPI() + +MODEL_ID = os.environ.get("MODEL_ID", "tiny") + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.get("/server_info") +async def server_info(): + # The sgl-router worker manager fetches this on every Added event and + # uses `served_model_name` to populate the registry's model index. + return {"served_model_name": MODEL_ID} + + +@app.get("/v1/models") +async def models(): + return { + "object": "list", + "data": [ + { + "id": MODEL_ID, + "object": "model", + "created": 0, + "owned_by": "sglang", + } + ], + } + + +@app.post("/v1/chat/completions") +async def chat_completions(request: Request): + payload = await request.json() + messages = payload.get("messages", []) + last_content = messages[-1]["content"] if messages else "" + return { + "id": "chatcmpl-mock", + "object": "chat.completion", + "model": payload.get("model", MODEL_ID), + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": f"echo: {last_content}", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=30000) diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/manifests/namespace.yaml b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/namespace.yaml new file mode 100644 index 000000000000..4a54986939c1 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sgl-router-test diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/manifests/rbac-cluster-scoped.yaml b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/rbac-cluster-scoped.yaml new file mode 100644 index 000000000000..32033dde5963 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/rbac-cluster-scoped.yaml @@ -0,0 +1,33 @@ +# Cluster-wide RBAC for the cross-namespace discovery test. +# Distinct ServiceAccount/ClusterRole names to avoid collision with +# the namespace-scoped Role in rbac.yaml used by the default router. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sgl-router-cluster + namespace: sgl-router-test +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sgl-router-cluster +rules: + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["services", "pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sgl-router-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sgl-router-cluster +subjects: + - kind: ServiceAccount + name: sgl-router-cluster + namespace: sgl-router-test diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/manifests/rbac.yaml b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/rbac.yaml new file mode 100644 index 000000000000..8080ee395049 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/rbac.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sgl-router + namespace: sgl-router-test +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sgl-router + namespace: sgl-router-test +rules: + # EndpointSlice watch (k8s discovery backend) + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + # Service list/watch (needed to resolve EndpointSlice owner) + - apiGroups: [""] + resources: ["services", "pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sgl-router + namespace: sgl-router-test +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sgl-router +subjects: + - kind: ServiceAccount + name: sgl-router + namespace: sgl-router-test diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/manifests/router-cluster-scoped.yaml b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/router-cluster-scoped.yaml new file mode 100644 index 000000000000..407173bedb49 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/router-cluster-scoped.yaml @@ -0,0 +1,60 @@ +# sgl-router deployment with ClusterRole for cross-namespace discovery test. +# Watches workers in ALL namespaces via cluster-scoped EndpointSlice access. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sgl-router-cluster + namespace: sgl-router-test +spec: + replicas: 1 + selector: + matchLabels: + app: sgl-router-cluster + template: + metadata: + labels: + app: sgl-router-cluster + spec: + serviceAccountName: sgl-router-cluster + containers: + - name: router + image: sgl-router:e2e + imagePullPolicy: Never + args: + - "--config" + - "/etc/config/router-cluster.toml" + ports: + - containerPort: 8091 + name: http + readinessProbe: + httpGet: + path: /readyz + port: 8091 + initialDelaySeconds: 3 + periodSeconds: 3 + livenessProbe: + httpGet: + path: /healthz + port: 8091 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/config + volumes: + - name: config + configMap: + name: sgl-router-cluster-config +--- +apiVersion: v1 +kind: Service +metadata: + name: sgl-router-cluster + namespace: sgl-router-test +spec: + selector: + app: sgl-router-cluster + ports: + - name: http + port: 8091 + targetPort: 8091 diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/manifests/router.yaml b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/router.yaml new file mode 100644 index 000000000000..0d9b3900ce1e --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/manifests/router.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sgl-router + namespace: sgl-router-test +spec: + replicas: 1 + selector: + matchLabels: + app: sgl-router + template: + metadata: + labels: + app: sgl-router + spec: + serviceAccountName: sgl-router + containers: + - name: router + image: sgl-router:e2e + imagePullPolicy: Never + args: + - "--config" + - "/etc/config/router.toml" + ports: + - containerPort: 8090 + name: http + readinessProbe: + httpGet: + path: /readyz + port: 8090 + initialDelaySeconds: 3 + periodSeconds: 3 + livenessProbe: + httpGet: + path: /healthz + port: 8090 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/config + volumes: + - name: config + configMap: + name: sgl-router-config +--- +apiVersion: v1 +kind: Service +metadata: + name: sgl-router + namespace: sgl-router-test +spec: + selector: + app: sgl-router + ports: + - name: http + port: 8090 + targetPort: 8090 diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/requirements.txt b/experimental/sgl-router/tests/e2e/k8s_integration/requirements.txt new file mode 100644 index 000000000000..835147a0324b --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/requirements.txt @@ -0,0 +1,2 @@ +httpx==0.27.2 +pytest==8.3.3 diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/setup.sh b/experimental/sgl-router/tests/e2e/k8s_integration/setup.sh new file mode 100755 index 000000000000..882a90a6031c --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/setup.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Bootstrap a kind cluster for sgl-router K8s integration E2E tests. +# +# Prerequisites: Docker, kind, kubectl +# +# Usage: +# ./tests/e2e/k8s_integration/setup.sh # full setup +# ./tests/e2e/k8s_integration/setup.sh teardown # delete the cluster + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../.." && pwd)" # repo root (above experimental/) +SGL_ROUTER_DIR="${REPO_ROOT}/experimental/sgl-router" +CLUSTER_NAME="${CLUSTER:-sgl-router-kind}" +NAMESPACE="${NAMESPACE:-sgl-router-test}" +CONTEXT="kind-${CLUSTER_NAME}" +MANIFESTS_DIR="${SCRIPT_DIR}/manifests" + +log() { echo "==> $*"; } + +teardown() { + log "Tearing down cluster '${CLUSTER_NAME}'..." + if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + kind delete cluster --name "${CLUSTER_NAME}" + else + log "Cluster '${CLUSTER_NAME}' not found, nothing to tear down." + fi + log "Done." +} + +if [[ "${1:-}" == "teardown" ]]; then + teardown + exit 0 +fi + +# --------------------------------------------------------------------------- +# Step 1: Create kind cluster (idempotent) +# --------------------------------------------------------------------------- +if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + log "Kind cluster '${CLUSTER_NAME}' already exists — reusing." +else + log "Creating kind cluster '${CLUSTER_NAME}'..." + kind create cluster --name "${CLUSTER_NAME}" --wait 60s +fi + +kubectl config use-context "${CONTEXT}" + +# --------------------------------------------------------------------------- +# Step 2: Build Docker images (unless SKIP_DOCKER_BUILD=1) +# --------------------------------------------------------------------------- +if [[ "${SKIP_DOCKER_BUILD:-}" == "1" ]]; then + log "SKIP_DOCKER_BUILD=1 — skipping docker build; expecting images to exist locally." + for img in sgl-router:e2e sgl-router-fake-worker:e2e; do + if ! docker image inspect "${img}" >/dev/null 2>&1; then + log "ERROR: ${img} not found locally; cannot continue without building." + exit 1 + fi + done +else + log "Building sgl-router:e2e from ${REPO_ROOT} ..." + docker build \ + -f "${SCRIPT_DIR}/Dockerfile.router" \ + -t sgl-router:e2e \ + "${REPO_ROOT}" + + log "Building sgl-router-fake-worker:e2e ..." + docker build \ + -f "${SCRIPT_DIR}/Dockerfile.fake_worker" \ + -t sgl-router-fake-worker:e2e \ + "${SCRIPT_DIR}" +fi + +# --------------------------------------------------------------------------- +# Step 3: Load images into kind +# --------------------------------------------------------------------------- +log "Loading images into kind cluster '${CLUSTER_NAME}'..." +kind load docker-image sgl-router:e2e --name "${CLUSTER_NAME}" +kind load docker-image sgl-router-fake-worker:e2e --name "${CLUSTER_NAME}" + +# --------------------------------------------------------------------------- +# Step 4: Apply namespace and RBAC +# --------------------------------------------------------------------------- +log "Applying namespace and RBAC..." +kubectl --context "${CONTEXT}" apply -f "${MANIFESTS_DIR}/namespace.yaml" +kubectl --context "${CONTEXT}" apply -f "${MANIFESTS_DIR}/rbac.yaml" + +# --------------------------------------------------------------------------- +# Step 5: Deploy 3 fake-worker replicas behind a Service +# The Service causes K8s to auto-create an EndpointSlice, which +# the sgl-router K8s discovery backend watches. +# --------------------------------------------------------------------------- +log "Deploying fake-worker Deployment + Service (3 replicas, app=sglang)..." +kubectl --context "${CONTEXT}" -n "${NAMESPACE}" apply -f - < None: + """Deploy a fake-worker pod with imagePullPolicy=Never in the given namespace.""" + pod_manifest = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": name, + "namespace": namespace, + "labels": {"app": "sglang", "cross-ns-test": "true"}, + }, + "spec": { + "containers": [ + { + "name": "worker", + "image": "sgl-router-fake-worker:e2e", + "imagePullPolicy": "Never", + "ports": [{"containerPort": 30000}], + "readinessProbe": { + "httpGet": {"path": "/health", "port": 30000}, + "initialDelaySeconds": 2, + "periodSeconds": 3, + }, + } + ] + }, + } + proc = subprocess.run( + ["kubectl", "--context", KUBECTL_CONTEXT, "apply", "-f", "-"], + input=json.dumps(pod_manifest), + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"Failed to deploy pod {name} in namespace {namespace} " + f"(rc={proc.returncode}): {proc.stderr.strip()!r}" + ) + logger.info("Deployed worker %s in namespace %s", name, namespace) + + +def _safe_delete_pod(name: str, namespace: str) -> None: + try: + _kubectl( + "delete", + "pod", + name, + "-n", + namespace, + "--ignore-not-found", + "--force", + "--grace-period=0", + ) + except Exception as exc: + logger.warning("Cleanup failed for pod %s in ns %s: %s", name, namespace, exc) + + +def _ensure_namespace(name: str) -> None: + manifest = {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": name}} + _apply_from_stdin(json.dumps(manifest)) + + +def _ensure_service_in_ns(namespace: str, selector: str = "app=sglang") -> None: + """Create a Service so K8s auto-creates an EndpointSlice for cross-ns workers. + + Service `metadata.labels` propagates to the auto-created EndpointSlice's + labels — and the cluster-scoped router filters slices server-side by + `app=sglang,cross-ns-test=true`. Without those labels on the Service, + its EndpointSlice gets filtered out and the cross-ns worker is invisible. + """ + svc_manifest = { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "fake-worker", + "namespace": namespace, + "labels": {"app": "sglang", "cross-ns-test": "true"}, + }, + "spec": { + "selector": {"app": "sglang", "cross-ns-test": "true"}, + "ports": [{"port": 30000, "targetPort": 30000}], + }, + } + _apply_from_stdin(json.dumps(svc_manifest)) + + +def _can_route(router_url: str) -> bool: + try: + r = httpx.post( + f"{router_url}/v1/chat/completions", + json={ + "model": "tiny", + "messages": [{"role": "user", "content": "cross-ns"}], + }, + timeout=8.0, + ) + return r.status_code == 200 + except Exception: + return False + + +@pytest.fixture(scope="module") +def cluster_scoped_router(k8s_cluster): + """Deploy the cluster-scoped RBAC + router, plus a second namespace.""" + rbac_manifest = MANIFESTS_DIR / "rbac-cluster-scoped.yaml" + router_manifest = MANIFESTS_DIR / "router-cluster-scoped.yaml" + + _kubectl("apply", "-f", str(rbac_manifest)) + _ensure_namespace(EXTRA_NAMESPACE) + _ensure_service_in_ns(EXTRA_NAMESPACE) + + # ConfigMap for the cluster-scoped router: empty namespace = watch all + cluster_config = """[server] +host = "0.0.0.0" +port = 8091 + +[[models]] +id = "tiny" +tokenizer_path = "/etc/tokenizer/tiny.json" +policy = "round_robin" + +[discovery] +backend = "k8s" + +[discovery.k8s] +namespace = "" +label_selector = "app=sglang,cross-ns-test=true" +""" + _kubectl( + "create", + "configmap", + "sgl-router-cluster-config", + f"--from-literal=router-cluster.toml={cluster_config}", + "-n", + NAMESPACE, + "--dry-run=client", + "-o", + "yaml", + check=True, + ) + # pipe through apply + proc = _kubectl( + "create", + "configmap", + "sgl-router-cluster-config", + f"--from-literal=router-cluster.toml={cluster_config}", + "-n", + NAMESPACE, + "--dry-run=client", + "-o", + "yaml", + ) + _apply_from_stdin(proc.stdout) + + _kubectl("apply", "-f", str(router_manifest)) + + # The cluster-scoped router's /readyz blocks on registry-not-empty, so + # without at least one matching worker the rollout-status check below + # would hang for 180s. Deploy a "bootstrap" worker in EXTRA_NAMESPACE + # with the label_selector match (app=sglang,cross-ns-test=true) so the + # router's k8s discovery picks it up before the readiness probe runs. + # The test body adds a SECOND worker later to verify dynamic discovery. + bootstrap_worker = "cross-ns-worker-bootstrap" + _deploy_fake_worker_in_ns(bootstrap_worker, EXTRA_NAMESPACE) + + pf = None + try: + _wait_for_deployment_ready("sgl-router-cluster") + pf = _port_forward_start( + NAMESPACE, "sgl-router-cluster", CLUSTER_ROUTER_PORT, 8091 + ) + yield f"http://127.0.0.1:{CLUSTER_ROUTER_PORT}" + finally: + if pf is not None: + _cleanup_port_forward("cluster_router", pf) + _safe_delete_pod(bootstrap_worker, EXTRA_NAMESPACE) + _kubectl( + "delete", "-f", str(router_manifest), "--ignore-not-found", check=False + ) + _kubectl("delete", "-f", str(rbac_manifest), "--ignore-not-found", check=False) + _kubectl( + "delete", + "namespace", + EXTRA_NAMESPACE, + "--ignore-not-found", + "--wait=true", + "--timeout=60s", + check=False, + ) + + +class TestClusterWideDiscovery: + """Router with ClusterRole and no namespace filter sees workers in every namespace.""" + + def test_router_routes_to_worker_in_extra_namespace(self, cluster_scoped_router): + """Deploy one fake-worker pod in the extra namespace behind a Service; + the cluster-scoped router must discover it (via its EndpointSlice) and + successfully route a chat completion to it.""" + router_url = cluster_scoped_router + worker_name = "cross-ns-worker-extra" + + try: + _deploy_fake_worker_in_ns(worker_name, EXTRA_NAMESPACE) + + _poll_until( + lambda: _can_route(router_url), + "cluster-scoped router routes to worker in extra namespace", + timeout=60, + interval=3, + ) + + r = httpx.post( + f"{router_url}/v1/chat/completions", + json={ + "model": "tiny", + "messages": [ + {"role": "user", "content": "cross-namespace routing"} + ], + }, + timeout=15.0, + ) + assert r.status_code == 200, f"expected 200, got {r.status_code}: {r.text}" + assert "echo:" in r.json()["choices"][0]["message"]["content"] + finally: + _safe_delete_pod(worker_name, EXTRA_NAMESPACE) diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/test_discovery.py b/experimental/sgl-router/tests/e2e/k8s_integration/test_discovery.py new file mode 100644 index 000000000000..83f0faab0bbe --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/test_discovery.py @@ -0,0 +1,84 @@ +"""E2E: sgl-router K8s discovery — basic routing. + +Verifies that sgl-router, configured with the k8s EndpointSlice backend, +discovers the 3 fake-worker replicas deployed by setup.sh and successfully +routes chat-completion requests to them. +""" + +from __future__ import annotations + +import httpx +import pytest +from conftest import NAMESPACE, _kubectl, _poll_until, logger + + +def _scale_fake_worker(replicas: int) -> None: + _kubectl( + "scale", + "deployment/fake-worker", + f"--replicas={replicas}", + "-n", + NAMESPACE, + ) + + +def test_router_routes_chat_to_a_worker(router_url): + """A /v1/chat/completions request through the router returns 200 with the + fake-worker echo payload, proving end-to-end routing works.""" + r = httpx.post( + f"{router_url}/v1/chat/completions", + json={ + "model": "tiny", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + }, + timeout=15.0, + ) + assert r.status_code == 200, f"expected 200, got {r.status_code}: {r.text}" + body = r.json() + assert "echo:" in body["choices"][0]["message"]["content"] + + +def test_router_lists_model(router_url): + """GET /v1/models returns the 'tiny' model entry from the router config.""" + r = httpx.get(f"{router_url}/v1/models", timeout=10.0) + assert r.status_code == 200, f"expected 200, got {r.status_code}: {r.text}" + body = r.json() + ids = [m["id"] for m in body["data"]] + assert "tiny" in ids, f"expected 'tiny' in model list, got {ids}" + + +def test_router_discovers_multiple_workers(router_url): + """Scale down from 3 to 1 and back to 3 replicas; router must continue + routing successfully after each transition (EndpointSlice watch reflects + the change).""" + # First confirm baseline routing + r = httpx.post( + f"{router_url}/v1/chat/completions", + json={ + "model": "tiny", + "messages": [{"role": "user", "content": "scale-test"}], + }, + timeout=15.0, + ) + assert r.status_code == 200 + + # Scale down to 1 — router should still route after reconverging + _scale_fake_worker(1) + _poll_until( + lambda: httpx.post( + f"{router_url}/v1/chat/completions", + json={ + "model": "tiny", + "messages": [{"role": "user", "content": "post-scale-down"}], + }, + timeout=10.0, + ).status_code + == 200, + "router routes after scale-down to 1", + timeout=60, + interval=3, + ) + + # Restore to 3 + _scale_fake_worker(3) diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/test_lifecycle.py b/experimental/sgl-router/tests/e2e/k8s_integration/test_lifecycle.py new file mode 100644 index 000000000000..ab94b32c3373 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/test_lifecycle.py @@ -0,0 +1,150 @@ +"""Worker lifecycle integration tests. + +Covers: +1. Scaling replicas up — new EndpointSlice entries are discovered. +2. Scaling replicas down — removed endpoints are deregistered. +3. Router restart — after the router pod is killed, the Deployment restarts + it and it re-lists the existing EndpointSlice entries without duplicates. + +These tests DO NOT use a /workers admin API (sgl-router does not expose +one). They verify behaviour through /v1/chat/completions responses and +by driving the deployment scale. +""" + +from __future__ import annotations + +import logging + +import httpx +import pytest +from conftest import ( + NAMESPACE, + _cleanup_port_forward, + _kubectl, + _poll_until, + _port_forward_start, + _wait_for_deployment_ready, + logger, +) + +ROUTER_RESTART_PORT = 8092 + + +def _scale(deployment: str, replicas: int) -> None: + _kubectl( + "scale", f"deployment/{deployment}", f"--replicas={replicas}", "-n", NAMESPACE + ) + + +def _can_route(router_url: str) -> bool: + try: + r = httpx.post( + f"{router_url}/v1/chat/completions", + json={"model": "tiny", "messages": [{"role": "user", "content": "ping"}]}, + timeout=8.0, + ) + return r.status_code == 200 + except Exception: + return False + + +class TestScaleUp: + """Scaling fake-worker replicas up must not break routing.""" + + def test_router_routes_after_scale_up(self, router_url): + """Restore 3 replicas (in case a prior test left 1), verify routing.""" + _scale("fake-worker", 3) + _poll_until( + lambda: _can_route(router_url), + "router routes after scale-up to 3", + timeout=60, + interval=3, + ) + + +class TestScaleDown: + """Scaling to 0 then back up must restore routing.""" + + def test_router_recovers_after_scale_to_zero_and_back(self, router_url): + try: + _scale("fake-worker", 0) + # After scale-to-0 the router may return 503 (no healthy workers) + # That is expected behaviour — assert it transitions back on scale-up. + _scale("fake-worker", 2) + _poll_until( + lambda: _can_route(router_url), + "router routes again after scale-up from 0", + timeout=90, + interval=3, + ) + finally: + _scale("fake-worker", 3) + + +class TestRouterRestart: + """Killing the router pod forces a Deployment restart; the new pod must + re-discover workers via the EndpointSlice watch without duplicates.""" + + def test_router_rediscovers_workers_after_restart(self, k8s_cluster): + # Use a dedicated port to avoid clashing with the session fixture + pf_holder: list = [None] + try: + _wait_for_deployment_ready("sgl-router") + pf_holder[0] = _port_forward_start( + NAMESPACE, "sgl-router", ROUTER_RESTART_PORT, 8090 + ) + restart_url = f"http://127.0.0.1:{ROUTER_RESTART_PORT}" + + # Baseline: routing works pre-restart + _poll_until( + lambda: _can_route(restart_url), + "baseline routing works pre-restart", + timeout=30, + interval=2, + ) + + # Kill the router pod — the Deployment ReplicaSet will restart it + res = _kubectl( + "get", + "pod", + "-n", + NAMESPACE, + "-l", + "app=sgl-router", + "-o", + "jsonpath={.items[0].metadata.name}", + check=False, + ) + old_pod = res.stdout.strip() + if old_pod: + _kubectl( + "delete", + "pod", + old_pod, + "-n", + NAMESPACE, + "--force", + "--grace-period=0", + ) + + # Tear down the old port-forward before waiting for the new pod + if pf_holder[0] is not None: + _cleanup_port_forward("router-restart-pre-kill", pf_holder[0]) + pf_holder[0] = None + + _wait_for_deployment_ready("sgl-router") + + pf_holder[0] = _port_forward_start( + NAMESPACE, "sgl-router", ROUTER_RESTART_PORT, 8090 + ) + + # After restart, routing must come back (EndpointSlice re-watch) + _poll_until( + lambda: _can_route(restart_url), + "routing restored after router restart", + timeout=60, + interval=3, + ) + finally: + if pf_holder[0] is not None: + _cleanup_port_forward("router-restart", pf_holder[0]) diff --git a/experimental/sgl-router/tests/e2e/k8s_integration/test_reconciliation.py b/experimental/sgl-router/tests/e2e/k8s_integration/test_reconciliation.py new file mode 100644 index 000000000000..811f26516089 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/k8s_integration/test_reconciliation.py @@ -0,0 +1,163 @@ +"""K8s discovery reconciliation integration tests. + +Tests verify that: +1. The K8s EndpointSlice watcher correctly discovers new workers as Services + and backing Deployments are updated. +2. Workers are removed from the router's registry after the backing EndpointSlice + entries disappear (pod deleted / deployment scaled to 0). +3. After a simulated watch-connection interruption (router restarted), the + registry converges back to the correct worker set. + +Note: sgl-router does not currently expose a Prometheus /metrics endpoint, +so the SMG-style metric assertions are not used here. Disconnect/reconnect +coverage is provided by test_lifecycle.TestRouterRestart. +""" + +from __future__ import annotations + +import logging +import time + +import httpx +import pytest +from conftest import ( + NAMESPACE, + RECONCILIATION_WAIT_SECS, + _kubectl, + _poll_until, + logger, +) + + +def _scale_fake_worker(replicas: int) -> None: + _kubectl( + "scale", "deployment/fake-worker", f"--replicas={replicas}", "-n", NAMESPACE + ) + + +def _can_route(router_url: str) -> bool: + try: + r = httpx.post( + f"{router_url}/v1/chat/completions", + json={ + "model": "tiny", + "messages": [{"role": "user", "content": "reconcile"}], + }, + timeout=8.0, + ) + return r.status_code == 200 + except Exception: + return False + + +class TestWatcherDiscovery: + """The EndpointSlice watcher discovers new endpoints on Deployment scale-up.""" + + def test_watcher_discovers_new_endpoints_on_scale_up(self, router_url): + """Scale from 1 to 3 replicas; router must continue routing successfully.""" + _scale_fake_worker(1) + # Wait for scale-down to propagate and routing to stabilise + _poll_until( + lambda: _can_route(router_url), + "router routes with 1 replica", + timeout=60, + interval=3, + ) + + _scale_fake_worker(3) + _poll_until( + lambda: _can_route(router_url), + "router routes with 3 replicas (after scale-up)", + timeout=60, + interval=3, + ) + + +class TestStaleEndpointRemoval: + """When fake-worker replicas drop, the router must stop routing to the + removed endpoints. + + Because sgl-router has no /workers admin API, we verify removal + indirectly: scale to 0, assert the router returns non-200 (or at least + that scaling back to 2 restores routing), then restore. + """ + + def test_routing_restores_after_scale_down_and_back_up(self, router_url): + """Scale to 0 (no workers → expect non-200), then restore to 2. + After restore the router must route again within the reconciliation window. + """ + try: + _scale_fake_worker(0) + + # Expect routing to fail eventually (503 or connection error) + deadline = time.time() + RECONCILIATION_WAIT_SECS + routing_failed = False + while time.time() < deadline: + try: + r = httpx.post( + f"{router_url}/v1/chat/completions", + json={ + "model": "tiny", + "messages": [{"role": "user", "content": "no-workers"}], + }, + timeout=5.0, + ) + if r.status_code != 200: + routing_failed = True + break + except Exception: + routing_failed = True + break + time.sleep(3) + + # If after RECONCILIATION_WAIT_SECS the router is still routing, + # that means old endpoints are cached — not necessarily wrong for + # a watcher that hasn't ticked yet, but log a warning. + if not routing_failed: + logger.warning( + "Router still returning 200 after scale-to-0; " + "EndpointSlice event may be delayed — continuing test." + ) + + # Restore workers and verify routing comes back + _scale_fake_worker(2) + _poll_until( + lambda: _can_route(router_url), + "routing restored after scale back up to 2", + timeout=RECONCILIATION_WAIT_SECS, + interval=3, + ) + finally: + _scale_fake_worker(3) + + +class TestReconciliationConsistency: + """Routing remains stable over multiple reconciliation windows with steady + worker state — no spurious deregistrations or duplicate registrations.""" + + @pytest.mark.slow + def test_routing_stable_over_multiple_reconciliation_cycles(self, router_url): + """Deploy 3 workers, sample routing success over ~150s (2 reconciliation + cycles + margin), assert no interruptions.""" + _scale_fake_worker(3) + _poll_until( + lambda: _can_route(router_url), + "baseline routing with 3 workers", + timeout=30, + interval=2, + ) + + # Sample every 15s for 150s + wait_secs = RECONCILIATION_WAIT_SECS + 60 + end_time = time.time() + wait_secs + failures = [] + while time.time() < end_time: + ok = _can_route(router_url) + if not ok: + failures.append(time.time()) + time.sleep(15) + + assert not failures, ( + f"Routing failed at {len(failures)} sample(s) during stability window; " + f"timestamps: {failures}" + ) diff --git a/experimental/sgl-router/tests/e2e/pyproject.toml b/experimental/sgl-router/tests/e2e/pyproject.toml new file mode 100644 index 000000000000..1faba8be9bef --- /dev/null +++ b/experimental/sgl-router/tests/e2e/pyproject.toml @@ -0,0 +1,26 @@ +# Pytest configuration for sgl-router tests/e2e/. +# Lives next to conftest.py so `pytest experimental/sgl-router/tests/e2e/` +# picks it up automatically. + +[tool.pytest.ini_options] +minversion = "8.0" +# Default discovery: smoke tests (top-level test_*.py) and the +# multi-worker chat_completions suite. k8s_integration is intentionally +# not in the default set — it requires a kind/k8s cluster and is +# invoked explicitly. +testpaths = [ + ".", + "chat_completions", +] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +markers = [ + "real_gpu: requires at least one NVIDIA GPU (skipped on CPU-only hosts)", + "pd_mode: requires the router started in PD-disaggregation mode", + "slow: takes >30s (model load, multi-request convergence checks)", +] +log_cli = true +log_cli_level = "INFO" +log_cli_format = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" +log_cli_date_format = "%H:%M:%S" diff --git a/experimental/sgl-router/tests/e2e/requirements.txt b/experimental/sgl-router/tests/e2e/requirements.txt new file mode 100644 index 000000000000..f5222844b08e --- /dev/null +++ b/experimental/sgl-router/tests/e2e/requirements.txt @@ -0,0 +1,15 @@ +httpx==0.27.2 +pytest==8.3.3 +pytest-asyncio==0.24.0 +# huggingface_hub is intentionally NOT pinned here. SGLang's +# `scripts/ci/cuda/ci_install_dependency.sh` already installs a +# version compatible with the rest of its transitive deps +# (transformers / diffusers / kernels, which require +# huggingface_hub >= 1.5 / >= 0.34 / >= 1.3 respectively). An earlier +# pin of `huggingface_hub==0.26.2` here got installed AFTER the SGLang +# deps and downgraded huggingface_hub past `is_offline_mode`'s top- +# level export, which broke `from sglang.srt.server_args import …` +# at module import time and turned every smoke test into a 5-minute +# `/health` timeout with no actionable signal. +# The e2e suite only uses huggingface_hub's `try_to_load_from_cache`, +# which is available in every release SGLang would install. diff --git a/experimental/sgl-router/tests/e2e/test_chat_smoke.py b/experimental/sgl-router/tests/e2e/test_chat_smoke.py new file mode 100644 index 000000000000..16d030bd3258 --- /dev/null +++ b/experimental/sgl-router/tests/e2e/test_chat_smoke.py @@ -0,0 +1,64 @@ +""" +Smoke tests for /v1/models and /v1/chat/completions (streaming + non-streaming). +""" + +from __future__ import annotations + +import httpx +import pytest + +MODEL = "Qwen/Qwen3-0.6B" + + +def test_models(router: str) -> None: + """GET /v1/models must list the configured model.""" + resp = httpx.get(f"{router}/v1/models", timeout=30) + assert resp.status_code == 200, resp.text + data = resp.json() + ids = [m["id"] for m in data.get("data", [])] + assert any( + MODEL in mid for mid in ids + ), f"Model {MODEL!r} not found in /v1/models response: {ids}" + + +def test_chat_non_streaming(router: str) -> None: + """POST /v1/chat/completions (stream=False) returns an assistant message.""" + payload = { + "model": MODEL, + "messages": [{"role": "user", "content": "Say hi."}], + "max_tokens": 10, + "stream": False, + } + resp = httpx.post(f"{router}/v1/chat/completions", json=payload, timeout=60) + assert resp.status_code == 200, resp.text + body = resp.json() + choice = body["choices"][0] + assert choice["message"]["role"] == "assistant" + assert choice["message"]["content"], "Expected non-empty assistant content" + + +def test_chat_streaming(router: str) -> None: + """POST /v1/chat/completions (stream=True) returns >=2 SSE chunks incl. [DONE].""" + payload = { + "model": MODEL, + "messages": [{"role": "user", "content": "Say hi."}], + "max_tokens": 10, + "stream": True, + } + chunks: list[str] = [] + with httpx.stream( + "POST", + f"{router}/v1/chat/completions", + json=payload, + timeout=60, + ) as resp: + assert resp.status_code == 200, resp.read().decode() + for line in resp.iter_lines(): + line = line.strip() + if line.startswith("data:"): + chunks.append(line) + + assert len(chunks) >= 2, f"Expected >=2 SSE chunks, got {len(chunks)}: {chunks}" + assert any( + "[DONE]" in c for c in chunks + ), f"No [DONE] chunk found in SSE stream: {chunks}" diff --git a/experimental/sgl-router/tests/e2e/test_tokenize_smoke.py b/experimental/sgl-router/tests/e2e/test_tokenize_smoke.py new file mode 100644 index 000000000000..cee049f87ebc --- /dev/null +++ b/experimental/sgl-router/tests/e2e/test_tokenize_smoke.py @@ -0,0 +1,37 @@ +""" +Smoke test for /v1/tokenize and /v1/detokenize round-trip. +""" + +from __future__ import annotations + +import httpx + +MODEL = "Qwen/Qwen3-0.6B" +TEXT = "Hello, world!" + + +def test_tokenize_round_trip(router: str) -> None: + """POST /v1/tokenize then /v1/detokenize must recover the original text.""" + # Tokenize + tok_resp = httpx.post( + f"{router}/v1/tokenize", + json={"model": MODEL, "prompt": TEXT}, + timeout=30, + ) + assert tok_resp.status_code == 200, tok_resp.text + tokens = tok_resp.json()["tokens"] + assert ( + isinstance(tokens, list) and len(tokens) > 0 + ), f"Expected non-empty token list, got: {tokens}" + + # Detokenize + detok_resp = httpx.post( + f"{router}/v1/detokenize", + json={"model": MODEL, "tokens": tokens}, + timeout=30, + ) + assert detok_resp.status_code == 200, detok_resp.text + recovered = detok_resp.json()["text"] + assert ( + TEXT in recovered or recovered in TEXT + ), f"Round-trip mismatch: original={TEXT!r}, recovered={recovered!r}" diff --git a/experimental/sgl-router/tests/fixtures/kv_events_hash_parity.json b/experimental/sgl-router/tests/fixtures/kv_events_hash_parity.json new file mode 100644 index 000000000000..e33c76a60586 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/kv_events_hash_parity.json @@ -0,0 +1,232 @@ +[ + { + "name": "single_full_block", + "tokens": [ + 1, + 2, + 3, + 4 + ], + "block_size": 4, + "expected_i64_hashes": [ + -3488128144981237669 + ] + }, + { + "name": "partial_last_block", + "tokens": [ + 1, + 2, + 3, + 4, + 5 + ], + "block_size": 4, + "expected_i64_hashes": [ + -3488128144981237669, + -3787494577174227566 + ] + }, + { + "name": "multi_block", + "tokens": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80 + ], + "block_size": 2, + "expected_i64_hashes": [ + 978178666101069530, + -895308556211281782, + -8033692805846017938, + 835415944263129316 + ] + }, + { + "name": "empty_tokens", + "tokens": [], + "block_size": 4, + "expected_i64_hashes": [] + }, + { + "name": "block_size_one", + "tokens": [ + 7, + 8, + 9 + ], + "block_size": 1, + "expected_i64_hashes": [ + -1702009526849766914, + 903318264012984157, + -8265893088400908305 + ] + }, + { + "name": "odd_boundary", + "tokens": [ + 100, + 200, + 300, + 400, + 500, + 600, + 700 + ], + "block_size": 3, + "expected_i64_hashes": [ + -7293070039731858224, + -5869816562584529365, + 7513319606423624955 + ] + }, + { + "name": "long_sequence", + "tokens": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128 + ], + "block_size": 16, + "expected_i64_hashes": [ + 8635429971592222890, + 1256577331724852459, + 5689809685380680247, + 3927976462491479733, + 5639345789331840936, + -4601255381563393033, + 3368460852864325515, + 1233425155141659070 + ] + } +] diff --git a/experimental/sgl-router/tests/fixtures/tiny_tokenizer.json b/experimental/sgl-router/tests/fixtures/tiny_tokenizer.json new file mode 100644 index 000000000000..e2c428e80f60 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tiny_tokenizer.json @@ -0,0 +1 @@ +{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":50256,"special":true,"content":"<|endoftext|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true}],"normalizer":null,"pre_tokenizer":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"post_processor":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":false},"decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true},"model":{"dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":{"!":0,"\"":1,"#":2,"$":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"<":27,"=":28,">":29,"?":30,"@":31,"A":32,"B":33,"C":34,"D":35,"E":36,"F":37,"G":38,"H":39,"I":40,"J":41,"K":42,"L":43,"M":44,"N":45,"O":46,"P":47,"Q":48,"R":49,"S":50,"T":51,"U":52,"V":53,"W":54,"X":55,"Y":56,"Z":57,"[":58,"\\":59,"]":60,"^":61,"_":62,"`":63,"a":64,"b":65,"c":66,"d":67,"e":68,"f":69,"g":70,"h":71,"i":72,"j":73,"k":74,"l":75,"m":76,"n":77,"o":78,"p":79,"q":80,"r":81,"s":82,"t":83,"u":84,"v":85,"w":86,"x":87,"y":88,"z":89,"{":90,"|":91,"}":92,"~":93,"¡":94,"¢":95,"£":96,"¤":97,"¥":98,"¦":99,"§":100,"¨":101,"©":102,"ª":103,"«":104,"¬":105,"®":106,"¯":107,"°":108,"±":109,"²":110,"³":111,"´":112,"µ":113,"¶":114,"·":115,"¸":116,"¹":117,"º":118,"»":119,"¼":120,"½":121,"¾":122,"¿":123,"À":124,"Á":125,"Â":126,"Ã":127,"Ä":128,"Å":129,"Æ":130,"Ç":131,"È":132,"É":133,"Ê":134,"Ë":135,"Ì":136,"Í":137,"Î":138,"Ï":139,"Ð":140,"Ñ":141,"Ò":142,"Ó":143,"Ô":144,"Õ":145,"Ö":146,"×":147,"Ø":148,"Ù":149,"Ú":150,"Û":151,"Ü":152,"Ý":153,"Þ":154,"ß":155,"à":156,"á":157,"â":158,"ã":159,"ä":160,"å":161,"æ":162,"ç":163,"è":164,"é":165,"ê":166,"ë":167,"ì":168,"í":169,"î":170,"ï":171,"ð":172,"ñ":173,"ò":174,"ó":175,"ô":176,"õ":177,"ö":178,"÷":179,"ø":180,"ù":181,"ú":182,"û":183,"ü":184,"ý":185,"þ":186,"ÿ":187,"Ā":188,"ā":189,"Ă":190,"ă":191,"Ą":192,"ą":193,"Ć":194,"ć":195,"Ĉ":196,"ĉ":197,"Ċ":198,"ċ":199,"Č":200,"č":201,"Ď":202,"ď":203,"Đ":204,"đ":205,"Ē":206,"ē":207,"Ĕ":208,"ĕ":209,"Ė":210,"ė":211,"Ę":212,"ę":213,"Ě":214,"ě":215,"Ĝ":216,"ĝ":217,"Ğ":218,"ğ":219,"Ġ":220,"ġ":221,"Ģ":222,"ģ":223,"Ĥ":224,"ĥ":225,"Ħ":226,"ħ":227,"Ĩ":228,"ĩ":229,"Ī":230,"ī":231,"Ĭ":232,"ĭ":233,"Į":234,"į":235,"İ":236,"ı":237,"IJ":238,"ij":239,"Ĵ":240,"ĵ":241,"Ķ":242,"ķ":243,"ĸ":244,"Ĺ":245,"ĺ":246,"Ļ":247,"ļ":248,"Ľ":249,"ľ":250,"Ŀ":251,"ŀ":252,"Ł":253,"ł":254,"Ń":255,"Ġt":256,"Ġa":257,"he":258,"in":259,"re":260,"on":261,"Ġthe":262,"er":263,"Ġs":264,"at":265,"Ġw":266,"Ġo":267,"en":268,"Ġc":269,"it":270,"is":271,"an":272,"or":273,"es":274,"Ġb":275,"ed":276,"Ġf":277,"ing":278,"Ġp":279,"ou":280,"Ġan":281,"al":282,"ar":283,"Ġto":284,"Ġm":285,"Ġof":286,"Ġin":287,"Ġd":288,"Ġh":289,"Ġand":290,"ic":291,"as":292,"le":293,"Ġth":294,"ion":295,"om":296,"ll":297,"ent":298,"Ġn":299,"Ġl":300,"st":301,"Ġre":302,"ve":303,"Ġe":304,"ro":305,"ly":306,"Ġbe":307,"Ġg":308,"ĠT":309,"ct":310,"ĠS":311,"id":312,"ot":313,"ĠI":314,"ut":315,"et":316,"ĠA":317,"Ġis":318,"Ġon":319,"im":320,"am":321,"ow":322,"ay":323,"ad":324,"se":325,"Ġthat":326,"ĠC":327,"ig":328,"Ġfor":329,"ac":330,"Ġy":331,"ver":332,"ur":333,"Ġu":334,"ld":335,"Ġst":336,"ĠM":337,"'s":338,"Ġhe":339,"Ġit":340,"ation":341,"ith":342,"ir":343,"ce":344,"Ġyou":345,"il":346,"ĠB":347,"Ġwh":348,"ol":349,"ĠP":350,"Ġwith":351,"Ġ1":352,"ter":353,"ch":354,"Ġas":355,"Ġwe":356,"Ġ(":357,"nd":358,"ill":359,"ĠD":360,"if":361,"Ġ2":362,"ag":363,"ers":364,"ke":365,"Ġ\"":366,"ĠH":367,"em":368,"Ġcon":369,"ĠW":370,"ĠR":371,"her":372,"Ġwas":373,"Ġr":374,"od":375,"ĠF":376,"ul":377,"ate":378,"Ġat":379,"ri":380,"pp":381,"ore":382,"ĠThe":383,"Ġse":384,"us":385,"Ġpro":386,"Ġha":387,"um":388,"Ġare":389,"Ġde":390,"ain":391,"and":392,"Ġor":393,"igh":394,"est":395,"ist":396,"ab":397,"rom":398,"ĠN":399,"th":400,"Ġcom":401,"ĠG":402,"un":403,"op":404,"00":405,"ĠL":406,"Ġnot":407,"ess":408,"Ġex":409,"Ġv":410,"res":411,"ĠE":412,"ew":413,"ity":414,"ant":415,"Ġby":416,"el":417,"os":418,"ort":419,"oc":420,"qu":421,"Ġfrom":422,"Ġhave":423,"Ġsu":424,"ive":425,"ould":426,"Ġsh":427,"Ġthis":428,"nt":429,"ra":430,"pe":431,"ight":432,"art":433,"ment":434,"Ġal":435,"ust":436,"end":437,"--":438,"all":439,"ĠO":440,"ack":441,"Ġch":442,"Ġle":443,"ies":444,"red":445,"ard":446,"âĢ":447,"out":448,"ĠJ":449,"Ġab":450,"ear":451,"iv":452,"ally":453,"our":454,"ost":455,"gh":456,"pt":457,"Ġpl":458,"ast":459,"Ġcan":460,"ak":461,"ome":462,"ud":463,"The":464,"Ġhis":465,"Ġdo":466,"Ġgo":467,"Ġhas":468,"ge":469,"'t":470,"ĠU":471,"rou":472,"Ġsa":473,"Ġj":474,"Ġbut":475,"Ġwor":476,"Ġall":477,"ect":478,"Ġk":479,"ame":480,"Ġwill":481,"ok":482,"Ġwhe":483,"Ġthey":484,"ide":485,"01":486,"ff":487,"ich":488,"pl":489,"ther":490,"Ġtr":491,"..":492,"Ġint":493,"ie":494,"ure":495,"age":496,"Ġne":497,"ial":498,"ap":499,"ine":500,"ice":501,"Ġme":502,"Ġout":503,"ans":504,"one":505,"ong":506,"ions":507,"Ġwho":508,"ĠK":509,"Ġup":510,"Ġtheir":511,"Ġad":512,"Ġ3":513,"Ġus":514,"ated":515,"ous":516,"Ġmore":517,"ue":518,"og":519,"ĠSt":520,"ind":521,"ike":522,"Ġso":523,"ime":524,"per":525,".\"":526,"ber":527,"iz":528,"act":529,"Ġone":530,"Ġsaid":531,"Ġ-":532,"are":533,"Ġyour":534,"cc":535,"ĠTh":536,"Ġcl":537,"ep":538,"ake":539,"able":540,"ip":541,"Ġcont":542,"Ġwhich":543,"ia":544,"Ġim":545,"Ġabout":546,"Ġwere":547,"very":548,"ub":549,"Ġhad":550,"Ġen":551,"Ġcomp":552,",\"":553,"ĠIn":554,"Ġun":555,"Ġag":556,"ire":557,"ace":558,"au":559,"ary":560,"Ġwould":561,"ass":562,"ry":563,"ĠâĢ":564,"cl":565,"ook":566,"ere":567,"so":568,"ĠV":569,"ign":570,"ib":571,"Ġoff":572,"Ġte":573,"ven":574,"ĠY":575,"ile":576,"ose":577,"ite":578,"orm":579,"Ġ201":580,"Ġres":581,"Ġman":582,"Ġper":583,"Ġother":584,"ord":585,"ult":586,"Ġbeen":587,"Ġlike":588,"ase":589,"ance":590,"ks":591,"ays":592,"own":593,"ence":594,"Ġdis":595,"ction":596,"Ġany":597,"Ġapp":598,"Ġsp":599,"int":600,"ress":601,"ations":602,"ail":603,"Ġ4":604,"ical":605,"Ġthem":606,"Ġher":607,"ount":608,"ĠCh":609,"Ġar":610,"Ġif":611,"Ġthere":612,"Ġpe":613,"Ġyear":614,"av":615,"Ġmy":616,"Ġsome":617,"Ġwhen":618,"ough":619,"ach":620,"Ġthan":621,"ru":622,"ond":623,"ick":624,"Ġover":625,"vel":626,"Ġqu":627,"ĊĊ":628,"Ġsc":629,"reat":630,"ree":631,"ĠIt":632,"ound":633,"port":634,"Ġalso":635,"Ġpart":636,"fter":637,"Ġkn":638,"Ġbec":639,"Ġtime":640,"ens":641,"Ġ5":642,"ople":643,"Ġwhat":644,"Ġno":645,"du":646,"mer":647,"ang":648,"Ġnew":649,"----":650,"Ġget":651,"ory":652,"ition":653,"ings":654,"Ġjust":655,"Ġinto":656,"Ġ0":657,"ents":658,"ove":659,"te":660,"Ġpeople":661,"Ġpre":662,"Ġits":663,"Ġrec":664,"Ġtw":665,"ian":666,"irst":667,"ark":668,"ors":669,"Ġwork":670,"ade":671,"ob":672,"Ġshe":673,"Ġour":674,"wn":675,"ink":676,"lic":677,"Ġ19":678,"ĠHe":679,"ish":680,"nder":681,"ause":682,"Ġhim":683,"ons":684,"Ġ[":685,"Ġro":686,"form":687,"ild":688,"ates":689,"vers":690,"Ġonly":691,"oll":692,"Ġspe":693,"ck":694,"ell":695,"amp":696,"Ġacc":697,"Ġbl":698,"ious":699,"urn":700,"ft":701,"ood":702,"Ġhow":703,"hed":704,"Ġ'":705,"Ġafter":706,"aw":707,"Ġatt":708,"ov":709,"ne":710,"Ġplay":711,"erv":712,"ict":713,"Ġcould":714,"itt":715,"Ġam":716,"Ġfirst":717,"Ġ6":718,"Ġact":719,"Ġ$":720,"ec":721,"hing":722,"ual":723,"ull":724,"Ġcomm":725,"oy":726,"old":727,"ces":728,"ater":729,"Ġfe":730,"Ġbet":731,"we":732,"iff":733,"Ġtwo":734,"ock":735,"Ġback":736,").":737,"ident":738,"Ġunder":739,"rough":740,"sel":741,"xt":742,"Ġmay":743,"round":744,"Ġpo":745,"ph":746,"iss":747,"Ġdes":748,"Ġmost":749,"Ġdid":750,"Ġadd":751,"ject":752,"Ġinc":753,"fore":754,"Ġpol":755,"ont":756,"Ġagain":757,"clud":758,"tern":759,"Ġknow":760,"Ġneed":761,"Ġcons":762,"Ġco":763,"Ġ.":764,"Ġwant":765,"Ġsee":766,"Ġ7":767,"ning":768,"iew":769,"ĠThis":770,"ced":771,"Ġeven":772,"Ġind":773,"ty":774,"ĠWe":775,"ath":776,"Ġthese":777,"Ġpr":778,"Ġuse":779,"Ġbecause":780,"Ġfl":781,"ng":782,"Ġnow":783,"ĠâĢĵ":784,"com":785,"ise":786,"Ġmake":787,"Ġthen":788,"ower":789,"Ġevery":790,"ĠUn":791,"Ġsec":792,"oss":793,"uch":794,"Ġem":795,"Ġ=":796,"ĠRe":797,"ied":798,"rit":799,"Ġinv":800,"lect":801,"Ġsupp":802,"ating":803,"Ġlook":804,"man":805,"pect":806,"Ġ8":807,"row":808,"Ġbu":809,"Ġwhere":810,"ific":811,"Ġyears":812,"ily":813,"Ġdiff":814,"Ġshould":815,"Ġrem":816,"Th":817,"In":818,"Ġev":819,"day":820,"'re":821,"rib":822,"Ġrel":823,"ss":824,"Ġdef":825,"Ġright":826,"Ġsy":827,"),":828,"les":829,"000":830,"hen":831,"Ġthrough":832,"ĠTr":833,"__":834,"Ġway":835,"Ġdon":836,"Ġ,":837,"Ġ10":838,"ased":839,"Ġass":840,"ublic":841,"Ġreg":842,"ĠAnd":843,"ix":844,"Ġvery":845,"Ġinclud":846,"other":847,"Ġimp":848,"oth":849,"Ġsub":850,"ĠâĢĶ":851,"Ġbeing":852,"arg":853,"ĠWh":854,"==":855,"ible":856,"Ġdoes":857,"ange":858,"ram":859,"Ġ9":860,"ert":861,"ps":862,"ited":863,"ational":864,"Ġbr":865,"Ġdown":866,"Ġmany":867,"aking":868,"Ġcall":869,"uring":870,"ities":871,"Ġph":872,"ics":873,"als":874,"Ġdec":875,"ative":876,"ener":877,"Ġbefore":878,"ility":879,"Ġwell":880,"Ġmuch":881,"erson":882,"Ġthose":883,"Ġsuch":884,"Ġke":885,"Ġend":886,"ĠBut":887,"ason":888,"ting":889,"Ġlong":890,"ef":891,"Ġthink":892,"ys":893,"Ġbel":894,"Ġsm":895,"its":896,"ax":897,"Ġown":898,"Ġprov":899,"Ġset":900,"ife":901,"ments":902,"ble":903,"ward":904,"Ġshow":905,"Ġpres":906,"ms":907,"omet":908,"Ġob":909,"Ġsay":910,"ĠSh":911,"ts":912,"ful":913,"Ġeff":914,"Ġgu":915,"Ġinst":916,"und":917,"ren":918,"cess":919,"Ġent":920,"ĠYou":921,"Ġgood":922,"Ġstart":923,"ince":924,"Ġmade":925,"tt":926,"stem":927,"olog":928,"up":929,"Ġ|":930,"ump":931,"Ġhel":932,"vern":933,"ular":934,"ually":935,"Ġac":936,"Ġmon":937,"Ġlast":938,"Ġ200":939,"10":940,"Ġstud":941,"ures":942,"ĠAr":943,"self":944,"ars":945,"meric":946,"ues":947,"cy":948,"Ġmin":949,"ollow":950,"Ġcol":951,"io":952,"Ġmod":953,"Ġcount":954,"ĠCom":955,"hes":956,"Ġfin":957,"air":958,"ier":959,"âĢĶ":960,"read":961,"ank":962,"atch":963,"ever":964,"Ġstr":965,"Ġpoint":966,"ork":967,"ĠNew":968,"Ġsur":969,"ool":970,"alk":971,"ement":972,"Ġused":973,"ract":974,"ween":975,"Ġsame":976,"oun":977,"ĠAl":978,"ci":979,"Ġdiffere":980,"Ġwhile":981,"--------":982,"Ġgame":983,"cept":984,"Ġsim":985,"...":986,"Ġinter":987,"ek":988,"Ġreport":989,"Ġprodu":990,"Ġstill":991,"led":992,"ah":993,"Ġhere":994,"Ġworld":995,"Ġthough":996,"Ġnum":997,"arch":998,"imes":999,"ale":1000,"ĠSe":1001,"ĠIf":1002,"//":1003,"ĠLe":1004,"Ġret":1005,"Ġref":1006,"Ġtrans":1007,"ner":1008,"ution":1009,"ters":1010,"Ġtake":1011,"ĠCl":1012,"Ġconf":1013,"way":1014,"ave":1015,"Ġgoing":1016,"Ġsl":1017,"ug":1018,"ĠAmeric":1019,"Ġspec":1020,"Ġhand":1021,"Ġbetween":1022,"ists":1023,"ĠDe":1024,"oot":1025,"It":1026,"Ġear":1027,"Ġagainst":1028,"Ġhigh":1029,"gan":1030,"az":1031,"ather":1032,"Ġexp":1033,"Ġop":1034,"Ġins":1035,"Ġgr":1036,"Ġhelp":1037,"Ġrequ":1038,"ets":1039,"ins":1040,"ĠPro":1041,"ism":1042,"Ġfound":1043,"land":1044,"ata":1045,"uss":1046,"ames":1047,"Ġperson":1048,"Ġgreat":1049,"pr":1050,"Ġsign":1051,"ĠAn":1052,"'ve":1053,"Ġsomet":1054,"Ġser":1055,"hip":1056,"Ġrun":1057,"Ġ:":1058,"Ġter":1059,"irect":1060,"Ġfollow":1061,"Ġdet":1062,"ices":1063,"Ġfind":1064,"12":1065,"Ġmem":1066,"Ġcr":1067,"ered":1068,"ex":1069,"Ġext":1070,"uth":1071,"ense":1072,"co":1073,"Ġteam":1074,"ving":1075,"ouse":1076,"ash":1077,"att":1078,"ved":1079,"Ġsystem":1080,"ĠAs":1081,"der":1082,"ives":1083,"min":1084,"Ġlead":1085,"ĠBl":1086,"cent":1087,"Ġaround":1088,"Ġgovern":1089,"Ġcur":1090,"velop":1091,"any":1092,"Ġcour":1093,"alth":1094,"ages":1095,"ize":1096,"Ġcar":1097,"ode":1098,"Ġlaw":1099,"Ġread":1100,"'m":1101,"con":1102,"Ġreal":1103,"Ġsupport":1104,"Ġ12":1105,"....":1106,"Ġreally":1107,"ness":1108,"Ġfact":1109,"Ġday":1110,"Ġboth":1111,"ying":1112,"Ġserv":1113,"ĠFor":1114,"Ġthree":1115,"Ġwom":1116,"Ġmed":1117,"ody":1118,"ĠThey":1119,"50":1120,"Ġexper":1121,"ton":1122,"Ġeach":1123,"akes":1124,"Ġche":1125,"Ġcre":1126,"ines":1127,"Ġrep":1128,"19":1129,"gg":1130,"illion":1131,"Ġgrou":1132,"ute":1133,"ik":1134,"We":1135,"get":1136,"ER":1137,"Ġmet":1138,"Ġsays":1139,"ox":1140,"Ġduring":1141,"ern":1142,"ized":1143,"ared":1144,"Ġfam":1145,"ically":1146,"Ġhapp":1147,"ĠIs":1148,"Ġchar":1149,"med":1150,"vent":1151,"Ġgener":1152,"ient":1153,"ple":1154,"iet":1155,"rent":1156,"11":1157,"ves":1158,"ption":1159,"Ġ20":1160,"formation":1161,"Ġcor":1162,"Ġoffic":1163,"ield":1164,"Ġtoo":1165,"ision":1166,"Ġinf":1167,"ĠZ":1168,"the":1169,"oad":1170,"Ġpublic":1171,"Ġprog":1172,"ric":1173,"**":1174,"Ġwar":1175,"Ġpower":1176,"view":1177,"Ġfew":1178,"Ġloc":1179,"Ġdifferent":1180,"Ġstate":1181,"Ġhead":1182,"'ll":1183,"Ġposs":1184,"Ġstat":1185,"ret":1186,"ants":1187,"Ġval":1188,"Ġiss":1189,"Ġcle":1190,"ivers":1191,"anc":1192,"Ġexpl":1193,"Ġanother":1194,"ĠQ":1195,"Ġav":1196,"thing":1197,"nce":1198,"Wh":1199,"Ġchild":1200,"Ġsince":1201,"ired":1202,"less":1203,"Ġlife":1204,"Ġdevelop":1205,"ittle":1206,"Ġdep":1207,"Ġpass":1208,"ãĥ":1209,"Ġturn":1210,"orn":1211,"This":1212,"bers":1213,"ross":1214,"ĠAd":1215,"Ġfr":1216,"Ġresp":1217,"Ġsecond":1218,"oh":1219,"Ġ/":1220,"Ġdisc":1221,"Ġ&":1222,"Ġsomething":1223,"Ġcomple":1224,"Ġed":1225,"Ġfil":1226,"Ġmonth":1227,"aj":1228,"uc":1229,"Ġgovernment":1230,"Ġwithout":1231,"Ġleg":1232,"Ġdist":1233,"Ġput":1234,"Ġquest":1235,"ann":1236,"Ġprot":1237,"20":1238,"Ġnever":1239,"ience":1240,"Ġlevel":1241,"Ġart":1242,"Ġthings":1243,"Ġmight":1244,"Ġeffect":1245,"Ġcontro":1246,"Ġcent":1247,"Ġ18":1248,"Ġallow":1249,"Ġbelie":1250,"chool":1251,"ott":1252,"Ġincre":1253,"Ġfeel":1254,"Ġresult":1255,"Ġlot":1256,"Ġfun":1257,"ote":1258,"Ġty":1259,"erest":1260,"Ġcontin":1261,"Ġusing":1262,"Ġbig":1263,"201":1264,"Ġask":1265,"Ġbest":1266,"Ġ)":1267,"IN":1268,"Ġopp":1269,"30":1270,"Ġnumber":1271,"iness":1272,"St":1273,"lease":1274,"Ġca":1275,"Ġmust":1276,"Ġdirect":1277,"Ġgl":1278,"Ġ<":1279,"Ġopen":1280,"Ġpost":1281,"Ġcome":1282,"Ġseem":1283,"ording":1284,"Ġweek":1285,"ately":1286,"ital":1287,"Ġel":1288,"riend":1289,"Ġfar":1290,"Ġtra":1291,"inal":1292,"Ġpri":1293,"ĠUS":1294,"Ġplace":1295,"Ġform":1296,"Ġtold":1297,"\":":1298,"ains":1299,"ature":1300,"ĠTrump":1301,"Ġstand":1302,"Ġ#":1303,"ider":1304,"ĠFr":1305,"Ġnext":1306,"Ġsoc":1307,"Ġpur":1308,"Ġlet":1309,"Ġlittle":1310,"Ġhum":1311,"Ġi":1312,"ron":1313,"15":1314,"Ġ15":1315,"Ġcommun":1316,"Ġmark":1317,"ĠThere":1318,"Ġwr":1319,"ĠThat":1320,"Ġinformation":1321,"ways":1322,"Ġbus":1323,"app":1324,"Ġinvest":1325,"me":1326,"Ġhard":1327,"ained":1328,"ead":1329,"Ġimport":1330,"Ġappro":1331,"Ġtest":1332,"Ġtri":1333,"Ġrest":1334,"osed":1335,"Ġfull":1336,"Ġcare":1337,"ĠSp":1338,"Ġcase":1339,"ON":1340,"Ġsk":1341,"Ġless":1342,"Ġ+":1343,"Ġpartic":1344,"ĠPl":1345,"ably":1346,"uck":1347,"ished":1348,"chn":1349,"be":1350,"Ġlist":1351,"ator":1352,"Ġtop":1353,"Ġadv":1354,"ĠBe":1355,"ruct":1356,"Ġdem":1357,"ration":1358,"ling":1359,"gy":1360,"reen":1361,"ger":1362,"Ġhome":1363,"Ġleft":1364,"Ġbetter":1365,"Ġdata":1366,"Ġ11":1367,"Ġattack":1368,"Ġproble":1369,"line":1370,"ards":1371,"Ġbeh":1372,"ral":1373,"ĠHow":1374,"ĠShe":1375,"arge":1376,"Ġ--":1377,"://":1378,"Ġbro":1379,"ĠPh":1380,"ats":1381,"Ġbuild":1382,"ww":1383,"ided":1384,"aim":1385,"ases":1386,"ency":1387,"Ġmain":1388,"ined":1389,"Ġincluding":1390,"Ġ{":1391,"Ġgot":1392,"Ġinterest":1393,"Ġkeep":1394,"ĠX":1395,"Ġeas":1396,"aining":1397,"Ġclass":1398,"â̦":1399,"ĠNo":1400,"Ġvar":1401,"Ġsmall":1402,"ample":1403,"AT":1404,"Ġide":1405,"ĠSo":1406,"Ġrece":1407,"Ġpolit":1408,"Ġmov":1409,"Ġplan":1410,"Ġpercent":1411,"iving":1412,"Ġcamp":1413,"Ġpay":1414,"14":1415,"sc":1416,"ised":1417,"Ġunt":1418,"oney":1419,"ploy":1420,"====":1421,"Ġdidn":1422,"ĠInd":1423,"els":1424,"ertain":1425,"Ġpos":1426,"____":1427,"iver":1428,"Ġprocess":1429,"Ġprogram":1430,"ified":1431,"ĠRep":1432,"16":1433,"uro":1434,"ology":1435,"atter":1436,"ina":1437,"Ġname":1438,"ĠAll":1439,"Ġfour":1440,"Ġreturn":1441,"vious":1442,"bs":1443,"Ġcalled":1444,"Ġmove":1445,"ĠSc":1446,"ird":1447,"Ġgroup":1448,"Ġbre":1449,"Ġmen":1450,"Ġcap":1451,"ten":1452,"ee":1453,"Ġdri":1454,"leg":1455,"here":1456,"uthor":1457,"Ġpat":1458,"Ġcurrent":1459,"ides":1460,"Ġpop":1461,"to":1462,"ention":1463,"Ġalways":1464,"Ġmil":1465,"Ġwomen":1466,"Ġ16":1467,"Ġold":1468,"iven":1469,"raph":1470,"ĠOr":1471,"ror":1472,"ently":1473,"Ġnear":1474,"ĠEx":1475,"ream":1476,"sh":1477,"Ġ14":1478,"Ġfree":1479,"ission":1480,"stand":1481,"ĠCon":1482,"ality":1483,"used":1484,"13":1485,"Ġdesign":1486,"Ġchange":1487,"Ġchang":1488,"Ġbo":1489,"Ġvis":1490,"ember":1491,"Ġbook":1492,"ready":1493,"Ġkill":1494,"25":1495,"pped":1496,"Ġaway":1497,"Ġable":1498,"Ġcountry":1499,"Ġconst":1500,"arn":1501,"Ġorder":1502,"AR":1503,"ior":1504,"ium":1505,"orth":1506,"18":1507,"ailable":1508,"Ġsw":1509,"Ġmillion":1510,"Ġ13":1511,"atic":1512,"ted":1513,"ĠGo":1514,"Ġoper":1515,"eng":1516,"Ġthing":1517,"ajor":1518,"conom":1519,"ĠComm":1520,"Ġwhy":1521,"ured":1522,"ural":1523,"Ġschool":1524,"by":1525,"ĠMar":1526,"Ġaff":1527,"Ġdays":1528,"Ġann":1529,"ush":1530,"ane":1531,"If":1532,"eg":1533,"Ġprof":1534,"Ġhealth":1535,"outh":1536,"But":1537,"ional":1538,".,":1539,"Ġsol":1540,"Ġalready":1541,"Ġ30":1542,"Ġcharact":1543,"He":1544,"Ġfriend":1545,"ES":1546,"ians":1547,"icle":1548,"'d":1549,"ĠOn":1550,"Ġleast":1551,"Ġprom":1552,"Ġdr":1553,"Ġhist":1554,"ither":1555,"Ġest":1556,"iqu":1557,"17":1558,"son":1559,"Ġtell":1560,"Ġtalk":1561,"ohn":1562,"oint":1563,"lection":1564,"AN":1565,"Ġuntil":1566,"augh":1567,"Ġlater":1568,"Ġve":1569,"Ġview":1570,"ending":1571,"ived":1572,"Ġword":1573,"ware":1574,"Ġcost":1575,"Ġenough":1576,"Ġgive":1577,"ĠUnited":1578,"Ġtechn":1579,"arent":1580,"OR":1581,"Ġpar":1582,"ĠDr":1583,"Ġ2016":1584,"rist":1585,"ering":1586,"ĠÂ":1587,"Ġlarge":1588,"side":1589,"acy":1590,"ccess":1591,"Ġwin":1592,"Ġimportant":1593,"Ġ199":1594,"Ġdoesn":1595,"Ġ17":1596,"Ġbusiness":1597,"Ġclear":1598,"Ġrese":1599,"\",":1600,"ury":1601,"Ġequ":1602,"aster":1603,"alf":1604,"ĠAmerican":1605,"nect":1606,"Ġexpect":1607,"iversity":1608,"Ġocc":1609,"ĠFl":1610,"Ġkind":1611,"Ġmean":1612,"Ġpast":1613,"Ġdev":1614,"Ġbas":1615,"let":1616,"raft":1617,"Ġorgan":1618,"Ġdel":1619,"Ġperform":1620,"Ġstory":1621,"Ġseason":1622,"ĠCol":1623,"Ġclaim":1624,"Ġcame":1625,"Ġwithin":1626,"Ġline":1627,"Ġproject":1628,"ĠAt":1629,"Ġcontrol":1630,"ended":1631,"ĠSy":1632,"Ġair":1633,"ization":1634,"Ġ*":1635,"ley":1636,"Ġmoney":1637,"idd":1638,"You":1639,"for":1640,"Ġfamily":1641,"Ġmaking":1642,"Ġbit":1643,"Ġpolice":1644,"Ġhappen":1645,"Ġvers":1646,"ony":1647,"uff":1648,"ĠWhen":1649,"Ġsit":1650,"ideo":1651,"lf":1652,"ison":1653,"Ġsure":1654,"gin":1655,"Ġappear":1656,"Ġlight":1657,"Ġes":1658,"of":1659,"Ġwater":1660,"Ġtimes":1661,"not":1662,"Ġgrow":1663,"Ġcompany":1664,"ĠTe":1665,"ows":1666,"Ġmar":1667,"ource":1668,"iol":1669,"arm":1670,"br":1671,"Ġexample":1672,"Ġconc":1673,"Ġfore":1674,"ĠTo":1675,"pro":1676,"EN":1677,"ries":1678,"Ġ25":1679,"ĠCan":1680,"ney":1681,"Ġactually":1682,"Ġever":1683,"urity":1684,"aken":1685,"aps":1686,"Ġtax":1687,"Ġmajor":1688,"ama":1689,"Ġoften":1690,"eral":1691,"Ġhuman":1692,"Ġjob":1693,"ister":1694,"Ġavailable":1695,"ocr":1696,"enn":1697,"aid":1698,"ivid":1699,"Ġrecord":1700,"?\"":1701,"Ġsing":1702,"ĠAm":1703,"idence":1704,"Ġnews":1705,"ster":1706,"Ġeconom":1707,"Ġfollowing":1708,"ĠBr":1709,"ising":1710,"Ġhour":1711,"most":1712,"ument":1713,"Ġsex":1714,"Ġdesc":1715,"Ġbecome":1716,"ĠEd":1717,"Ġtook":1718,"Ġhaving":1719,"Ġproduct":1720,"ault":1721,"As":1722,"aring":1723,"Ġmeans":1724,"Ġhop":1725,"une":1726,"Ġcho":1727,"Ġcertain":1728,"Ġnon":1729,"Ġdeal":1730,"24":1731,"lement":1732,"oci":1733,"ene":1734,"Ġside":1735,"ĠPr":1736,"ĠMay":1737,"Ġreason":1738,"ued":1739,"ched":1740,"ulation":1741,"Ġelect":1742,"Ġofficial":1743,"Ġpossible":1744,"Ġhold":1745,"ands":1746,"ots":1747,"Ġcity":1748,"ories":1749,"Ġsever":1750,"Ġchildren":1751,"Ġonce":1752,"Ġactiv":1753,"ler":1754,"Ġnight":1755,"itions":1756,"ĠJohn":1757,"ape":1758,"play":1759,"Ġdone":1760,"Ġlim":1761,"Ġworking":1762,"ĠPres":1763,"orld":1764,"eb":1765,"ĠCo":1766,"Ġbody":1767,"ails":1768,"utes":1769,"ĠMr":1770,"Ġwhether":1771,"Ġauthor":1772,"rop":1773,"Ġproper":1774,"Ġseen":1775,");":1776,"Ġfac":1777,"ĠSu":1778,"Ġcond":1779,"iting":1780,"Ġcourse":1781,"Ġ}":1782,"----------------":1783,"aign":1784,"Ġevent":1785,"Ġeng":1786,"Ġpot":1787,"Ġintern":1788,"iam":1789,"Ġshort":1790,"empt":1791,"ãĤ":1792,"ĠGod":1793,"ilar":1794,"80":1795,"Ġorig":1796,"IS":1797,"ourn":1798,"ability":1799,"itive":1800,"Ġdam":1801,"Ġ100":1802,"Ġpress":1803,"Ġdoing":1804,"Ġprotect":1805,"ring":1806,"Ġthought":1807,"Ġquestion":1808,"rew":1809,"ĠWar":1810,"Ġseveral":1811,"ĠState":1812,"Ġgiven":1813,"Ġfund":1814,"ĠTw":1815,"Ġwent":1816,"ances":1817,"work":1818,"por":1819,"my":1820,"40":1821,"Ġarg":1822,"artment":1823,"ustom":1824,"Ġpolic":1825,"Ġmeet":1826,"Ġcreat":1827,"22":1828,"ĠStates":1829,"Ġgames":1830,"raw":1831,"uture":1832,"Ġunderstand":1833,"urs":1834,"ĠOb":1835,"lish":1836,"sy":1837,"Ġmakes":1838,"Ġwon":1839,"agon":1840,"Ġhtt":1841,"Ġlove":1842,"ential":1843,"Ġcomplete":1844,"par":1845,"ĠIm":1846,"AL":1847,"Ġaccount":1848,"Âł":1849,"ored":1850,"vert":1851,"Ġident":1852,"Ġ2015":1853,"Ġothers":1854,"ĠMin":1855,"iber":1856,"verage":1857,"There":1858,"itional":1859,"dd":1860,"Ġprob":1861,"Ġyoung":1862,"Ġalong":1863,"Ġaccording":1864,"Ġyet":1865,"Ġmembers":1866,"ĠWhat":1867,"oid":1868,"ĠMan":1869,"And":1870,"Ġamong":1871,"ai":1872,"Ġemploy":1873,"ĠRes":1874,"Ġ>":1875,"Ġinvol":1876,"Ġlow":1877,"af":1878,"ĠCar":1879,"Ġhig":1880,"ĠOne":1881,"ĠSec":1882,"ination":1883,"Ġlikely":1884,"Ġant":1885,"aged":1886,"ĠRuss":1887,"Ġben":1888,"Ġrele":1889,"For":1890,"back":1891,"ĠNot":1892,"Ġpresident":1893,"ball":1894,"Ġaccess":1895,"ividual":1896,"ĠDem":1897,"ĠEuro":1898,"60":1899,"Ġknown":1900,"irl":1901,"ĠGr":1902,"Ġearly":1903,"use":1904,"iety":1905,"âĢĵ":1906,"Ġfight":1907,"Ġsent":1908,"Ġtoday":1909,"Ġmarket":1910,"\".":1911,"Ġbased":1912,"Ġstrong":1913,"urther":1914,"Ġdeb":1915,"mber":1916,"Ġproblem":1917,"Ġdeath":1918,"Ġsocial":1919,"imate":1920,"AS":1921,"ortun":1922,"Ġcampaign":1923,"ery":1924,"Ch":1925,"Ġey":1926,"ially":1927,"Ġmus":1928,"wh":1929,"pos":1930,"Ġer":1931,"Ġsaf":1932,"Ġmonths":1933,"iron":1934,"Ġviol":1935,"Ġfive":1936,"Ġstre":1937,"Ġplayers":1938,"inc":1939,"ald":1940,"year":1941,"aun":1942,"Ġsuccess":1943,"Ġpresent":1944,"erence":1945,"Ġ2014":1946,"Ġsugg":1947,"Ġparticular":1948,"Ġtry":1949,"Ġsuggest":1950,"ĠChrist":1951,"ones":1952,"Ġpriv":1953,"23":1954,"Ġcrit":1955,"Ġland":1956,"Ġlocal":1957,"ify":1958,"29":1959,"Ġaut":1960,"ED":1961,"ĠGu":1962,"Ġmult":1963,"Ġpolitical":1964,"Ġasked":1965,"Ġformer":1966,"itter":1967,"ript":1968,"Ġclose":1969,"Ġpract":1970,"ĠYork":1971,"Ġgetting":1972,"Ġacross":1973,"Ġcomb":1974,"Ġbelieve":1975,"Ġz":1976,"Ġtoget":1977,"Ġtogether":1978,"ĠCent":1979,"irc":1980,"Ġindividual":1981,"ĠMc":1982,"27":1983,"isk":1984,"ĠEng":1985,"Ġface":1986,"Ġ24":1987,"Ġvalue":1988,"Ġarea":1989,"ev":1990,"Ġwrit":1991,"ĠPresident":1992,"Ġvot":1993,"Ġkey":1994,"Ġmom":1995,"put":1996,"Ġanything":1997,"Ġexperience":1998,"attle":1999,"Ġmind":2000,"aff":2001,"omm":2002,"Ġfuture":2003,"ged":2004,"Ġcut":2005,"Ġtot":2006,"itch":2007,"Ġvideo":2008,"Ġinvestig":2009,"Ġnet":2010,"ĠMy":2011,"rict":2012,"ien":2013,".)":2014,"Ġimpro":2015,"though":2016,"wards":2017,"Ġconnect":2018,"ĠMed":2019,"selves":2020,"ensive":2021,"mb":2022,"ober":2023,"ators":2024,"An":2025,"Ġ50":2026,"Ġredu":2027,"resent":2028,"Ġabove":2029,"Ġfre":2030,"ĠEurope":2031,"sw":2032,"Ġamount":2033,"ĠApp":2034,"Ġeither":2035,"Ġmilit":2036,"Ġanal":2037,"Ġfail":2038,"ĠEn":2039,"ales":2040,"Ġspecial":2041,"Ġblack":2042,"IT":2043,"cher":2044,"Ġlooking":2045,"Ġfire":2046,"yn":2047,"Ġalmost":2048,"oon":2049,"Ġstudy":2050,"Ġmiss":2051,"ches":2052,"rown":2053,"Ġtre":2054,"Ġcommunity":2055,"Ġmedia":2056,"Ġfood":2057,"Ġcomes":2058,"ĠUniversity":2059,"Ġsingle":2060,"What":2061,"uly":2062,"Ġhalf":2063,"ague":2064,"hod":2065,"ĠRepublic":2066,"Ġstarted":2067,"Ġquick":2068,"oto":2069,"book":2070,"Ġissue":2071,"itor":2072,"Ġelse":2073,"Ġconsider":2074,"26":2075,"rodu":2076,"Ġtaken":2077,"28":2078,"99":2079,"ĠWith":2080,"Ġtrue":2081,"Ġwa":2082,"Ġtrad":2083,"Ġago":2084,"Ġmess":2085,"ief":2086,"Ġadded":2087,"oke":2088,"Ġbad":2089,"Ġfav":2090,"33":2091,"Ġsimilar":2092,"ask":2093,"ĠDon":2094,"Ġcharacter":2095,"orts":2096,"ĠHouse":2097,"Ġreported":2098,"Ġtype":2099,"val":2100,"iod":2101,"ĠHowever":2102,"Ġtarg":2103,"Ġentire":2104,"pping":2105,"Ġhistory":2106,"Ġlive":2107,"ffic":2108,"........":2109,"ederal":2110,"Ġtrying":2111,"Ġdiscuss":2112,"ĠHar":2113,"aces":2114,"lished":2115,"Ġself":2116,"osp":2117,"rest":2118,"Ġroom":2119,"elt":2120,"Ġfall":2121,"olution":2122,"Ġet":2123,"Ġx":2124,"Ġisn":2125,"Ġidea":2126,"bo":2127,"Ġsound":2128,"ĠDep":2129,"Ġsomeone":2130,"cially":2131,"ully":2132,"Ġfoc":2133,"Ġobject":2134,"ift":2135,"aper":2136,"Ġplayer":2137,"Ġrather":2138,"Ġservice":2139,"ashing":2140,"ĠDo":2141,"ĠPart":2142,"rug":2143,"mon":2144,"ply":2145,"Ġmor":2146,"Ġnothing":2147,"Ġprovide":2148,"IC":2149,"ung":2150,"Ġparty":2151,"Ġexist":2152,"Ġmag":2153,"70":2154,"Ġrul":2155,"Ġhouse":2156,"Ġbehind":2157,"Ġhowever":2158,"ĠWorld":2159,"Ġsum":2160,"Ġapplic":2161,"Ġ;":2162,"Ġfunction":2163,"gr":2164,"ĠPol":2165,"Ġfront":2166,"200":2167,"Ġseries":2168,"Ġtem":2169,"Ġtyp":2170,"ills":2171,"Ġopt":2172,"Ġpoints":2173,"Ġbelow":2174,"itted":2175,"Ġspecific":2176,"Ġ2017":2177,"umb":2178,"Ġra":2179,"Ġprevious":2180,"Ġpret":2181,"reme":2182,"Ġcustom":2183,"Ġcourt":2184,"ĠMe":2185,"Ġrepl":2186,"Ġwhole":2187,"go":2188,"cer":2189,"Ġtreat":2190,"ĠAct":2191,"Ġprobably":2192,"Ġlearn":2193,"ender":2194,"ĠAss":2195,"Ġversion":2196,"now":2197,"Ġcheck":2198,"ĠCal":2199,"RE":2200,"minist":2201,"On":2202,"ources":2203,"Ġbenef":2204,"Ġdoc":2205,"Ġdeter":2206,"Ġenc":2207,"Ġsuper":2208,"Ġaddress":2209,"Ġvict":2210,"Ġ2013":2211,"Ġmeas":2212,"tr":2213,"Ġfield":2214,"When":2215,"Ġsignific":2216,"uge":2217,"Ġfeat":2218,"Ġcommon":2219,"load":2220,"Ġbegin":2221,"Ġbring":2222,"Ġaction":2223,"erman":2224,"Ġdescrib":2225,"Ġindust":2226,"Ġwanted":2227,"ried":2228,"ming":2229,"Ġattempt":2230,"45":2231,"fer":2232,"Ġdue":2233,"ression":2234,"##":2235,"Ġshall":2236,"Ġsix":2237,"oo":2238,"Ġstep":2239,"Ġpub":2240,"Ġhimself":2241,"Ġ23":2242,"Ġcop":2243,"Ġdest":2244,"Ġstop":2245,"AC":2246,"ibility":2247,"Ġlab":2248,"icult":2249,"Ġhours":2250,"Ġcreate":2251,"Ġfurther":2252,"ĠAmerica":2253,"ĠCity":2254,"Ġdou":2255,"head":2256,"ST":2257,"ĠNorth":2258,"cing":2259,"Ġnational":2260,"ule":2261,"ĠInst":2262,"Ġtaking":2263,"ĠQu":2264,"irt":2265,"Ġred":2266,"Ġresearch":2267,"viron":2268,"ĠGe":2269,"Ġbreak":2270,"ana":2271,"Ġspace":2272,"aterial":2273,"Ġrecent":2274,"ĠAb":2275,"Ġgeneral":2276,"Ġhit":2277,"Ġperiod":2278,"Ġeverything":2279,"ively":2280,"Ġphys":2281,"Ġsaying":2282,"anks":2283,"Ġcou":2284,"Ġcult":2285,"aced":2286,"eal":2287,"uation":2288,"Ġcoun":2289,"lu":2290,"Ġinclude":2291,"Ġposition":2292,"ĠAfter":2293,"ĠCanad":2294,"ĠEm":2295,"Ġimm":2296,"ĠRed":2297,"Ġpick":2298,"Ġcompl":2299,"Ġmatter":2300,"reg":2301,"ext":2302,"angu":2303,"isc":2304,"ole":2305,"aut":2306,"Ġcompet":2307,"eed":2308,"fect":2309,"Ġ21":2310,"ĠSen":2311,"ĠThese":2312,"asing":2313,"Ġcannot":2314,"Ġinit":2315,"Ġrelations":2316,"ached":2317,"Ġbar":2318,"Ġ40":2319,"ĠTH":2320,"Ġ2012":2321,"Ġvol":2322,"Ġground":2323,"Ġsecurity":2324,"Ġupd":2325,"ilt":2326,"35":2327,"Ġconcern":2328,"ĠJust":2329,"Ġwhite":2330,"Ġseems":2331,"ĠHer":2332,"pecially":2333,"ients":2334,"Ġannoun":2335,"Ġfig":2336,"ights":2337,"Ġstri":2338,"like":2339,"ids":2340,"Ġsus":2341,"Ġwatch":2342,"Ġâ":2343,"Ġwind":2344,"ĠCont":2345,"Ġitself":2346,"Ġmass":2347,"Al":2348,"yle":2349,"ique":2350,"ĠNational":2351,"Ġabs":2352,"Ġpack":2353,"Ġoutside":2354,"Ġanim":2355,"Ġpain":2356,"eter":2357,"Ġmanag":2358,"duct":2359,"ogn":2360,"Ġ]":2361,"ĠSept":2362,"sec":2363,"off":2364,"ĠJan":2365,"Ġfoot":2366,"ades":2367,"Ġthird":2368,"Ġmot":2369,"Ġevidence":2370,"inton":2371,"Ġthreat":2372,"apt":2373,"ples":2374,"cle":2375,"Ġlo":2376,"Ġdecl":2377,"Ġitem":2378,"medi":2379,"Ġrepresent":2380,"omb":2381,"amer":2382,"Ġsignificant":2383,"ograph":2384,"su":2385,"Ġcal":2386,"ires":2387,"0000":2388,"ID":2389,"AM":2390,"Ġsimply":2391,"Ġlonger":2392,"Ġfile":2393,"OT":2394,"che":2395,"So":2396,"ateg":2397,"org":2398,"ĠHis":2399,"Ġener":2400,"Ġdom":2401,"Ġupon":2402,"ili":2403,"\":\"":2404,"Ġthemselves":2405,"Ġcoming":2406,"Ġquite":2407,"Ġdifficult":2408,"ĠBar":2409,"ilities":2410,"rel":2411,"ends":2412,"cial":2413,"64":2414,"Ġwoman":2415,"rap":2416,"yr":2417,"Ġnecess":2418,"ips":2419,"Ġtext":2420,"Ġrequire":2421,"Ġmilitary":2422,"Ġreview":2423,"Ġrespons":2424,"75":2425,"Ġsubject":2426,"Ġinstead":2427,"Ġissues":2428,"Ġgen":2429,"\",\"":2430,"Ġminutes":2431,"Ġweap":2432,"ray":2433,"amed":2434,"time":2435,"bl":2436,"How":2437,"Ġcode":2438,"ĠSm":2439,"Ġhigher":2440,"ĠSte":2441,"ris":2442,"Ġpage":2443,"Ġstudents":2444,"ĠIntern":2445,"Ġmethod":2446,"ĠAug":2447,"ĠPer":2448,"ĠAg":2449,"Ġpolicy":2450,"ĠSw":2451,"Ġexec":2452,"Ġaccept":2453,"ume":2454,"ribut":2455,"Ġwords":2456,"Ġfinal":2457,"Ġchanges":2458,"ĠDemocr":2459,"Ġfriends":2460,"Ġrespect":2461,"Ġep":2462,"Ġcompan":2463,"ivil":2464,"Ġdamage":2465,"****":2466,"ogle":2467,"vironment":2468,"Ġneg":2469,"ental":2470,"Ġap":2471,"Ġtotal":2472,"ival":2473,"!\"":2474,"lim":2475,"Ġneeds":2476,"Ġagre":2477,"Ġdevelopment":2478,"Ġage":2479,"iple":2480,"21":2481,"Ġresults":2482,"ĠAf":2483,"Sh":2484,"Ġgun":2485,"ĠObama":2486,"roll":2487,"Ġ@":2488,"Ġrights":2489,"ĠBrit":2490,"Ġrunning":2491,"Ġwasn":2492,"Ġport":2493,"Ġrate":2494,"Ġpretty":2495,"Ġtarget":2496,"Ġsaw":2497,"Ġcirc":2498,"Ġworks":2499,"icro":2500,"alt":2501,"over":2502,"www":2503,"That":2504,"lier":2505,"Ġeveryone":2506,"ude":2507,"Ġpie":2508,"iddle":2509,"rael":2510,"Ġrad":2511,"Ġblock":2512,"Ġwalk":2513,"To":2514,"ãģ":2515,"nes":2516,"ĠAust":2517,"aul":2518,"rote":2519,"ĠSouth":2520,"ession":2521,"oph":2522,"Ġshows":2523,"Ġsite":2524,"Ġjo":2525,"Ġrisk":2526,"clus":2527,"lt":2528,"Ġinj":2529,"iding":2530,"ĠSpe":2531,"Ġchall":2532,"irm":2533,"Ġ22":2534,"itting":2535,"str":2536,"Ġhy":2537,"LE":2538,"key":2539,"Ġbegan":2540,"atur":2541,"ashington":2542,"lam":2543,"ĠDav":2544,"bit":2545,"Ġsize":2546,"ĠPar":2547,"38":2548,"ournal":2549,"face":2550,"Ġdecision":2551,"Ġlarg":2552,"Ġjud":2553,"rect":2554,"Ġcontinue":2555,"ĠOct":2556,"overed":2557,"ĠInt":2558,"========":2559,"Ġparent":2560,"ĠWill":2561,"Ġeasy":2562,"Ġdrug":2563,"anger":2564,"Ġsense":2565,"Ġdi":2566,"iday":2567,"Ġenergy":2568,"istic":2569,"Ġassoci":2570,"arter":2571,"obal":2572,"eks":2573,"ĠEl":2574,"urch":2575,"Ġgirl":2576,"oe":2577,"itle":2578,"Ġ28":2579,"ĠChe":2580,"Ġrequest":2581,"Ġsoon":2582,"Ġhost":2583,"ky":2584,"Ġstates":2585,"omes":2586,"Ġmaterial":2587,"lex":2588,"Ġmoment":2589,"Ġansw":2590,"onse":2591,"Ġespecially":2592,"Ġnorm":2593,"Ġservices":2594,"pite":2595,"ran":2596,"Ġrole":2597,"44":2598,"):":2599,"Ġcred":2600,"Cl":2601,"________":2602,"Ġmat":2603,"Ġlog":2604,"ĠClinton":2605,"OU":2606,"Ġoffice":2607,"Ġ26":2608,"Ġcharg":2609,"Ġtrack":2610,"ma":2611,"Ġheart":2612,"Ġball":2613,"Ġpersonal":2614,"Ġbuilding":2615,"na":2616,"set":2617,"body":2618,"ĠBlack":2619,"Ġincrease":2620,"itten":2621,"Ġneeded":2622,"36":2623,"32":2624,"=\"":2625,"Ġlost":2626,"Ġbecame":2627,"Ġgroups":2628,"ĠMus":2629,"Ġwrote":2630,"ĠPe":2631,"Ġprop":2632,"joy":2633,"é":2634,"ĠWhite":2635,"Ġdead":2636,".'":2637,"Ġhttp":2638,"Ġwebs":2639,"OS":2640,"Ġinside":2641,"Ġwrong":2642,"Ġstatement":2643,"Ġ...":2644,"yl":2645,"Ġfilm":2646,"Ġmusic":2647,"Ġshare":2648,"ification":2649,"Ġrelease":2650,"Ġforward":2651,"Ġstay":2652,"Ġcomput":2653,"itte":2654,"ser":2655,"Ġoriginal":2656,"Ġcard":2657,"Ġcand":2658,"Ġdiv":2659,"atural":2660,"Ġfavor":2661,"OM":2662,"Ġcases":2663,"uses":2664,"Ġsection":2665,"Ġleave":2666,"ging":2667,"oved":2668,"ĠWashington":2669,"39":2670,"ĠGl":2671,"Ġrequired":2672,"action":2673,"apan":2674,"oor":2675,"iter":2676,"ĠKing":2677,"Ġcountries":2678,"ĠGerman":2679,"lling":2680,"Ġ27":2681,"34":2682,"Ġquestions":2683,"Ġprim":2684,"Ġcell":2685,"Ġshoot":2686,"Ġanyone":2687,"ĠWest":2688,"Ġaffect":2689,"epend":2690,"Ġonline":2691,"ĠIsrael":2692,"ĠSeptember":2693,"Ġability":2694,"Ġcontent":2695,"ises":2696,"Ġreve":2697,"Ġlaun":2698,"Ġindic":2699,"Ġforce":2700,"cast":2701,"Ġsold":2702,"aving":2703,"fl":2704,"Ġsoft":2705,"Ġcompanies":2706,"ceed":2707,"Ġarticle":2708,"Ġaud":2709,"Ġrev":2710,"Ġeduc":2711,"Ġplaying":2712,"05":2713,"Ġheld":2714,"ctor":2715,"Ġreleased":2716,"Ġfederal":2717,"37":2718,"Ġadminist":2719,"Ġinterview":2720,"Ġinstall":2721,"Ġreceived":2722,"Ġsource":2723,"uk":2724,"Ph":2725,"Ġserious":2726,"Ġcreated":2727,"Ġcause":2728,"Ġimmedi":2729,"Ġdefin":2730,"uel":2731,"ĠDepartment":2732,"ctions":2733,"ĠCour":2734,"ĠNow":2735,"ze":2736,"ites":2737,"itution":2738,"Ġlate":2739,"Ġspeak":2740,"ners":2741,"Ġlegal":2742,"ari":2743,"ĠCor":2744,"Ġweeks":2745,"Ġmodel":2746,"Ġpred":2747,"Ġexact":2748,"BC":2749,"ĠBy":2750,"ING":2751,"osing":2752,"Ġtakes":2753,"Ġregard":2754,"Ġopportun":2755,"Ġprice":2756,"Ġ198":2757,"ĠApr":2758,"fully":2759,"Ġord":2760,"Ġproblems":2761,"ruction":2762,"ham":2763,"ĠCount":2764,"lege":2765,"Ġleaders":2766,"ET":2767,"lev":2768,"Ġdeep":2769,"ological":2770,"ese":2771,"haps":2772,"ĠSome":2773,"Ġpers":2774,"Ġcontract":2775,"Ġrelationship":2776,"sp":2777,"oud":2778,"Ġbase":2779,"48":2780,"mit":2781,"Ad":2782,"ancial":2783,"Ġconsum":2784,"Ġpotential":2785,"Ġlangu":2786,"rem":2787,"eth":2788,"Ġrelig":2789,"ressed":2790,"66":2791,"Ġlink":2792,"Ġlower":2793,"ayer":2794,"ĠJune":2795,"Ġfem":2796,"unt":2797,"erc":2798,"urd":2799,"Ġcontact":2800,"Ġill":2801,"Ġmother":2802,"Ġestab":2803,"htt":2804,"ĠMarch":2805,"ĠBro":2806,"ĠChina":2807,"Ġ29":2808,"Ġsqu":2809,"Ġprovided":2810,"Ġaverage":2811,"asons":2812,"Ġ2011":2813,"Ġexam":2814,"lin":2815,"55":2816,"ned":2817,"Ġperfect":2818,"Ġtou":2819,"alse":2820,"ux":2821,"Ġbuy":2822,"Ġshot":2823,"Ġcollect":2824,"Ġphot":2825,"Ġplayed":2826,"Ġsurpr":2827,"Ġofficials":2828,"Ġsimple":2829,"avy":2830,"Ġindustry":2831,"Ġhands":2832,"ground":2833,"Ġpull":2834,"Ġround":2835,"Ġuser":2836,"Ġrange":2837,"uary":2838,"Ġprivate":2839,"ops":2840,"ees":2841,"Ġways":2842,"ĠMich":2843,"Ġveh":2844,"Ġexcept":2845,"Ġterms":2846,"imum":2847,"pper":2848,"ION":2849,"ores":2850,"ĠDragon":2851,"oul":2852,"Ġden":2853,"Ġperformance":2854,"Ġbill":2855,"cil":2856,"47":2857,"Ġenvironment":2858,"Ġexc":2859,"add":2860,"Ġworth":2861,"Ġpict":2862,"Ġchance":2863,"Ġ2018":2864,"bor":2865,"Ġspeed":2866,"iction":2867,"Ġalleg":2868,"ĠJapan":2869,"atory":2870,"reet":2871,"Ġmatch":2872,"ĠII":2873,"Ġstru":2874,"order":2875,"Ġste":2876,"Ġliving":2877,"Ġstruct":2878,"ino":2879,"Ġsepar":2880,"hern":2881,"Ġresponse":2882,"Ġenjoy":2883,"Ġvia":2884,"AD":2885,"uments":2886,"acebook":2887,"Ġmember":2888,"ibr":2889,"izing":2890,"Ġtool":2891,"ĠMon":2892,"ĠWhile":2893,"hood":2894,"ĠAng":2895,"ĠDef":2896,"Ġoffer":2897,"Tr":2898,"aur":2899,"Ġturned":2900,"ĠJuly":2901,"down":2902,"anced":2903,"Ġrecently":2904,"ĠEar":2905,"Ġce":2906,"ĠStar":2907,"ĠCong":2908,"rought":2909,"Ġblood":2910,"Ġhope":2911,"Ġcomment":2912,"aint":2913,"Ġarri":2914,"iles":2915,"Ġparticip":2916,"ought":2917,"ription":2918,"08":2919,"49":2920,"Ġgave":2921,"Ġselect":2922,"Ġkilled":2923,"sych":2924,"Ġgoes":2925,"ij":2926,"Ġcoll":2927,"Ġimpact":2928,"atives":2929,"ĠSer":2930,"09":2931,"ĠAugust":2932,"Ġboy":2933,"de":2934,"ĠDes":2935,"Ġfelt":2936,"US":2937,"Ġexpected":2938,"Ġimage":2939,"ĠMark":2940,"ccording":2941,"oice":2942,"EC":2943,"ĠMag":2944,"ened":2945,"hold":2946,"ĠPost":2947,"Ġprevent":2948,"No":2949,"Ġinvolved":2950,"Ġeyes":2951,"Ġquickly":2952,"At":2953,"unk":2954,"Ġbehav":2955,"Ġur":2956,"Ġled":2957,"come":2958,"ey":2959,"Ġcandid":2960,"Ġearlier":2961,"Ġfocus":2962,"ety":2963,"Pro":2964,"ledge":2965,"ixed":2966,"illed":2967,"Ġpopular":2968,"AP":2969,"Ġsett":2970,"light":2971,"Ġvarious":2972,"inks":2973,"Ġlevels":2974,"Ġroad":2975,"ellig":2976,"ables":2977,"hel":2978,"ittee":2979,"ĠGener":2980,"ype":2981,"Ġheard":2982,"icles":2983,"Ġmis":2984,"Ġusers":2985,"ĠSan":2986,"Ġimprove":2987,"Ġfather":2988,"Ġsearch":2989,"They":2990,"vil":2991,"Ġprofess":2992,"Ġknew":2993,"Ġloss":2994,"Ġevents":2995,"65":2996,"Ġbillion":2997,"07":2998,"02":2999,"ĠNews":3000,"ĠAM":3001,"Ġcover":3002,"where":3003,"ension":3004,"Ġbott":3005,"Ġareas":3006,"ences":3007,"ope":3008,"ĠTwitter":3009,"ael":3010,"Ġgets":3011,"ĠGoogle":3012,"Ġsn":3013,"iant":3014,"Ġvote":3015,"Ġnearly":3016,"Ġincluded":3017,"Ġrecogn":3018,"zz":3019,"mm":3020,"aled":3021,"Ġhappened":3022,"04":3023,"Ġhot":3024,"Ġwhose":3025,"Ġcivil":3026,"Ġsuff":3027,"oes":3028,"itiz":3029,"ĠSyri":3030,"Ġrespond":3031,"Ġhon":3032,"Ġfeatures":3033,"Ġeconomic":3034,"ĠApril":3035,"rim":3036,"Ġtechnology":3037,"Ġoption":3038,"aging":3039,"Ġpurch":3040,"Re":3041,"Ġlat":3042,"chie":3043,"isl":3044,"Ġrecomm":3045,"uf":3046,"Ġtraining":3047,"Ġeffects":3048,"Ġfast":3049,"Ġ2010":3050,"Ġoccur":3051,"Ġwebsite":3052,"Ġemail":3053,"Ġsens":3054,"ech":3055,"Ġoil":3056,"Ġinflu":3057,"Ġcurrently":3058,"ĠSch":3059,"ĠAdd":3060,"Ġgoal":3061,"Ġscient":3062,"Ġconv":3063,"100":3064,"emy":3065,"Ġdecided":3066,"Ġtravel":3067,"Ġmention":3068,"LL":3069,"03":3070,"Ġelection":3071,"Ġphone":3072,"Ġlooks":3073,"Ġsituation":3074,"Ġcy":3075,"Ġhor":3076,"bed":3077,"ĠCourt":3078,"aily":3079,"aves":3080,"Ġquality":3081,"ĠComp":3082,"wise":3083,"Ġtable":3084,"Ġstaff":3085,"ĠWind":3086,"ett":3087,"Ġtried":3088,"idered":3089,"Ġaddition":3090,"Ġbox":3091,"Ġlack":3092,"arily":3093,"Ġwide":3094,"Ġmid":3095,"Ġboard":3096,"ysis":3097,"Ġanti":3098,"ha":3099,"Ġdig":3100,"ening":3101,"Ġdro":3102,"Con":3103,"68":3104,"Ġslow":3105,"based":3106,"sequ":3107,"Ġpath":3108,"Ex":3109,"aker":3110,"Ġworked":3111,"Ġpen":3112,"Ġengine":3113,"Ġlooked":3114,"ĠSuper":3115,"ĠServ":3116,"Ġvictim":3117,"Un":3118,"Ġproperty":3119,"Ġintrodu":3120,"Ġexecut":3121,"ĠPM":3122,"Le":3123,"Ġcolor":3124,"ĠMore":3125,"Ġ60":3126,"Ġnetwork":3127,"Ġdate":3128,"cul":3129,"idge":3130,"Ġextra":3131,"31":3132,"Ġsle":3133,"67":3134,"Ġwond":3135,"Ġreports":3136,"just":3137,"ĠAustral":3138,"Ġcapital":3139,"Ġens":3140,"Ġcommand":3141,"Ġallowed":3142,"Ġprep":3143,"Ġcapt":3144,"hib":3145,"Ġnumbers":3146,"chan":3147,"Ġfair":3148,"mp":3149,"oms":3150,"Ġreach":3151,"With":3152,"tain":3153,"Ġbroad":3154,"Ġcouple":3155,"ecause":3156,"lying":3157,"ĠFeb":3158,"Ġscreen":3159,"Ġlives":3160,"Ġprior":3161,"ĠCongress":3162,"Ar":3163,"Ġapproach":3164,"Ġemer":3165,"aries":3166,"ĠDis":3167,"serv":3168,"ĠNe":3169,"Ġbuilt":3170,"cies":3171,"Ġrepe":3172,"Ġrules":3173,"force":3174,"ĠPal":3175,"Ġfinancial":3176,"Ġconsidered":3177,"ĠChar":3178,"nces":3179,"ĠIS":3180,"Ġbrought":3181,"Ġbi":3182,"iers":3183,"ĠSim":3184,"OP":3185,"Ġproducts":3186,"Ġvisit":3187,"Ġdocument":3188,"Ġconduct":3189,"Ġcompletely":3190,"ining":3191,"ĠCalif":3192,"ibly":3193,"Ġwritten":3194,"ĠTV":3195,"ements":3196,"Ġdraw":3197,"One":3198,"Ġpublished":3199,"Ġsecret":3200,"rain":3201,"het":3202,"ĠFacebook":3203,"onday":3204,"ĠUp":3205,"Ġsexual":3206,"Ġthous":3207,"ĠPat":3208,"Ġess":3209,"Ġstandard":3210,"Ġarm":3211,"ges":3212,"ection":3213,"Ġfell":3214,"Ġforeign":3215,"ani":3216,"ĠFriday":3217,"Ġregular":3218,"inary":3219,"Ġincreased":3220,"Ġusually":3221,"Ġdemon":3222,"Ġdark":3223,"Ġadditional":3224,"rol":3225,"ĠOf":3226,"Ġproduction":3227,"!!":3228,"undred":3229,"Ġinternational":3230,"idents":3231,"ĠFree":3232,"roup":3233,"Ġrace":3234,"Ġmach":3235,"Ġhuge":3236,"All":3237,"lear":3238,"ovember":3239,"Ġtown":3240,"Ġattention":3241,"ĠOff":3242,"yond":3243,"ĠThen":3244,"field":3245,"Ġterror":3246,"raz":3247,"ĠBo":3248,"Ġmeeting":3249,"ĠPark":3250,"Ġarrest":3251,"Ġfear":3252,"Ġaw":3253,"ĠVal":3254,"oring":3255,"',":3256,"Ġextreme":3257,"arr":3258,"Ġworkers":3259,"After":3260,"Ġ31":3261,"net":3262,"ament":3263,"Ġdirectly":3264,"Ġpopulation":3265,"ube":3266,"ĠOctober":3267,"ĠIN":3268,"ĠJanuary":3269,"59":3270,"ĠDavid":3271,"Ġcross":3272,"cember":3273,"ĠFirst":3274,"Ġmessage":3275,"irit":3276,"Ġnation":3277,"Ġpoll":3278,"isions":3279,"Ġanswer":3280,"ny":3281,"isode":3282,"Ġcarry":3283,"ĠRussia":3284,"Ġhear":3285,"ength":3286,"roy":3287,"Ġnatural":3288,"inally":3289,"Ġdog":3290,"mitted":3291,"Ġtrade":3292,"Ġsubst":3293,"Ġmultiple":3294,"ĠAfric":3295,"Ġfans":3296,"Ġsort":3297,"Ġglobal":3298,"ication":3299,"ĠWed":3300,"ara":3301,"Ġachie":3302,"Ġlanguage":3303,"vey":3304,"Ġtal":3305,"Ġnecessary":3306,"Ġdetails":3307,"Ġsen":3308,"ĠSund":3309,"ĠReg":3310,"ĠRec":3311,"06":3312,"Ġsil":3313,"ressive":3314,"Ġmedical":3315,"unch":3316,"ornia":3317,"Ġund":3318,"fort":3319,"ocks":3320,"ĠMonday":3321,"uesday":3322,"craft":3323,"77":3324,"urt":3325,"Ġver":3326,"ĠHill":3327,"Ġreceive":3328,"Ġmorning":3329,"estern":3330,"Ġbank":3331,"Ġsat":3332,"irth":3333,"ĠHigh":3334,"Ġdevice":3335,"ĠTHE":3336,"ĠCenter":3337,"Ġsafe":3338,"Ġple":3339,"ĠCanada":3340,"Ġsystems":3341,"Ġassist":3342,"Ġsurv":3343,"Ġbattle":3344,"ĠSoc":3345,"vertis":3346,"She":3347,"Ġpaper":3348,"Ġgrowth":3349,"Ġcast":3350,"Sc":3351,"Ġplans":3352,"lled":3353,"Ġparts":3354,"Ġwall":3355,"Ġmovement":3356,"Ġpractice":3357,"imately":3358,"Ġdisplay":3359,"Ġsometimes":3360,"omp":3361,"ĠPaul":3362,"ĠYes":3363,"king":3364,"58":3365,"oly":3366,"Ġson":3367,"Ġavoid":3368,"okes":3369,"ĠJew":3370,"Ġtowards":3371,"asc":3372,"Ġ//":3373,"ĠKore":3374,"Ġtalking":3375,"Ġcorrect":3376,"Ġspent":3377,"icks":3378,"iable":3379,"eared":3380,"Ġterm":3381,"Ġwants":3382,"oming":3383,"Ġut":3384,"Ġdoub":3385,"Ġforces":3386,"Ġplease":3387,"69":3388,"ĠNovember":3389,"atform":3390,"ondon":3391,"Ġones":3392,"Ġimmediately":3393,"ĠRussian":3394,"ĠMet":3395,"Ġdeg":3396,"Ġparents":3397,"CH":3398,"ĠAmericans":3399,"aly":3400,"ĠMod":3401,"Ġshown":3402,"Ġconditions":3403,"Ġstuff":3404,"Ġreb":3405,"ĠYour":3406,"Ġincludes":3407,"nown":3408,"ĠSam":3409,"Ġexperien":3410,"mission":3411,"ĠEven":3412,"aught":3413,"Ġannounced":3414,"ĠRepublican":3415,"Ġdetermin":3416,"Ġdescribed":3417,"ĠCounty":3418,"()":3419,"Ġdoor":3420,"Ġchanged":3421,"Ġneigh":3422,"ĠHere":3423,"Ġclean":3424,"Ġpan":3425,"ĠDecember":3426,"ĠEuropean":3427,"iring":3428,"apter":3429,"Ġclub":3430,"ĠTuesday":3431,"Ġpaid":3432,"ĠNet":3433,"Ġattacks":3434,"Ġcharacters":3435,"Ġalone":3436,"Ġdirector":3437,"dom":3438,"Ġ35":3439,"Ġload":3440,"Ġrout":3441,"ĠCalifornia":3442,"Ġfinally":3443,"Ġrac":3444,"Ġcontr":3445,"Ġexactly":3446,"resh":3447,"pri":3448,"ĠIslam":3449,"Ġnature":3450,"Ġcareer":3451,"Ġlatest":3452,"Ġconvers":3453,"ĠSl":3454,"pose":3455,"cient":3456,"ĠInc":3457,"ivity":3458,"88":3459,"ĠAtt":3460,"ĠMor":3461,"nesday":3462,"Ġweight":3463,"ken":3464,"Ġnote":3465,"Ġteams":3466,"Ġ\\":3467,"airs":3468,"ĠGreen":3469,"Ġhundred":3470,"onent":3471,"Ġstreng":3472,"Ġconsist":3473,"icated":3474,"Ġregul":3475,"Ġlic":3476,"astic":3477,"Ġten":3478,"ursday":3479,"elligence":3480,"ously":3481,"ĠUK":3482,"BI":3483,"Ġcosts":3484,"Ġindepend":3485,"ĠAP":3486,"Ġnormal":3487,"Ġhom":3488,"Ġobvious":3489,"Ġswe":3490,"Ġstar":3491,"Ġready":3492,"acher":3493,"Ġimplement":3494,"gest":3495,"Ġsong":3496,"ĠGet":3497,"ĠLab":3498,"Ġinteresting":3499,"using":3500,"Ġgiving":3501,"ĠSunday":3502,"Ġetc":3503,"Ġmiddle":3504,"Ġremember":3505,"right":3506,"osition":3507,"utions":3508,"Ġmax":3509,"46":3510,"Ġyourself":3511,"Ġdemand":3512,"Ġtreatment":3513,"Ġdanger":3514,"ĠCons":3515,"Ġguy":3516,"ĠBritish":3517,"Ġphysical":3518,"Ġrelated":3519,"Ġremain":3520,"Ġcouldn":3521,"Ġrefer":3522,"Ġcitiz":3523,"box":3524,"ENT":3525,"board":3526,"Ġinn":3527,"IG":3528,"ero":3529,"ĠStreet":3530,"ospital":3531,"rench":3532,"chers":3533,"Ġstra":3534,"OL":3535,"ager":3536,"ĠAN":3537,"Ġeasily":3538,"IA":3539,"enge":3540,"iny":3541,"Ġclos":3542,"ocked":3543,"Ġuses":3544,"ĠCoun":3545,"Im":3546,"uild":3547,"??":3548,"more":3549,"Ġang":3550,"Ġwrite":3551,"olute":3552,"57":3553,"Ġleader":3554,"Ġreading":3555,"":3784,"Ġfigure":3785,"Ġdisapp":3786,"enty":3787,"Ġsoftware":3788,"Ġult":3789,"Ġofficers":3790,"New":3791,"Is":3792,"Ġremains":3793,"ĠIndia":3794,"Ġpsych":3795,"rief":3796,"Ġcat":3797,"esc":3798,"Ġobserv":3799,"Ġstage":3800,"ĠDark":3801,"Ġenter":3802,"change":3803,"Ġpassed":3804,"Ġdespite":3805,"ĠOut":3806,"Ġmovie":3807,"rs":3808,"Ġvoice":3809,"mine":3810,"ĠPlay":3811,"Ġtoward":3812,"ĠTer":3813,"Ġregion":3814,"Ġvalues":3815,"orters":3816,"Ġmount":3817,"Ġofficer":3818,"ĠOther":3819,"ban":3820,"Ġhous":3821,"wood":3822,"room":3823,"IV":3824,"ĠSun":3825,"see":3826,"ĠOver":3827,"rog":3828,"90":3829,"Ġlay":3830,"ĠTur":3831,"awn":3832,"Ġpressure":3833,"ĠSub":3834,"Ġbooks":3835,"edom":3836,"ĠSand":3837,"AA":3838,"ago":3839,"Ġreasons":3840,"ford":3841,"Ġactivity":3842,"UT":3843,"Now":3844,"ĠSenate":3845,"cell":3846,"night":3847,"Ġcalls":3848,"inter":3849,"Ġletter":3850,"ĠRob":3851,"ĠJe":3852,"Ġchoose":3853,"ĠLaw":3854,"Get":3855,"Be":3856,"Ġrob":3857,"Ġtypes":3858,"Ġplatform":3859,"Ġquarter":3860,"RA":3861,"ĠTime":3862,"Ġmaybe":3863,"ĠCr":3864,"95":3865,"pre":3866,"Ġmoving":3867,"Ġlif":3868,"Ġgold":3869,"Ġsom":3870,"Ġpatients":3871,"Ġtruth":3872,"ĠKe":3873,"urance":3874,"antly":3875,"mar":3876,"Ġcharge":3877,"ĠGreat":3878,"Ġcele":3879,"--------------------------------":3880,"Ġrock":3881,"roid":3882,"ancy":3883,"Ġcredit":3884,"aud":3885,"By":3886,"ĠEvery":3887,"Ġmoved":3888,"inger":3889,"ribution":3890,"Ġnames":3891,"Ġstraight":3892,"ĠHealth":3893,"ĠWell":3894,"Ġfeature":3895,"Ġrule":3896,"Ġsche":3897,"inated":3898,"ĠMichael":3899,"berg":3900,"41":3901,"iled":3902,"band":3903,"Ġclick":3904,"ĠAngel":3905,"onents":3906,"ÂŃ":3907,"ĠIraq":3908,"ĠSaturday":3909,"Ġaware":3910,"part":3911,"Ġpattern":3912,"OW":3913,"ĠLet":3914,"Ġgrad":3915,"igned":3916,"Ġassociated":3917,"Ġstyle":3918,"no":3919,"iation":3920,"aith":3921,"ilies":3922,"Ġstories":3923,"uration":3924,"Ġindividuals":3925,"Ġâ̦":3926,"miss":3927,"ĠAssoci":3928,"ishing":3929,"aby":3930,"Ġsummer":3931,"ĠBen":3932,"Ġ32":3933,"Ġarch":3934,"uty":3935,"ĠTexas":3936,"hol":3937,"Ġfully":3938,"Ġmill":3939,"Ġfollowed":3940,"ĠBill":3941,"ĠIndian":3942,"ĠSecret":3943,"ĠBel":3944,"ĠFebruary":3945,"Ġjobs":3946,"Ġseemed":3947,"ĠGovern":3948,"ipped":3949,"Ġreality":3950,"Ġlines":3951,"Ġpark":3952,"Ġmeasure":3953,"ĠOur":3954,"IM":3955,"Ġbrother":3956,"Ġgrowing":3957,"Ġban":3958,"Ġestim":3959,"Ġcry":3960,"ĠSchool":3961,"Ġmechan":3962,"ĠOF":3963,"ĠWindows":3964,"Ġrates":3965,"ĠOh":3966,"Ġpositive":3967,"Ġculture":3968,"istics":3969,"ica":3970,"Ġhar":3971,"ya":3972,"itely":3973,"ipp":3974,"Ġmap":3975,"encies":3976,"ĠWilliam":3977,"II":3978,"akers":3979,"56":3980,"ĠMart":3981,"ĠRem":3982,"Ġaltern":3983,"itude":3984,"Ġcoach":3985,"rowd":3986,"Don":3987,"Ġkids":3988,"Ġjournal":3989,"Ġcorpor":3990,"Ġfalse":3991,"Ġweb":3992,"Ġsleep":3993,"Ġcontain":3994,"Ġsto":3995,"Ġbed":3996,"iverse":3997,"ĠRich":3998,"ĠChinese":3999,"Ġpun":4000,"Ġmeant":4001,"known":4002,"Ġnotice":4003,"Ġfavorite":4004,"aven":4005,"Ġcondition":4006,"Ġpurpose":4007,"))":4008,"Ġorganization":4009,"Ġchalleng":4010,"Ġmanufact":4011,"Ġsusp":4012,"ĠAc":4013,"Ġcritic":4014,"unes":4015,"uclear":4016,"Ġmer":4017,"vention":4018,"Ġ80":4019,"Ġmist":4020,"ĠUs":4021,"ĠTor":4022,"http":4023,"olf":4024,"Ġlarger":4025,"Ġadvant":4026,"Ġresear":4027,"Ġactions":4028,"ml":4029,"Ġkept":4030,"Ġaim":4031,",'":4032,"col":4033,"Ġbenefits":4034,"ifying":4035,"Ġactual":4036,"ĠInternational":4037,"Ġvehicle":4038,"Ġchief":4039,"Ġefforts":4040,"ĠLeague":4041,"ĠMost":4042,"Ġwait":4043,"Ġadult":4044,"Ġoverall":4045,"Ġspeech":4046,"Ġhighly":4047,"Ġfemale":4048,"Ġerror":4049,"Ġeffective":4050,"54":4051,"Ġencour":4052,"well":4053,"Ġfailed":4054,"Ġconserv":4055,"Ġprograms":4056,"Ġtrou":4057,"Ġahead":4058,"500":4059,"vertisement":4060,"IP":4061,"ĠFound":4062,"pir":4063,"Ġ%":4064,"Ġcrime":4065,"ander":4066,"Ġlocation":4067,"ĠIran":4068,"Ġbehavior":4069,"azing":4070,"Ġrare":4071,"Ġemb":4072,"Ġcaused":4073,"Ġship":4074,"Ġactive":4075,"Ġcontribut":4076,"Ġgreen":4077,"Ġacqu":4078,"Ġreflect":4079,"venue":4080,"Ġfirm":4081,"Ġbirth":4082,"].":4083,"Ġclearly":4084,"Ġemot":4085,"Ġagency":4086,"riage":4087,"Ġmemory":4088,"98":4089,"SA":4090,"ĠSee":4091,"acing":4092,"CC":4093,"Ġbiggest":4094,"Ġrap":4095,"Ġbasic":4096,"Ġband":4097,"eat":4098,"Ġsuspect":4099,"ĠMac":4100,"Ġ90":4101,"mark":4102,"istan":4103,"Ġspread":4104,"ams":4105,"ki":4106,"asy":4107,"rav":4108,"ĠRober":4109,"Ġdemonstr":4110,"rated":4111,"Ġabsolute":4112,"Ġplaces":4113,"Ġimpl":4114,"ibrary":4115,"Ġcards":4116,"Ġdestroy":4117,"Ġvirt":4118,"vere":4119,"Ġappeared":4120,"yan":4121,"point":4122,"Ġbeg":4123,"Ġtemper":4124,"spe":4125,"anted":4126,"ears":4127,"ĠDirect":4128,"Ġlength":4129,"Ġblog":4130,"amb":4131,"Ġinteg":4132,"Ġresources":4133,"acc":4134,"iful":4135,"Ġspot":4136,"Ġforced":4137,"Ġthousands":4138,"ĠMinister":4139,"Ġqual":4140,"ĠFrench":4141,"atically":4142,"Ġgenerally":4143,"Ġdrink":4144,"Ġthus":4145,"IL":4146,"odes":4147,"Ġappropri":4148,"ĠRead":4149,"Ġwhom":4150,"Ġeye":4151,"Ġcollege":4152,"Ġ45":4153,"irection":4154,"Ġensure":4155,"Ġapparent":4156,"iders":4157,"Ġreligious":4158,"Ġminor":4159,"olic":4160,"Ġtro":4161,"ĠWhy":4162,"ribute":4163,"met":4164,"Ġprimary":4165,"Ġdeveloped":4166,"Ġpeace":4167,"Ġskin":4168,"ste":4169,"ava":4170,"Ġblue":4171,"Ġfamilies":4172,"Ġir":4173,"Ġapply":4174,"Ġinform":4175,"ĠSmith":4176,"CT":4177,"ii":4178,"Ġlimit":4179,"Ġresist":4180,"................":4181,"umn":4182,"Ġconflic":4183,"Ġtwe":4184,"udd":4185,"ĠTom":4186,"Ġliter":4187,"que":4188,"bon":4189,"Ġhair":4190,"Ġeventually":4191,"Ġpus":4192,"Ġhelped":4193,"Ġagg":4194,"orney":4195,"ĠApple":4196,"Ġfit":4197,"ĠSur":4198,"Ġprem":4199,"Ġsales":4200,"Ġseconds":4201,"Ġstrength":4202,"Ġfeeling":4203,"¿½":4204,"Ġtour":4205,"Ġknows":4206,"oom":4207,"Ġexerc":4208,"Ġsomew":4209,"�":4210,">>":4211,"Ġspokes":4212,"Ġideas":4213,"Ġregist":4214,"soft":4215,"ĠDel":4216,"ĠPC":4217,"Ġpropos":4218,"Ġlaunch":4219,"Ġbottom":4220,"TH":4221,"ĠPlease":4222,"vest":4223,"itz":4224,"ĠInter":4225,"Ġscript":4226,"Ġrat":4227,"arning":4228,"Ġil":4229,"ĠJer":4230,"ĠAre":4231,"Ġwhatever":4232,"oken":4233,"cience":4234,"Ġmode":4235,"Ġagree":4236,"Ġsources":4237,"Ġinitial":4238,"Ġrestrict":4239,"Ġwonder":4240,"usion":4241,"####":4242,"ĠSil":4243,"ville":4244,"Ġburn":4245,"tw":4246,"asion":4247,"Ġ£":4248,"Ġnor":4249,"uing":4250,"Ġreached":4251,"Ġsun":4252,"Ġcateg":4253,"igration":4254,"Ġcook":4255,"Ġpromot":4256,"Ġmale":4257,"Ġclimate":4258,"Ġfix":4259,"Ġalleged":4260,"UR":4261,"alled":4262,"Ġimages":4263,"Cont":4264,"ota":4265,"Ġschools":4266,"ios":4267,"Ġdrop":4268,"Ġstream":4269,"ĠMo":4270,"Ġpreviously":4271,"aling":4272,"Ġpet":4273,"Ġdouble":4274,"Ġ(@":4275,"annel":4276,"Ġdefault":4277,"ties":4278,"Ġrank":4279,"ĠDec":4280,"ĠCouncil":4281,"Ġweapon":4282,"Ġstock":4283,"Ġanaly":4284,"ĠStr":4285,"Ġpicture":4286,"ĠPolice":4287,"ference":4288,"Ġcentury":4289,"Ġcitizens":4290,"Ġonto":4291,"Ġexpand":4292,"Ġhero":4293,"ĠSol":4294,"Ġwild":4295,"Ġupdate":4296,"Ġcustomers":4297,"ront":4298,"def":4299,"Ġlik":4300,"Ġcriminal":4301,"ĠChristian":4302,"SP":4303,"76":4304,"Ġleaving":4305,"Ġotherwise":4306,"ĠDist":4307,"Ġbasis":4308,"52":4309,"53":4310,"icip":4311,"ĠBer":4312,"Ġrecommend":4313,"Ġfloor":4314,"Ġcrowd":4315,"oles":4316,"Ġ70":4317,"Ġcentral":4318,"ĠEv":4319,"Ġdream":4320,"Ġdownload":4321,"Ġconfir":4322,"ĠThom":4323,"Ġwindow":4324,"Ġhappens":4325,"Ġunit":4326,"Ġtend":4327,"Ġspl":4328,"Ġbecomes":4329,"Ġfighting":4330,"Ġpredict":4331,"ĠPress":4332,"ĠPower":4333,"Ġheavy":4334,"aked":4335,"Ġfan":4336,"orter":4337,"ategy":4338,"BA":4339,"izes":4340,"Ġspend":4341,"Here":4342,"Ġ2007":4343,"Ġadop":4344,"ĠHam":4345,"Ġfootball":4346,"ĠPort":4347,"oday":4348,"51":4349,"ampions":4350,"Ġtransfer":4351,"ht":4352,"Ġ38":4353,"term":4354,"acity":4355,"Ġbur":4356,"],":4357,"ternal":4358,"rig":4359,"but":4360,"Ġtherefore":4361,"ĠBecause":4362,"resp":4363,"rey":4364,"Ġmission":4365,"Some":4366,"Ġnoted":4367,"Ġassum":4368,"Ġdisease":4369,"Ġedit":4370,"Ġprogress":4371,"rd":4372,"ĠBrown":4373,"ocal":4374,"Ġadding":4375,"Ġraised":4376,"ĠAny":4377,"Ġtick":4378,"Ġseeing":4379,"ĠPeople":4380,"Ġagreement":4381,"Ġserver":4382,"Ġwat":4383,"Ġdebate":4384,"Ġsupposed":4385,"iling":4386,"Ġlargest":4387,"Ġsuccessful":4388,"ĠPri":4389,"ĠDemocratic":4390,"Ġjump":4391,"ĠSyria":4392,"Ġowners":4393,"Ġoffers":4394,"Ġshooting":4395,"Ġeffic":4396,"sey":4397,"Ġhaven":4398,"verse":4399,"tered":4400,"ĠLight":4401,"imal":4402,"ĠBig":4403,"Ġdefend":4404,"Ġbeat":4405,"Ġrecords":4406,"%)":4407,"Ġscen":4408,"Ġemployees":4409,"Ġdevices":4410,"hem":4411,"Ġcommer":4412,"ĠMex":4413,"Ġbenefit":4414,"ĠProf":4415,"Ġilleg":4416,"Ġsurface":4417,"ĠAlso":4418,"Ġharm":4419,"ingly":4420,"wide":4421,"ĠAlex":4422,"Ġshut":4423,"ĠCur":4424,"Ġlose":4425,"pm":4426,"Ġchallenge":4427,"semb":4428,"Ġstation":4429,"Ġintelligence":4430,"Ġaccur":4431,"ĠFlor":4432,"Ġrequires":4433,"ĠMal":4434,"bum":4435,"Ġhospital":4436,"Ġspirit":4437,"Ġoffered":4438,"Ġproduce":4439,"ĠCommun":4440,"Ġcreating":4441,"Ġcris":4442,"spect":4443,"Ġended":4444,"Ġdaily":4445,"Ġvoters":4446,"lands":4447,"ias":4448,"ih":4449,"ona":4450,"Ġsmart":4451,"ĠOffice":4452,"ĠLord":4453,"rial":4454,"ĠInternet":4455,"Ġcircum":4456,"Ġextremely":4457,"'.":4458,"Ġopinion":4459,"ĠMil":4460,"Ġgain":4461,"BS":4462,"ĠFin":4463,"yp":4464,"Ġuseful":4465,"Ġbudget":4466,"Ġcomfort":4467,"isf":4468,"Ġbackground":4469,"eline":4470,"Ġepisode":4471,"Ġenemy":4472,"Ġtrial":4473,"Ġestablish":4474,"date":4475,"ĠCap":4476,"Ġcontinues":4477,"Ġshowing":4478,"ĠUnion":4479,"with":4480,"Ġposted":4481,"ĠSystem":4482,"Ġeat":4483,"rian":4484,"Ġrise":4485,"ĠGermany":4486,"ils":4487,"Ġsigned":4488,"Ġvill":4489,"Ġgrand":4490,"mor":4491,"ĠEngland":4492,"Ġprojects":4493,"umber":4494,"Ġconference":4495,"za":4496,"Ġresponsible":4497,"ĠArab":4498,"Ġlearned":4499,"âĢĶâĢĶ":4500,"ipping":4501,"ĠGeorge":4502,"OC":4503,"Ġreturned":4504,"ĠAustralia":4505,"Ġbrief":4506,"Qu":4507,"Ġbrand":4508,"illing":4509,"abled":4510,"Ġhighest":4511,"Ġtrain":4512,"ĠCommission":4513,"while":4514,"Ġnom":4515,"ception":4516,"Ġmut":4517,"ĠBlue":4518,"Ġincident":4519,"vant":4520,"86":4521,"ĠID":4522,"Ġnuclear":4523,"74":4524,"ĠLike":4525,"ĠRE":4526,"ĠMicro":4527,"li":4528,"mail":4529,"Ġcharges":4530,"89":4531,"Ġadjust":4532,"ado":4533,"Ġearth":4534,"NA":4535,"Ġprices":4536,"PA":4537,"Ġdraft":4538,"Ġruns":4539,"Ġcandidate":4540,"enses":4541,"Ġmanagement":4542,"ĠPhil":4543,"ĠMiss":4544,"Ġteach":4545,"gram":4546,"Ġunderstanding":4547,"ait":4548,"icago":4549,"Add":4550,"ĠEp":4551,"secut":4552,"Ġseparate":4553,"Ġinstance":4554,"Ġeth":4555,"Ġunless":4556,"********":4557,"ĠFore":4558,"inate":4559,"Ġoperations":4560,"Sp":4561,"Ġfaith":4562,"gar":4563,"ĠChurch":4564,"ronic":4565,"Ġconfig":4566,"osure":4567,"Ġactivities":4568,"Ġtraditional":4569,"Ġ36":4570,"Ġdirection":4571,"Ġmachine":4572,"Ġsurround":4573,"Ġpush":4574,"unction":4575,"ĠEU":4576,"Ġeasier":4577,"Ġargument":4578,"GB":4579,"Ġmicro":4580,"Ġspending":4581,"izations":4582,"Ġtheory":4583,"adow":4584,"Ġcalling":4585,"ĠLast":4586,"Ġder":4587,"Ġinfluence":4588,"Ġcommit":4589,"Ġphoto":4590,"Ġunc":4591,"istry":4592,"gn":4593,"aste":4594,"acks":4595,"Ġdisp":4596,"ady":4597,"do":4598,"ĠGood":4599,"Ġ`":4600,"Ġwish":4601,"Ġrevealed":4602,"³³":4603,"lig":4604,"Ġenforce":4605,"ĠCommittee":4606,"Ġchem":4607,"Ġmiles":4608,"Ġinterested":4609,"Ġsolution":4610,"icy":4611,"inct":4612,"Ġ->":4613,"ĠDet":4614,"Ġremoved":4615,"Ġcompar":4616,"eah":4617,"Ġplant":4618,"ĠSince":4619,"Ġachieve":4620,"Ġadvantage":4621,"Ġslightly":4622,"bing":4623,"Ġplaced":4624,"under":4625,"2015":4626,"ĠMad":4627,"Ġtim":4628,"oses":4629,"Ġcru":4630,"ĠRock":4631,"Ġmostly":4632,"Ġnegative":4633,"Ġsetting":4634,"Ġproduced":4635,"Ġmur":4636,"Ġconnection":4637,"ĠMer":4638,"Ġdriver":4639,"Ġexecutive":4640,"Ġassault":4641,"Ġborn":4642,"ĠVer":4643,"tained":4644,"Ġstructure":4645,"Ġreduce":4646,"Ġdecades":4647,"Ġded":4648,"uke":4649,"ĠMany":4650,"idden":4651,"Ġleague":4652,"Se":4653,"Ġjoin":4654,"Ġdisco":4655,"Ġdie":4656,"cks":4657,"actions":4658,"Ġassess":4659,"agn":4660,"Ġgoals":4661,"ours":4662,"IR":4663,"Ġsenior":4664,"iller":4665,"mod":4666,"ipment":4667,"ocol":4668,"uy":4669,"ĠQue":4670,"Ġparties":4671,"irgin":4672,"Ġlearning":4673,"itable":4674,"Ġstreet":4675,"Ġcamera":4676,"App":4677,"Ġskills":4678,"bre":4679,"cious":4680,"Ġcelebr":4681,"ĠFranc":4682,"Ġexisting":4683,"Ġwilling":4684,"lor":4685,"Ġid":4686,"ĠSpace":4687,"Ġcritical":4688,"ĠLa":4689,"ortunately":4690,"Ġserve":4691,"Ġcold":4692,"Ġspecies":4693,"TS":4694,"Ġanimals":4695,"ĠBay":4696,"Ġolder":4697,"ĠUnder":4698,"estic":4699,"ĠTre":4700,"Ġteacher":4701,"Ġprefer":4702,"vis":4703,"Ġthread":4704,"ĠMatt":4705,"Ġmanager":4706,"ãĥ»":4707,"Ġprofessional":4708,"ĠVol":4709,"Ġnotes":4710,"These":4711,"ula":4712,"Ġfresh":4713,"ented":4714,"uzz":4715,"edy":4716,"clusion":4717,"ĠRel":4718,"Ġdoubt":4719,"EO":4720,"Ġopened":4721,"ĠBit":4722,"Advertisement":4723,"Ġguess":4724,"ĠUN":4725,"Ġsequ":4726,"Ġexplain":4727,"otten":4728,"Ġattract":4729,"aks":4730,"Ġstring":4731,"Ġcontext":4732,"ossible":4733,"ĠRepublicans":4734,"Ġsolid":4735,"Ġcities":4736,"Ġasking":4737,"Ġrandom":4738,"ups":4739,"uries":4740,"arant":4741,"dden":4742,"gl":4743,"ĠFlorida":4744,"Ġdepend":4745,"ĠScott":4746,"Ġ33":4747,"ĠiT":4748,"icon":4749,"Ġmentioned":4750,"Ġ2000":4751,"Ġclaimed":4752,"Ġdefinitely":4753,"ulf":4754,"Ġcore":4755,"Ġopening":4756,"ĠConst":4757,"which":4758,"ĠTra":4759,"AG":4760,"72":4761,"Ġbelieved":4762,"ada":4763,"Ġ48":4764,"ĠSecurity":4765,"yright":4766,"ĠPet":4767,"ĠLou":4768,"Ġholding":4769,"================":4770,"Ġice":4771,"Ġbrow":4772,"Ġauthorities":4773,"host":4774,"word":4775,"Ġscore":4776,"ĠDiv":4777,"Ġcells":4778,"Ġtransl":4779,"Ġneighbor":4780,"Ġremove":4781,"uct":4782,"Ġdistrict":4783,"ĠAccording":4784,"Ġworse":4785,"Ġconcerns":4786,"Ġpresidential":4787,"Ġpolicies":4788,"ĠHall":4789,"73":4790,"Ġhus":4791,"AY":4792,"Ġ2006":4793,"ĠJud":4794,"Ġindependent":4795,"ĠJustice":4796,"iliar":4797,"print":4798,"ighter":4799,"Ġprotection":4800,"zen":4801,"Ġsudden":4802,"house":4803,"ĠJes":4804,"PR":4805,"ĠInf":4806,"Ġbul":4807,"Ġ_":4808,"ĠService":4809,"ĠPR":4810,"Ġstrategy":4811,"ffect":4812,"Ġgirls":4813,"Ġmissing":4814,"oyal":4815,"ĠTeam":4816,"ulated":4817,"Ġdat":4818,"Ġpolitics":4819,"abor":4820,"According":4821,"Ġspell":4822,"Ġgraph":4823,"orthern":4824,"TC":4825,"Ab":4826,"Ġlabor":4827,"isher":4828,"Ġkick":4829,"ĠiTunes":4830,"Ġsteps":4831,"poses":4832,"Ġsmaller":4833,"En":4834,"bert":4835,"Ġroll":4836,"Ġresearchers":4837,"Ġclosed":4838,"Ġtransport":4839,"Ġlawy":4840,"________________":4841,"ĠChicago":4842,"Ġaspect":4843,"Ġnone":4844,"Ġmarriage":4845,"96":4846,"Ġelements":4847,"ĠFre":4848,"ĠSal":4849,"Ġdram":4850,"FC":4851,"top":4852,"equ":4853,"Ġhearing":4854,"Ġsupported":4855,"Ġtesting":4856,"cohol":4857,"Ġmassive":4858,"Ġstick":4859,"Ġguard":4860,"isco":4861,"phone":4862,"From":4863,"However":4864,"Ġborder":4865,"Ġcopy":4866,"ography":4867,"list":4868,"71":4869,"Ġowner":4870,"class":4871,"ruit":4872,"rate":4873,"ĠOnce":4874,"Ġdigital":4875,"Ġtask":4876,"ERS":4877,"Ġincred":4878,"tes":4879,"++":4880,"ĠFrance":4881,"Ġbreat":4882,"owl":4883,"Ġissued":4884,"ĠWestern":4885,"Ġdetect":4886,"Ġpartners":4887,"Ġshared":4888,"ĠCall":4889,"Ġcancer":4890,"ache":4891,"ribe":4892,"Ġexplained":4893,"Ġheat":4894,"{\"":4895,"Ġinvestment":4896,"ĠBook":4897,"Ġwood":4898,"Ġtools":4899,"ĠAlthough":4900,"Ġbelief":4901,"Ġcrisis":4902,"Ġge":4903,"ĠMP":4904,"Ġoperation":4905,"type":4906,"~~":4907,"ga":4908,"Ġcontains":4909,"anta":4910,"Ġexpress":4911,"ĠGroup":4912,"ĠJournal":4913,"ka":4914,"Ġamb":4915,"ĠUSA":4916,"Ġfinding":4917,"Ġfunding":4918,"how":4919,"Ġestablished":4920,"ideos":4921,"Ġdegree":4922,"Ġdangerous":4923,"anging":4924,"Ġfreedom":4925,"pport":4926,"outhern":4927,"Ġchurch":4928,"Ġcatch":4929,"ĠTwo":4930,"Ġpresence":4931,"ĠGuard":4932,"Up":4933,"Ġauthority":4934,"ĠProject":4935,"Ġbutton":4936,"Ġconsequ":4937,"Ġvalid":4938,"Ġweak":4939,"Ġstarts":4940,"Ġreference":4941,"ĠMem":4942,"\")":4943,"UN":4944,"orage":4945,"ĠOpen":4946,"Ġcollection":4947,"ym":4948,"gency":4949,"Ġbeautiful":4950,"ros":4951,"Ġtells":4952,"Ġwaiting":4953,"nel":4954,"Ġproviding":4955,"ĠDemocrats":4956,"Ġdaughter":4957,"Ġmaster":4958,"Ġpurposes":4959,"ĠJapanese":4960,"Ġequal":4961,"Ġturns":4962,"Ġdocuments":4963,"Ġwatching":4964,"Res":4965,"Ġran":4966,"2014":4967,"Ġreject":4968,"ĠKorea":4969,"Ġvictims":4970,"Level":4971,"erences":4972,"Ġwitness":4973,"Ġ34":4974,"Ġreform":4975,"coming":4976,"Ġoccup":4977,"Ġcaught":4978,"Ġtraffic":4979,"ading":4980,"Ġmodels":4981,"ario":4982,"Ġserved":4983,"Ġbatter":4984,"uate":4985,"ĠSecretary":4986,"Ġagreed":4987,"Ġtruly":4988,"ynam":4989,"ĠRet":4990,"Ġunits":4991,"ĠResearch":4992,"hand":4993,"azine":4994,"ĠMike":4995,"Ġvariety":4996,"otal":4997,"Ġamazing":4998,"Ġconfirmed":4999,"Ġentirely":5000,"Ġpurchase":5001,"Ġelement":5002,"Ġcash":5003,"Ġdetermine":5004,"De":5005,"Ġcars":5006,"ĠWall":5007,"âĸ":5008,"Ġviews":5009,"Ġdrugs":5010,"Ġdepartment":5011,"ĠStep":5012,"uit":5013,"Ġ39":5014,"asure":5015,"ĠClass":5016,"Ġcovered":5017,"ĠBank":5018,"Ġmere":5019,"uana":5020,"Ġmulti":5021,"Ġmix":5022,"Ġunlike":5023,"levision":5024,"Ġstopped":5025,"Ġsem":5026,"ĠGal":5027,"ules":5028,"Ġwel":5029,"ĠJohnson":5030,"la":5031,"Ġskill":5032,"Ġbecoming":5033,"rie":5034,"Ġappropriate":5035,"fe":5036,"ellow":5037,"ĠProt":5038,"ulate":5039,"ocation":5040,"Ġweekend":5041,"odies":5042,"Ġsites":5043,"Ġanimal":5044,"ĠTim":5045,"Ġscale":5046,"Ġcharged":5047,"Ġinstruct":5048,"illa":5049,"Ġmethods":5050,"Ġcert":5051,"Ġjudge":5052,"ĠHel":5053,"Ġdollars":5054,"Ġstanding":5055,"ĠSqu":5056,"Ġdebt":5057,"liam":5058,"Ġdriving":5059,"ĠSum":5060,"ĠEdition":5061,"Ġalbum":5062,"andon":5063,"IF":5064,"ĠUk":5065,"63":5066,"ader":5067,"Ġcommercial":5068,"esh":5069,"ĠGovernment":5070,"Ġdiscovered":5071,"Ġoutput":5072,"ĠHillary":5073,"ĠCarol":5074,"Ġ2005":5075,"Ġabuse":5076,"ancing":5077,"Ġswitch":5078,"Ġannual":5079,"Tw":5080,"Ġstated":5081,"agement":5082,"inner":5083,"Ġdemocr":5084,"Ġresidents":5085,"Ġallowing":5086,"Ġfactors":5087,"odd":5088,"Ġfuck":5089,"emies":5090,"Ġoccurred":5091,"oti":5092,"Ġnorth":5093,"ĠPublic":5094,"Ġinjury":5095,"Ġinsurance":5096,"CL":5097,"olly":5098,"ãĢ":5099,"Ġrepeated":5100,"Ġarms":5101,"anged":5102,"Ġconstruction":5103,"Ġfle":5104,"PU":5105,"icians":5106,"Ġforms":5107,"ĠMcC":5108,"antic":5109,"Ġmental":5110,"pire":5111,"Ġequipment":5112,"Ġfant":5113,"Ġdiscussion":5114,"Ġregarding":5115,"kin":5116,"arp":5117,"Ġchair":5118,"ogue":5119,"Ġproceed":5120,"ĠId":5121,"Our":5122,"Ġmurder":5123,"Man":5124,"Ġ49":5125,"asp":5126,"Ġsupply":5127,"Ġinput":5128,"Ġwealth":5129,"liament":5130,"Ġproced":5131,"orial":5132,"ĠStat":5133,"ĠNFL":5134,"hens":5135,"ĠInstitute":5136,"Ġputting":5137,"ournament":5138,"etic":5139,"Ġlocated":5140,"Ġkid":5141,"eria":5142,"run":5143,"Ġprinc":5144,"Ġ!":5145,"going":5146,"ĠBet":5147,"Ġclot":5148,"Ġtelling":5149,"Ġproposed":5150,"iot":5151,"orry":5152,"Ġfunds":5153,"gment":5154,"ĠLife":5155,"Ġbaby":5156,"ĠBack":5157,"Ġspoke":5158,"Image":5159,"Ġearn":5160,"ĠAT":5161,"gu":5162,"Ġexchange":5163,"ĠLin":5164,"oving":5165,"Ġpair":5166,"More":5167,"azon":5168,"Ġarrested":5169,"Ġkilling":5170,"can":5171,"ĠCard":5172,"yd":5173,"Ġidentified":5174,"Ġmobile":5175,"Ġthanks":5176,"onym":5177,"ĠForm":5178,"Ġhundreds":5179,"ĠChris":5180,"ĠCat":5181,"Ġtrend":5182,"hat":5183,"ĠAv":5184,"oman":5185,"Ġelectric":5186,"ĠWil":5187,"SE":5188,"Of":5189,"Ġrestaur":5190,"oted":5191,"Ġtrig":5192,"Ġnine":5193,"Ġbomb":5194,"Why":5195,"¯":5196,"Ġcoverage":5197,"Ġappeal":5198,"ĠRobert":5199,"ĠSup":5200,"Ġfinished":5201,"Ġflow":5202,"Ġdeliver":5203,"Ġcalcul":5204,"Ġphotos":5205,"Ġphil":5206,"Ġpieces":5207,"Ġappre":5208,"kes":5209,"Ġrough":5210,"Do":5211,"Ġpartner":5212,"Ġconcerned":5213,"Ġ37":5214,"ĠGen":5215,"Col":5216,"ctors":5217,"Ġ=>":5218,"state":5219,"Ġsuggested":5220,"ĠForce":5221,"CE":5222,"Ġherself":5223,"ĠPlan":5224,"works":5225,"ooth":5226,"rency":5227,"Ġcorner":5228,"Ġhusband":5229,"Ġinternet":5230,"ĠAut":5231,"ems":5232,"osen":5233,"ĠAtl":5234,"gen":5235,"Ġbalance":5236,"62":5237,"Ġsounds":5238,"text":5239,"Ġarr":5240,"oves":5241,"Ġmillions":5242,"Ġradio":5243,"Ġsatisf":5244,"ĠDam":5245,"Mr":5246,"Go":5247,"Spe":5248,"Ġcombat":5249,"rant":5250,"ĠGree":5251,"Ġfuel":5252,"Ġdistance":5253,"Ġtests":5254,"Ġdecre":5255,"ĠEr":5256,"Ġmanaged":5257,"DS":5258,"Ġtit":5259,"Ġmeasures":5260,"ĠLiber":5261,"Ġattend":5262,"ashed":5263,"ĠJose":5264,"ĠNight":5265,"dit":5266,"ĠNov":5267,"ĠEnd":5268,"outs":5269,"Ġgeneration":5270,"Ġadvoc":5271,"yth":5272,"Ġconversation":5273,"ĠSky":5274,"active":5275,"cel":5276,"rier":5277,"ĠFrank":5278,"Ġgender":5279,"Ġconcent":5280,"Ġcarried":5281,"anda":5282,"ĠVirgin":5283,"Ġarrived":5284,"icide":5285,"aded":5286,"Ġfailure":5287,"Ġminimum":5288,"lets":5289,"Ġworst":5290,"Ġkeeping":5291,"Ġintended":5292,"Ġillegal":5293,"Ġsubsc":5294,"Ġdetermined":5295,"Ġtrip":5296,"Yes":5297,"Ġraise":5298,"Ġ~":5299,"Ġfeels":5300,"Ġpackage":5301,"ĠJo":5302,"hi":5303,"2016":5304,"real":5305,"Ġfra":5306,"Ġsymb":5307,"Me":5308,"ucky":5309,"pret":5310,"ĠKh":5311,"ĠEdit":5312,"ĠWeb":5313,"emic":5314,"ĠColor":5315,"Ġjustice":5316,"Int":5317,"Ġfarm":5318,"cknow":5319,"\">":5320,"eless":5321,"Ġreduced":5322,"Ġ500":5323,"xx":5324,"ĠRad":5325,"ĠWood":5326,"Ġclin":5327,"Ġhyp":5328,"iler":5329,"ura":5330,"kins":5331,"85":5332,"61":5333,"ĠTheir":5334,"ĠMary":5335,"Ġsan":5336,"Ġnovel":5337,"ĠWho":5338,"Ġcapacity":5339,"Ġimpossible":5340,"Ġplays":5341,"Ġminister":5342,"ijuana":5343,"icate":5344,"ĠSet":5345,"Ġfram":5346,"Ġing":5347,"Ġcommunities":5348,"ĠFBI":5349,"ita":5350,"Ġbon":5351,"Ġstrateg":5352,"Ġinterests":5353,"lock":5354,"gers":5355,"mas":5356,"ĠAND":5357,"Ġconflict":5358,"Ġrequirements":5359,"Ġsac":5360,"Ġoperating":5361,"ini":5362,"related":5363,"Ġcommitted":5364,"Ġrelatively":5365,"Ġsouth":5366,"¯¯":5367,"Ġafford":5368,"Ġidentity":5369,"Ġdecisions":5370,"Ġaccused":5371,"place":5372,"Ġvictory":5373,"och":5374,"iat":5375,"Name":5376,"Com":5377,"tion":5378,"eds":5379,"Ġseek":5380,"Ġtight":5381,"ĠImages":5382,"Ġiniti":5383,"Ġhumans":5384,"Ġfamiliar":5385,"Ġaudience":5386,"Ġinternal":5387,"venture":5388,"Ġsides":5389,"ĠTO":5390,"Ġdim":5391,"Ġconclud":5392,"Ġappoint":5393,"Ġenforcement":5394,"ĠJim":5395,"ĠAssociation":5396,"Ġcircumst":5397,"ĠCanadian":5398,"Ġjoined":5399,"Ġdifferences":5400,"ĠLos":5401,"Ġprotest":5402,"Ġtwice":5403,"win":5404,"Ġglass":5405,"arsh":5406,"ĠArmy":5407,"Ġexpression":5408,"Ġdecide":5409,"Ġplanning":5410,"ania":5411,"Ġhandle":5412,"ĠMicrosoft":5413,"ĠNor":5414,"Ġmaximum":5415,"ĠRev":5416,"Ġsea":5417,"Ġeval":5418,"Ġhelps":5419,"ref":5420,"Ġbound":5421,"Ġmouth":5422,"Ġstandards":5423,"Ġclim":5424,"ĠCamp":5425,"ĠFox":5426,"cles":5427,"Ġarmy":5428,"ĠTechn":5429,"acking":5430,"xy":5431,"SS":5432,"Ġ42":5433,"Ġbug":5434,"ĠUkrain":5435,"ĠMax":5436,"ĠJones":5437,"ĠShow":5438,"lo":5439,"Ġplanet":5440,"Ġ75":5441,"Ġwinning":5442,"Ġfaster":5443,"Ġspect":5444,"Ġbroken":5445,"TR":5446,"Ġdefined":5447,"Ġhealthy":5448,"Ġcompetition":5449,"https":5450,"ĠIsland":5451,"ĠFe":5452,"Ġannounce":5453,"ĠCup":5454,"ĠInstead":5455,"Ġclient":5456,"Ġpossibly":5457,"section":5458,"ocket":5459,"look":5460,"Ġfinish":5461,"Ġcrew":5462,"Ġreserv":5463,"Ġeditor":5464,"Ġhate":5465,"Ġsale":5466,"Ġcontrovers":5467,"Ġpages":5468,"wing":5469,"Ġnumer":5470,"Ġopposition":5471,"Ġ2004":5472,"Ġrefuge":5473,"Ġflight":5474,"Ġapart":5475,"ĠLat":5476,"Americ":5477,"ĠAfrica":5478,"Ġapplications":5479,"ĠPalest":5480,"ĠBur":5481,"Ġgar":5482,"ĠSocial":5483,"Ġupgr":5484,"Ġshape":5485,"Ġspeaking":5486,"ansion":5487,"ao":5488,"ĠSn":5489,"Ġworry":5490,"ĠBritain":5491,"Please":5492,"roud":5493,"Ġhun":5494,"Ġintroduced":5495,"Ġdiet":5496,"Ind":5497,"ĠSecond":5498,"Ġfunctions":5499,"uts":5500,"ĠEach":5501,"ĠJeff":5502,"Ġstress":5503,"Ġaccounts":5504,"Ġguarant":5505,"ĠAnn":5506,"edia":5507,"Ġhonest":5508,"Ġtree":5509,"ĠAfrican":5510,"ĠBush":5511,"},":5512,"Ġsch":5513,"ĠOnly":5514,"Ġfif":5515,"igan":5516,"Ġexercise":5517,"ĠExp":5518,"Ġscientists":5519,"Ġlegislation":5520,"ĠWork":5521,"ĠSpr":5522,"ÃĤ":5523,"ĠHuman":5524,"Ġè":5525,"Ġsurvey":5526,"Ġrich":5527,"rip":5528,"Ġmaintain":5529,"Ġflo":5530,"Ġleadership":5531,"stream":5532,"ĠIslamic":5533,"Ġ01":5534,"ĠCollege":5535,"Ġmagic":5536,"ĠPrime":5537,"Ġfigures":5538,"2017":5539,"inder":5540,"xual":5541,"ĠDead":5542,"Ġabsolutely":5543,"Ġfourth":5544,"Ġpresented":5545,"respond":5546,"rible":5547,"Ġalcohol":5548,"ato":5549,"ĠDE":5550,"porary":5551,"Ġgrab":5552,"Ġvari":5553,"Ġquant":5554,"ĠPhoto":5555,"Ġplus":5556,"rick":5557,"arks":5558,"Ġalternative":5559,"Ġpil":5560,"Ġapprox":5561,"that":5562,"Ġobjects":5563,"ĠRo":5564,"ĠAndroid":5565,"Ġsignificantly":5566,"ĠRoad":5567,"kay":5568,"Read":5569,"avor":5570,"Ġacknow":5571,"ĠHD":5572,"ĠSing":5573,"Or":5574,"ĠMont":5575,"Ġuns":5576,"prof":5577,"Ġnegoti":5578,"ĠArch":5579,"iki":5580,"Ġtelevision":5581,"ĠJewish":5582,"Ġcommittee":5583,"Ġmotor":5584,"Ġappearance":5585,"Ġsitting":5586,"Ġstrike":5587,"ĠDown":5588,"comp":5589,"ĠHist":5590,"Ġfold":5591,"acement":5592,"ĠLouis":5593,"Ġbelong":5594,"ĠâĢ¢":5595,"Ġmort":5596,"Ġprepared":5597,"Ġ64":5598,"ĠMaster":5599,"Ġindeed":5600,"ĠDen":5601,"Ġrent":5602,"TA":5603,"ourney":5604,"arc":5605,"Su":5606,"97":5607,"Ġadvice":5608,"Ġchanging":5609,"Ġlisted":5610,"Ġlaunched":5611,"isation":5612,"ĠPeter":5613,"ishes":5614,"Ġlived":5615,"ĠMel":5616,"ĠSupreme":5617,"ĠFederal":5618,"Ġ);":5619,"ructure":5620,"Ġsets":5621,"Ġphilos":5622,"uous":5623,"ĠÂł":5624,"Ġapplied":5625,"ĠNOT":5626,"Ġhousing":5627,"ĠMount":5628,"Ġodd":5629,"Ġsust":5630,"DA":5631,"fficient":5632,"Ġ?":5633,"olved":5634,"Ġpowers":5635,"Ġthr":5636,"Ġremaining":5637,"ĠWater":5638,"LC":5639,"Ġcauses":5640,"ãģ®":5641,"Ġmanner":5642,"ads":5643,"Ġsuggests":5644,"Ġends":5645,"standing":5646,"fig":5647,"ĠDun":5648,"idth":5649,"Ġgay":5650,"Ġtermin":5651,"ĠAngeles":5652,"MS":5653,"Ġscientific":5654,"Ġcoal":5655,"apers":5656,"bar":5657,"ĠThomas":5658,"Ġsym":5659,"ĠRun":5660,"this":5661,"PC":5662,"igrants":5663,"Ġminute":5664,"ĠDistrict":5665,"cellent":5666,"Ġleaves":5667,"Ġcompleted":5668,"amin":5669,"Ġfocused":5670,"Ġmonitor":5671,"Ġvehicles":5672,"MA":5673,"ĠMass":5674,"ĠGrand":5675,"Ġaffected":5676,"itutional":5677,"Ġconstruct":5678,"Ġfollows":5679,"Ġton":5680,"reens":5681,"Ġhomes":5682,"ĠExt":5683,"ĠLevel":5684,"rast":5685,"ĠIr":5686,"Ġelim":5687,"Ġlargely":5688,"ĠJoe":5689,"Ġvotes":5690,"alls":5691,"Ġbusinesses":5692,"ĠFoundation":5693,"ĠCentral":5694,"Ġyards":5695,"Ġmaterials":5696,"ulner":5697,"Ġguide":5698,"Ġcloser":5699,"ums":5700,"Ġsports":5701,"eder":5702,"Just":5703,"Ġtaxes":5704,"84":5705,"ĠOld":5706,"Ġdecade":5707,"ola":5708,"Ġvir":5709,"Ġdropped":5710,"Ġdelay":5711,"itect":5712,"Ġsecure":5713,"stein":5714,"level":5715,"Ġtreated":5716,"Ġfiled":5717,"aine":5718,"Ġvan":5719,"Ġmir":5720,"Ġcolumn":5721,"icted":5722,"eper":5723,"Ġrot":5724,"Ġconsult":5725,"Ġentry":5726,"Ġmarijuana":5727,"ĠDou":5728,"Ġapparently":5729,"oking":5730,"clusive":5731,"Ġincreases":5732,"ano":5733,"Ġspecifically":5734,"Ġtele":5735,"ensions":5736,"Ġreligion":5737,"abilities":5738,"Ġframe":5739,"ĠNote":5740,"ĠLee":5741,"Ġhelping":5742,"Ġedge":5743,"oston":5744,"Ġorganizations":5745,"Ãĥ":5746,"ĠBoth":5747,"hips":5748,"Ġbigger":5749,"Ġboost":5750,"ĠStand":5751,"Ġrow":5752,"uls":5753,"abase":5754,"Ġrid":5755,"Let":5756,"aren":5757,"rave":5758,"Ġstret":5759,"PD":5760,"Ġvision":5761,"Ġwearing":5762,"Ġappreci":5763,"Ġaward":5764,"ĠUse":5765,"Ġfactor":5766,"war":5767,"ulations":5768,")(":5769,"Ġgod":5770,"Ġterrit":5771,"Ġparam":5772,"asts":5773,"87":5774,"Ġenemies":5775,"ĠGames":5776,"FF":5777,"Ġaccident":5778,"Well":5779,"ĠMartin":5780,"TER":5781,"Ġath":5782,"ĠHell":5783,"Ġforg":5784,"Ġveter":5785,"ĠMedic":5786,"free":5787,"Ġstars":5788,"Ġexpensive":5789,"Ġacad":5790,"rawn":5791,"ĠWhe":5792,"Ġlock":5793,"Ġformat":5794,"Ġsoldiers":5795,"sm":5796,"Ġagent":5797,"Ġresponsibility":5798,"ora":5799,"ĠScience":5800,"Ġrapid":5801,"Ġtough":5802,"ĠJesus":5803,"Ġbelieves":5804,"ML":5805,"Ġwear":5806,"lete":5807,"ÃĥÃĤ":5808,"ĠDri":5809,"Ġcommission":5810,"ĠBob":5811,"Oh":5812,"aped":5813,"Ġwarm":5814,"ÃĥÃĤÃĥÃĤ":5815,"Ġ2003":5816,"ortion":5817,"Ġhasn":5818,"uster":5819,"Ġunivers":5820,"ĠIll":5821,"Ġking":5822,"ologies":5823,"94":5824,"ĠTem":5825,"ĠMos":5826,"Ġpatient":5827,"ĠMexico":5828,"cean":5829,"ĠDeath":5830,"ĠSanders":5831,"you":5832,"ĠCast":5833,"ĠCompany":5834,"pty":5835,"Ġhappening":5836,"FP":5837,"ĠBattle":5838,"Ġbought":5839,"Am":5840,"Mod":5841,"Us":5842,"uters":5843,"ĠCre":5844,"ĠThose":5845,"Ġ44":5846,"iser":5847,"Ġsoul":5848,"ĠTop":5849,"ĠHarry":5850,"ĠAw":5851,"Ġseat":5852,"ffee":5853,"Ġrevolution":5854,"Ġ(\"":5855,"ĠDuring":5856,"ette":5857,"Ġring":5858,"Ġoffensive":5859,"Ġreturns":5860,"Ġvideos":5861,"Ġdiscl":5862,"Ġfamous":5863,"enced":5864,"ĠSign":5865,"ĠRiver":5866,"Ġ300":5867,"PM":5868,"ĠBus":5869,"ĠCH":5870,"Ġcandidates":5871,"arden":5872,"Ġpercentage":5873,"Ġvisual":5874,"Ġthank":5875,"Ġtrouble":5876,"nergy":5877,"Ġ2001":5878,"Ġprove":5879,"ashion":5880,"Ġenh":5881,"ĠLong":5882,"UM":5883,"Ġconnected":5884,"Ġpossibility":5885,"Over":5886,"Ġexpert":5887,"Ġlibrary":5888,"arts":5889,"ĠDirector":5890,"Ġfellow":5891,"92":5892,"irty":5893,"Ġdry":5894,"Ġsigns":5895,"ĠLove":5896,"Ġquiet":5897,"foot":5898,"Ġpure":5899,"ĠHun":5900,"Ġfilled":5901,"phas":5902,"ĠElect":5903,"endment":5904,"ĠExpl":5905,"Ġunable":5906,"ns":5907,"mo":5908,"Ġvast":5909,"obe":5910,"Ġidentify":5911,"apping":5912,"ĠCarolina":5913,"gress":5914,"Ġprote":5915,"Ġfish":5916,"Ġcircumstances":5917,"razy":5918,"ĠPhot":5919,"Ġbodies":5920,"ĠMur":5921,"Ġdeveloping":5922,"ĠAR":5923,"Ġexperienced":5924,"Ġsubstant":5925,"ĠBoard":5926,"esome":5927,"Ġdomestic":5928,"Ġcombined":5929,"ĠPut":5930,"Ġchemical":5931,"ĠChild":5932,"Ġpool":5933,"ĠCy":5934,"Ġegg":5935,"cons":5936,"sters":5937,"Ġhurt":5938,"Ġmarkets":5939,"Ġconservative":5940,"Ġsupporters":5941,"Ġagencies":5942,"idel":5943,"Ob":5944,"urb":5945,"Ġ43":5946,"ĠDefense":5947,"ye":5948,"ĠAp":5949,"dule":5950,"Ġtemperature":5951,"Ġconducted":5952,"ĠChief":5953,"Ġpulled":5954,"Ġfol":5955,"Last":5956,"onto":5957,"osis":5958,"VER":5959,"Des":5960,"ĠPan":5961,"First":5962,"Ġadvance":5963,"Ġlicense":5964,"rors":5965,"ĠJon":5966,"Ġimagine":5967,"Ġhell":5968,"Ġfixed":5969,"Ġincor":5970,"osite":5971,"ĠLog":5972,"icken":5973,"]:":5974,"Ġsurprise":5975,"hab":5976,"Ġcraft":5977,"olt":5978,"ĠJul":5979,"Ġdial":5980,"Ġrelevant":5981,"Ġentered":5982,"Ġleads":5983,"ĠAD":5984,"ĠClean":5985,"Ġpictures":5986,"essor":5987,"Ġalt":5988,"Ġpaying":5989,"Per":5990,"ĠMarket":5991,"Ġupdates":5992,"amily":5993,"ĠType":5994,"ĠHome":5995,"Ġ55":5996,"sembly":5997,"rome":5998,"83":5999,"Ġgreatest":6000,"Ġheight":6001,"Ġheav":6002,"aints":6003,"Ġlisten":6004,"aser":6005,"ĠSH":6006,"Ġcapable":6007,"acle":6008,"Ġperspect":6009,"inating":6010,"Ġoffering":6011,"rypt":6012,"ĠDevelop":6013,"abin":6014,"rc":6015,"Ġbright":6016,"alty":6017,"arrow":6018,"Ġsuppl":6019,"inding":6020,"acked":6021,"gypt":6022,"ĠAnother":6023,"pg":6024,"ĠVirginia":6025,"ĠLu":6026,"Ġplanned":6027,"Ġpit":6028,"Ġsweet":6029,"Type":6030,"ĠDi":6031,"Ġtypically":6032,"ĠFrancisco":6033,"Ġprospect":6034,"ĠDan":6035,"Ġteen":6036,"rees":6037,"Ġsched":6038,"Ġhol":6039,"Ġscr":6040,"Ġlots":6041,"life":6042,"Ġnewsp":6043,"Ġforget":6044,"ĠNone":6045,"ĠMiddle":6046,"ĠRyan":6047,"edd":6048,"Ġsevere":6049,"Ġsuit":6050,"ller":6051,"93":6052,"Ġcorrespond":6053,"Ġexplos":6054,"uations":6055,"Ġflag":6056,"game":6057,"rid":6058,"Ġprin":6059,"ĠData":6060,"Ġdeploy":6061,"ĠEnter":6062,"suit":6063,"ghan":6064,"ĠMen":6065,"Ġthoughts":6066,"Ġmatters":6067,"Ġadapt":6068,"ĠAri":6069,"Ġfill":6070,"Ġforth":6071,"Ġsam":6072,"Ġ41":6073,"Ġpayment":6074,"ĠHor":6075,"Ġspring":6076,"duc":6077,"Ġlosing":6078,"Ġbringing":6079,"FO":6080,"ala":6081,"Ġdistribution":6082,"hered":6083,"bour":6084,"ĠIsraeli":6085,"oma":6086,"Ġcombination":6087,"Ġplenty":6088,"VE":6089,"Can":6090,"ĠHaw":6091,"Ġperman":6092,"ĠSpecial":6093,"Ġtow":6094,"Ġseeking":6095,"Ġexamples":6096,"Ġclasses":6097,"cr":6098,"Ġbeer":6099,"Ġmoves":6100,"ĠIP":6101,"ĠKn":6102,"Ġpanel":6103,"Even":6104,"Ġproperly":6105,"Ġris":6106,"Ġplug":6107,"Ġestimated":6108,"Every":6109,"Ġdefensive":6110,"agraph":6111,"Ġpregn":6112,"Ġinstit":6113,"ĠVict":6114,"Ġvolume":6115,"Ġpositions":6116,"Ġlinks":6117,"ĠProgram":6118,"ĠWeek":6119,"agues":6120,"Ġtransform":6121,"ker":6122,"ĠCEO":6123,"Ġcas":6124,"Ġopponent":6125,"Ġtweet":6126,"ĠCode":6127,"Ġshop":6128,"Ġfly":6129,"Ġtalks":6130,"Ġbag":6131,"Phone":6132,"Ġaid":6133,"Ġplants":6134,"Ġ65":6135,"Ġattorney":6136,"arters":6137,"quest":6138,"ĠMagic":6139,"Ġbegins":6140,"Ġmyster":6141,"Ġenvironmental":6142,"Ġstorage":6143,"NN":6144,"Ġmarg":6145,"Ġske":6146,"Ġmetal":6147,"elly":6148,"Ġordered":6149,"Ġremained":6150,"Ġloved":6151,"Ġprompt":6152,"Ġupdated":6153,"Ġexperts":6154,"Ġwalking":6155,"Ġancient":6156,"Ġperformed":6157,"ATE":6158,"Ġneither":6159,"iency":6160,"Ġmanufacture":6161,"ĠPak":6162,"Ġselected":6163,"Ġmine":6164,"Ġultimately":6165,"Ġexplan":6166,"Ġlabel":6167,"ĠServices":6168,"ributed":6169,"Trump":6170,"Ġsyn":6171,"ĠUlt":6172,"SC":6173,"Ġmeat":6174,"Ġgiant":6175,"ĠWars":6176,"ĠON":6177,"Ġadm":6178,"Ġinterpret":6179,"Ġevening":6180,"Ġevil":6181,"ĠBoston":6182,"ĠWild":6183,"ĠÃ":6184,"ĠBitcoin":6185,"ĠAmazon":6186,"Dr":6187,"ĠInformation":6188,"Ġobviously":6189,"Ġadvanced":6190,"Photo":6191,"olar":6192,"Ġweather":6193,"Ġsymbol":6194,"Ġsole":6195,"Ġpotentially":6196,"oster":6197,"Ġoriginally":6198,"mun":6199,"300":6200,"aze":6201,"essions":6202,"Ġdeck":6203,"Ġstood":6204,"Ġyouth":6205,"ĠBern":6206,"Rep":6207,"ĠTest":6208,"Ġbasically":6209,"otic":6210,"Ġinvolve":6211,"olit":6212,"lyn":6213,"See":6214,"Ġaircraft":6215,"Ġconfirm":6216,"EW":6217,"Ġmessages":6218,"ĠRichard":6219,"Ġkit":6220,"Ġprohib":6221,"Ġvulner":6222,"isters":6223,"Ġexistence":6224,"Ġturning":6225,"ĠSP":6226,"Ġdesire":6227,"Ġflat":6228,"Ġment":6229,"season":6230,"anges":6231,"Ġneighborhood":6232,"ĠLake":6233,"ATION":6234,"Ġpointed":6235,"bur":6236,"Ġinnov":6237,"ucks":6238,"UL":6239,"Ġprofessor":6240,"Ġexpressed":6241,"AB":6242,"icious":6243,"Ġ2002":6244,"ĠDev":6245,"Ġsession":6246,"Ġbare":6247,"sen":6248,"Ġdiss":6249,"ĠCath":6250,"ĠPass":6251,"ĠPoint":6252,"Ġdoctor":6253,"orrow":6254,"ailed":6255,"ĠRub":6256,"ĠDC":6257,"ĠCharl":6258,"person":6259,"Ġwriter":6260,"ighters":6261,"ureau":6262,"Ġoblig":6263,"Ġrecorded":6264,"Ġbroke":6265,"Ġorders":6266,"ilty":6267,"Ġmotion":6268,"inity":6269,"law":6270,"adium":6271,"Ġimmigration":6272,"Ġcontrast":6273,"Ġbatt":6274,"Ġexcellent":6275,"Ġtechnical":6276,"ami":6277,"Ġtun":6278,"Ġcloud":6279,"ĠYear":6280,"geon":6281,"Ġcreation":6282,"Ġstrange":6283,"Ġauth":6284,"Ġfort":6285,"born":6286,"Ġextent":6287,"ĠToday":6288,"ĠClub":6289,"Ġrain":6290,"Ġsample":6291,"Ġaccepted":6292,"Ġtact":6293,"Ġfired":6294,"ĠSon":6295,"Ġstands":6296,"Ġboot":6297,"Ġ47":6298,"Ġstatements":6299,"Ġversions":6300,"Ġselling":6301,"ounded":6302,"Ġ1990":6303,"Ġweren":6304,"ĠWatch":6305,"Ġexperiment":6306,"Post":6307,"Ġretail":6308,"uled":6309,"Inst":6310,"unte":6311,"ãĥ¼":6312,"Ġdepart":6313,"Ġbond":6314,"ivery":6315,"ompl":6316,"Ġreaction":6317,"ĠSyrian":6318,"ĠPac":6319,"apped":6320,"aniel":6321,"DP":6322,"Ġresolution":6323,"Ġreact":6324,"Ġapproved":6325,"onom":6326,"mond":6327,"ĠOffic":6328,"---":6329,"Ġreplace":6330,"Ġtack":6331,"Ġsport":6332,"Ġchain":6333,"Ġemergency":6334,"rad":6335,"ĠPalestin":6336,"Ġ46":6337,"Ġautomatically":6338,"Ġroute":6339,"Ġpal":6340,"Ġbanks":6341,"ĠParis":6342,"ĠMedia":6343,"road":6344,"icing":6345,"ixt":6346,"isted":6347,"Ġgrew":6348,"Ġcoord":6349,"ĠWhere":6350,"omin":6351,"Ġsubs":6352,"��":6353,"Ġ±":6354,"Ġcorporate":6355,"Ġselection":6356,"noon":6357,"ĠReport":6358,"cs":6359,"cluding":6360,"orders":6361,"anche":6362,"ĠIts":6363,"Ġslowly":6364,"ĠEgypt":6365,"ĠAcc":6366,"Ġcolle":6367,"iques":6368,"EX":6369,"Ġattempts":6370,"url":6371,"ĠCross":6372,"Ġfindings":6373,"ĠSC":6374,"ĠOR":6375,"Ġindex":6376,"ensity":6377,"ĠWay":6378,"ĠLand":6379,"Ġshock":6380,"dis":6381,"Ġdynam":6382,"Ġcart":6383,"mosp":6384,"Since":6385,"iest":6386,"ĠBoy":6387,"Ġstorm":6388,"ĠContin":6389,"2013":6390,"hew":6391,"ilit":6392,"Ġessential":6393,"iquid":6394,"Other":6395,"ivered":6396,"Ġreasonable":6397,"Act":6398,"Ġsubsequ":6399,"ĠPack":6400,"ĠFort":6401,"Ġconsidering":6402,"Ġuniversity":6403,"log":6404,"Ġmarried":6405,"Ġillust":6406,"ĠTrue":6407,"£ı":6408,"Ġnumerous":6409,"rastructure":6410,"Ġseriously":6411,"Ġreferred":6412,"ua":6413,"Ġconsistent":6414,"onna":6415,"ĠReal":6416,"ruption":6417,"ciples":6418,"Ġfacts":6419,"91":6420,"otes":6421,"erg":6422,"Then":6423,"Ġaccompl":6424,"Note":6425,"Ġrevenue":6426,"Ġpassing":6427,"Ġmal":6428,"een":6429,"ĠYet":6430,"Ġgather":6431,"terday":6432,"ework":6433,"ĠAuthor":6434,"Pe":6435,"Ġoptim":6436,"Ġrub":6437,"Ġè£ı":6438,"Ġunknown":6439,"stone":6440,"Ġunion":6441,"olve":6442,"Ġopportunities":6443,"Ġbrowser":6444,"ĠWal":6445,"ĠCost":6446,"Ġreporting":6447,"sts":6448,"pet":6449,"Ġsand":6450,"Ġsuddenly":6451,"Ġsurprising":6452,"ĠVR":6453,"Ġsomewhat":6454,"ĠBas":6455,"ulture":6456,"izz":6457,"ĠCD":6458,"Ġchallenges":6459,"Ġsettings":6460,"Ġexperiences":6461,"ĠFull":6462,"Ġcann":6463,"Ġreceiving":6464,"EST":6465,"Ġjoint":6466,"Ġcultural":6467,"Ġast":6468,"82":6469,"astern":6470,"ceived":6471,"ĠCru":6472,"Ġbull":6473,"pired":6474,"amm":6475,"Ġfacing":6476,"power":6477,"Ġboss":6478,"ĠHol":6479,"Ġinstr":6480,"Ġincreasingly":6481,"Ġshift":6482,"Ġstreets":6483,"ĠWilliams":6484,"abb":6485,"Ġlie":6486,"Ġlaugh":6487,"ĠCa":6488,"PL":6489,"Ġadults":6490,"Ġcustomer":6491,"Ġobtained":6492,"Ġsupporting":6493,"html":6494,"fire":6495,"Ġdetailed":6496,"Ġpicked":6497,"ĠRight":6498,"lder":6499,"EE":6500,"stood":6501,"ĠKim":6502,"Ġwire":6503,"Ġsight":6504,"Ġdevelopers":6505,"Ġpersons":6506,"Ġsad":6507,"Ġcup":6508,"Ġwarning":6509,"Ġboys":6510,"long":6511,"Ġbird":6512,"fo":6513,"Ġwal":6514,"Ġobserved":6515,"Ġzone":6516,"iveness":6517,"Ġchannel":6518,"cript":6519,"Ġrefused":6520,"ĠAgain":6521,"Ġsuc":6522,"Ġspokesman":6523,"ĠRef":6524,"rite":6525,"ouston":6526,"ãĥ³":6527,"ĠSher":6528,"Ġacts":6529,"ĠName":6530,"Ġstruggle":6531,"arry":6532,"ometimes":6533,"Ġdiscrim":6534,"HT":6535,"Ġcategory":6536,"Ġrealize":6537,"Ġemployee":6538,"ĠAfghan":6539,"enger":6540,"Ġguns":6541,"ĠSteve":6542,"ĠMot":6543,"ĠOl":6544,"oked":6545,"Ġthick":6546,"Ġfairly":6547,"illy":6548,"Ġsurve":6549,"ĠMat":6550,"weight":6551,"âĶ":6552,"Ġtroops":6553,"Ġagents":6554,"Ġbattery":6555,"Ġmotiv":6556,"á":6557,"Sec":6558,"den":6559,"overy":6560,"LS":6561,"Ġflu":6562,"Ġconfident":6563,"ĠOper":6564,"Ġempty":6565,"Ġphen":6566,"Ġsector":6567,"Ġexcited":6568,"Ġremote":6569,"aph":6570,"oen":6571,"Ġdestroyed":6572,"Ġmoral":6573,"ĠHP":6574,"ĠRon":6575,"Ġdress":6576,"ĠBat":6577,"Ġlit":6578,"ĠMS":6579,"Ġaf":6580,"HL":6581,"rum":6582,"isms":6583,"Ġshouldn":6584,"Ġsympt":6585,"ĠToronto":6586,"hetic":6587,"Ġcarbon":6588,"Ġinstalled":6589,"Ġviolent":6590,"Ġsolar":6591,"ja":6592,"Ġpractices":6593,"Ġride":6594,"ĠPenn":6595,"Ġimproved":6596,"Ġaudio":6597,"Ġbehavi":6598,"ĠPS":6599,"Ġeating":6600,"Data":6601,"ĠReview":6602,"pass":6603,"claim":6604,"uated":6605,"angers":6606,"chen":6607,"Ġproperties":6608,"Ġanywhere":6609,"Another":6610,"Ġblow":6611,"ĠJackson":6612,"Ġproud":6613,"Ġplane":6614,"lines":6615,"Ġsquare":6616,"Ġproof":6617,"ansas":6618,"Ġtalked":6619,"makers":6620,"Ġsister":6621,"Ġholds":6622,"Ġresident":6623,"Ġ==":6624,"Ġresistance":6625,"Ġsplit":6626,"Ġprosecut":6627,"Ġconfidence":6628,"resents":6629,"Ġcuts":6630,"Ġexception":6631,"Ġzero":6632,"Getty":6633,"Ġcopyright":6634,"Ġtotally":6635,"ormal":6636,"ifications":6637,"ĠAustralian":6638,"Ġsick":6639,"Ġ150":6640,"Ġhousehold":6641,"Ġfees":6642,"Ġdrivers":6643,"ogen":6644,"ĠNY":6645,"Ġnecessarily":6646,"Ġregulations":6647,"earing":6648,"sl":6649,"Ġperspective":6650,"care":6651,"icial":6652,"His":6653,"Ġescape":6654,"Ġsurprised":6655,"ĠVan":6656,"urrent":6657,"Ġvac":6658,"81":6659,"ĠThus":6660,"Ġemphas":6661,"ĠChampions":6662,"ĠIce":6663,"Ġnarr":6664,"Ġheads":6665,"Ġcausing":6666,"bel":6667,"fortunately":6668,"ĠMa":6669,"Ġtargets":6670,"cipl":6671,"Ġafternoon":6672,"Ġadds":6673,"ĠMaybe":6674,"ĠFour":6675,"essed":6676,"plete":6677,"Ġusual":6678,"cho":6679,"ingu":6680,"Ġwithd":6681,"ĠEnergy":6682,"ĠEconom":6683,"OO":6684,"Ġarticles":6685,"Ġinjured":6686,"Ġmanage":6687,"Ġexplains":6688,"Ġdiagn":6689,"Rec":6690,"atures":6691,"Ġlinked":6692,"Ġdiscussed":6693,"Ġexplo":6694,"Ġoccasion":6695,"athan":6696,"Ġopposite":6697,"Ġfaces":6698,"Ġdenied":6699,"ĠKnight":6700,"Ġnut":6701,"Ġapproximately":6702,"Ġdisappoint":6703,"onymous":6704,"ĠBest":6705,"ĠLo":6706,"ĠHy":6707,"ĠAff":6708,"Ġvoting":6709,"anwhile":6710,"ĠIII":6711,"Ġinstitutions":6712,"agram":6713,"ĠDaily":6714,"Ġdrag":6715,"Ġnearby":6716,"Ġguilty":6717,"Ġconver":6718,"Pre":6719,"ship":6720,"Ġreward":6721,"Ġphilosoph":6722,"ĠSS":6723,"ugh":6724,"Ġapps":6725,"friend":6726,"Ġupper":6727,"Ġadvert":6728,"Ġsnow":6729,"Ġfrust":6730,"Ġourselves":6731,"Fr":6732,"ĠDie":6733,"ampion":6734,"Ġdismiss":6735,"Ġcere":6736,"Ġsignal":6737,"from":6738,"Ġ).":6739,"Ġ52":6740,"Ġcrimes":6741,"itors":6742,"estival":6743,"useum":6744,"Ġcouncil":6745,"ĠSaud":6746,"May":6747,"ĠGun":6748,"ician":6749,"ether":6750,"Ġsufficient":6751,"ĠHen":6752,"sole":6753,"Ġhistorical":6754,"ĠFar":6755,"ĠTurn":6756,"Ġpin":6757,"Ġsucceed":6758,"mat":6759,"lymp":6760,"Ġtradition":6761,"ĠOk":6762,"Ġcro":6763,"Ġdescription":6764,"alle":6765,"Ġsky":6766,"Te":6767,"Ġwidely":6768,"Ġwave":6769,"Ġdefinition":6770,"ĠJews":6771,"Ġcycle":6772,"Ġrefere":6773,"Ġbrings":6774,"usal":6775,"Ġalive":6776,"Ġfrequently":6777,"Ġintention":6778,"ĠControl":6779,"lv":6780,"ystem":6781,"Ġprivacy":6782,"gent":6783,"rence":6784,"ĠQuest":6785,"ĠChristmas":6786,"Ġrail":6787,"Ġcooper":6788,"Ġtested":6789,"ĠCapt":6790,"asks":6791,"Ġcomfortable":6792,"Ġdelivered":6793,"scape":6794,"Ġdepth":6795,"ĠGOP":6796,"Ġwrites":6797,"Ġassets":6798,"Ġsav":6799,"iments":6800,"Ġtransition":6801,"Ġartist":6802,"ĠLook":6803,"Ġlob":6804,"Ġcomponents":6805,"arity":6806,"Ġwalked":6807,"Ġroot":6808,"Ġparticipants":6809,"Ġnoticed":6810,"Ġresc":6811,"Ġnav":6812,"ĠAdminist":6813,"da":6814,"utral":6815,"plate":6816,"Ġimportance":6817,"Ġassert":6818,"iously":6819,"cription":6820,"Ġinjuries":6821,"ĠCheck":6822,"Ġregistered":6823,"Ġintent":6824,"Ġmissed":6825,"ographic":6826,"Ġsentence":6827,"ounter":6828,"Ġassistance":6829,"evin":6830,"Ġdatabase":6831,"Ġbuildings":6832,"Ġclassic":6833,"Ġthinks":6834,"ĠOhio":6835,"Pr":6836,"ugg":6837,"Ġfee":6838,"pan":6839,"Ġeffectively":6840,"Ġfacility":6841,"Ġbear":6842,"Ġchapter":6843,"Ġdogs":6844,"ĠColumb":6845,"Ġlatter":6846,"itial":6847,"Ġadmitted":6848,"TV":6849,"ĠGeorg":6850,"Ġposts":6851,"\\\\":6852,"Ġlawyer":6853,"Ġequival":6854,"Ġmand":6855,"Ġcontrolled":6856,"ĠWalk":6857,"ĠAndrew":6858,"Ġmenu":6859,"amental":6860,"Ġprotected":6861,"va":6862,"Ġadministr":6863,"oral":6864,"Ġrein":6865,"ĠSar":6866,"Ġamounts":6867,"Ġnative":6868,"ĠMoon":6869,"Ġrepresents":6870,"Ġabandon":6871,"Ġcarrying":6872,"Ġtank":6873,"mary":6874,"Ġdeclared":6875,"Tube":6876,"Ġhat":6877,"Ġpunish":6878,"ellect":6879,"mes":6880,"Ġuniverse":6881,"ĠRod":6882,"phy":6883,"Ġinfrastructure":6884,"Ġ51":6885,"Ġopposed":6886,"ownt":6887,"ca":6888,"ĠMake":6889,"Ġhardware":6890,"Ġcoffee":6891,"Rel":6892,"bal":6893,"world":6894,"ĠSaf":6895,"ĠSea":6896,"inals":6897,"Ġowned":6898,"Ġhall":6899,"ersion":6900,"Ġdescribe":6901,"ĠPot":6902,"Ġportion":6903,"Ġatmosp":6904,"Ġgovernments":6905,"Ġdepending":6906,"Ġoffense":6907,"Ġtrick":6908,"awa":6909,"ĠLine":6910,"ĠVis":6911,"ĠHard":6912,"ĠOrig":6913,"ĠClick":6914,"Ġdesk":6915,"ĠValley":6916,"ĠSov":6917,"Ġmovies":6918,"Ġremark":6919,"Ġmail":6920,"Ġconscious":6921,"Ġruling":6922,"ĠRights":6923,"Ġmedic":6924,"hent":6925,"ĠWomen":6926,"><":6927,"Ġreplaced":6928,"ĠPrem":6929,"ĠThanks":6930,"Ġrenew":6931,"ĠBall":6932,"iform":6933,"Ġshots":6934,"Comm":6935,"Ġarmed":6936,"Ġconstant":6937,"Ġtaste":6938,"Ġrealized":6939,"Ġbuff":6940,"Ġmo":6941,"Ġefficient":6942,"Most":6943,"oration":6944,"ifies":6945,"Ġcommunication":6946,"Ġflood":6947,"Ġconsequences":6948,"Ġanyway":6949,"igg":6950,"ĠGM":6951,"ĠThank":6952,"Ġiron":6953,"Ġevolution":6954,"ĠCop":6955,"twitter":6956,"Ġ95":6957,"Ġrelationships":6958,"adel":6959,"ĠYoung":6960,"Ġproposal":6961,"ayers":6962,"uilding":6963,"ĠHot":6964,"ORE":6965,"cos":6966,"Ġcollabor":6967,"PG":6968,"axy":6969,"Ġknowing":6970,"Ġsupports":6971,"owed":6972,"Ġcontrols":6973,"Ġmerely":6974,"umer":6975,"Ġathlet":6976,"Ġfashion":6977,"path":6978,"Ġgift":6979,"Ġera":6980,"AND":6981,"Ġkinds":6982,"ĠKorean":6983,"Ġlegit":6984,"ulous":6985,"Ġessentially":6986,"Ġtherap":6987,"nic":6988,"Ġsuffered":6989,"Ġhur":6990,"Ġpromise":6991,"Ġexcess":6992,"Ġoverw":6993,"Ġprime":6994,"ĠHouston":6995,"erry":6996,"ĠMs":6997,"RS":6998,"2012":6999,"Ġstores":7000,"ĠOlymp":7001,"Ġjourney":7002,"Although":7003,"Sub":7004,"ĠEduc":7005,"ĠChapter":7006,"Ġrequests":7007,"Ġconsumers":7008,"Ġtiny":7009,"Ġisol":7010,"ĠFair":7011,"ba":7012,"ĠYOU":7013,"Ġcrash":7014,"celer":7015,"Ġemotional":7016,"Ġgoods":7017,"Ġelected":7018,"Ġmoder":7019,"ĠLinux":7020,"Ġblocks":7021,"Ġisland":7022,"ĠSociety":7023,"Ġelections":7024,"Ġbroadcast":7025,"Ġcheap":7026,"Ġnations":7027,"Ġseasons":7028,"400":7029,"Ġwaste":7030,"ĠSat":7031,"Ġfields":7032,"employ":7033,"Ġprofile":7034,"Ġauthors":7035,"ALL":7036,"ĠGra":7037,"west":7038,"ĠTy":7039,"Ġdeaths":7040,"Ġvacc":7041,"Ġformed":7042,"Ġdu":7043,"Ġongoing":7044,"ĠMuslims":7045,"elf":7046,"igure":7047,"Ġassume":7048,"ĠUkraine":7049,"water":7050,"Ġcoast":7051,"Ġvoted":7052,"gor":7053,"ĠAS":7054,"ĠMichigan":7055,"aza":7056,"ĠArm":7057,"iro":7058,"Ġflex":7059,"asters":7060,"''":7061,"Ġwelcome":7062,"arl":7063,"Ġlocations":7064,"igation":7065,"ĠFil":7066,"Ġbuying":7067,"Ġarchitect":7068,"Ġharder":7069,"ĠCub":7070,"Ġinterface":7071,"Ġrestaurant":7072,"Ġdiscover":7073,"Ġexceed":7074,"Ġfavour":7075,"gery":7076,"Ġduty":7077,"Ġpitch":7078,"ador":7079,"ĠMach":7080,"boy":7081,"Ġresponded":7082,"Ġextended":7083,"hers":7084,"Many":7085,"raid":7086,"ifer":7087,"ĠIns":7088,"Ser":7089,"Ġmedium":7090,"she":7091,"ĠSports":7092,"Ġmagazine":7093,"utation":7094,"Ġlimits":7095,"ĠGall":7096,"Ġexternal":7097,"razil":7098,"Ġyounger":7099,"tle":7100,"Ġremind":7101,"ĠCON":7102,"Ġimmediate":7103,"Ġhidden":7104,"Ġvolunte":7105,"Ġsimpl":7106,"odcast":7107,"Ġphase":7108,"dr":7109,"Ġplot":7110,"Ġexposure":7111,"RI":7112,"ograp":7113,"vin":7114,"anish":7115,"ĠAcad":7116,"ĠEngine":7117,"Ġexpansion":7118,"ĠPay":7119,"Your":7120,"Ġpushed":7121,"ĠEll":7122,"ĠHead":7123,"Ġmarketing":7124,"ĠAC":7125,"ket":7126,"Ġhits":7127,"Ġgro":7128,"ĠAge":7129,"ĠScot":7130,"][":7131,"Ġstim":7132,"ĠiPhone":7133,"ĪĴ":7134,"Ġnarrow":7135,"ĠGetty":7136,"ĠTurkey":7137,"Ġperfectly":7138,"Ġenable":7139,"utch":7140,"Ġprecise":7141,"Ġregime":7142,"Ġshif":7143,"Ġcompens":7144,"gun":7145,"div":7146,"Ġchosen":7147,"ĠKen":7148,"Any":7149,"Ġtrees":7150,"Ġrecommended":7151,"ĠRen":7152,"uable":7153,"ĠHT":7154,"Follow":7155,"EG":7156,"ĠHand":7157,"ĠKenn":7158,"Ġarguments":7159,"Ġexists":7160,"Ġbike":7161,"ĠConserv":7162,"Ġbreaking":7163,"ĠGar":7164,"Ġcrazy":7165,"Ġvirtual":7166,"aylor":7167,"ixel":7168,"Ġ1980":7169,"Ġpermission":7170,"ĠSeries":7171,"Ġconsumer":7172,"Ġclosely":7173,"called":7174,"Ġ54":7175,"Ġhopes":7176,"Ġarray":7177,"ĠWin":7178,"ĠLabour":7179,"Ġspons":7180,"ĠIre":7181,"Ġpow":7182,"Ġreaders":7183,"Ġemployment":7184,"Ġcreature":7185,"Ġresulting":7186,"Ġaccurate":7187,"Ġmoments":7188,"Ġargued":7189,"Ġped":7190,"During":7191,"Ġ53":7192,"ĠTal":7193,"Ġsought":7194,"Ġsuffering":7195,"Ġicon":7196,"lee":7197,"Ġ($":7198,"alian":7199,"°":7200,"Ġpra":7201,"Ġbonus":7202,"(\"":7203,"ko":7204,"Ġacting":7205,"DE":7206,"fall":7207,"Ġcomparison":7208,"Ġsmooth":7209,"ĠNAS":7210,"upp":7211,"ĠJoseph":7212,"eping":7213,"ĠTake":7214,"ĠMid":7215,"Ġsending":7216,"fast":7217,"ĠFall":7218,"Ġdealing":7219,"user":7220,"ĠOrgan":7221,"Co":7222,"Ġattached":7223,"Ġsees":7224,"%.":7225,"Ġtypical":7226,"ART":7227,"Ġfinds":7228,"ĠAsia":7229,"umin":7230,"ĠCore":7231,"ĠEnt":7232,"inent":7233,"uce":7234,"ĠBlood":7235,"ĠNever":7236,"Ġemails":7237,"Ġhighlight":7238,"Ġconfront":7239,"atus":7240,"uted":7241,"Ġunus":7242,"Ġtopic":7243,"ĠAdam":7244,"Ġble":7245,"ati":7246,"Ġunderstood":7247,"Set":7248,"struct":7249,"TP":7250,"Ġmob":7251,"aa":7252,"ĠStart":7253,"pected":7254,"sell":7255,"Ġdedicated":7256,"ĠCA":7257,"uan":7258,"Ġsongs":7259,"escription":7260,"Ġtech":7261,"Ġrape":7262,"Ġaside":7263,"Ġgrant":7264,"Ġ56":7265,"sub":7266,"Ġargue":7267,"Ġcontaining":7268,"Ġschedule":7269,"Ġliberal":7270,"Ġpublicly":7271,"Ġheavily":7272,"ĠUt":7273,"iner":7274,"ĠSection":7275,"ĠCare":7276,"weet":7277,"ls":7278,"Dis":7279,"âĶĢ":7280,"ĠFollow":7281,"Back":7282,"ĠIT":7283,"Ġbes":7284,"ji":7285,"ĠHit":7286,"ested":7287,"Ġeverybody":7288,"ĠSwed":7289,"Ġfemin":7290,"Ġfacilities":7291,"Ġconven":7292,"Comp":7293,"ĠOS":7294,"core":7295,"Ġanx":7296,"Ġdivision":7297,"ĠCam":7298,"ĠStan":7299,"mates":7300,"Ġexplore":7301,"plom":7302,"Ġshares":7303,"pload":7304,"anes":7305,"Ġideal":7306,"eters":7307,"ĠBase":7308,"Ġplastic":7309,"Ġdistinct":7310,"ĠNetwork":7311,"ĠSeattle":7312,"Ġtrading":7313,"ensus":7314,"intend":7315,"Ġexhib":7316,"Ġinitially":7317,"ĠFood":7318,"Ġthousand":7319,"ĠBusiness":7320,"acter":7321,"Ġparagraph":7322,"Ġroughly":7323,"Ġwww":7324,"Ġcreative":7325,"ĠConf":7326,"Ġconsumption":7327,"Ġfilms":7328,"agan":7329,"Ġobtain":7330,"Ġtall":7331,"Ġtor":7332,"Ġacknowled":7333,"Ġgrown":7334,"alo":7335,"KE":7336,"Ġ400":7337,"enders":7338,"taining":7339,"UG":7340,"Ġsuicide":7341,"Ġwatched":7342,"ĠList":7343,"ali":7344,"rehens":7345,"Ġsurrounding":7346,"Ġpip":7347,"Ġflying":7348,"ĠJava":7349,"ordan":7350,"Ġserving":7351,"inations":7352,"post":7353,"Ġsho":7354,"Av":7355,"Ġjail":7356,"zy":7357,"Ġ1999":7358,"Ġ>":9609,"orous":9610,"Ġfirms":9611,"screen":9612,"una":9613,"Ġembarrass":9614,"ulse":9615,"Ġletting":9616,"Ġthrew":9617,"iley":9618,"Ġchannels":9619,"lan":9620,"ĠVegas":9621,"Ġsear":9622,"Ġfantastic":9623,"arre":9624,"uzzle":9625,"ĠDer":9626,"Those":9627,"Ġswing":9628,"Ġsheet":9629,"index":9630,"cover":9631,"ogan":9632,"Ġvariables":9633,"ĠTech":9634,"Ġspoken":9635,"achel":9636,"ĠDa":9637,"ĠMountain":9638,"Ġloaded":9639,"Ġfootage":9640,"version":9641,"Ġunl":9642,"ĠPhoenix":9643,"Ġthrowing":9644,"Ġfiring":9645,"Ġtracking":9646,"Ġwidth":9647,"Ġstruggling":9648,"rooms":9649,"otion":9650,"Ġmonthly":9651,"ĠServer":9652,"Ġeggs":9653,"open":9654,"MC":9655,"Ġ1993":9656,"Ġhired":9657,"Ġstayed":9658,"ĠAllen":9659,"Ġstro":9660,"Ġ98":9661,"step":9662,"ĠTurkish":9663,"Ġfabric":9664,"isting":9665,"ĠDom":9666,"Ġdates":9667,"Ġpron":9668,"Ġbasketball":9669,"Ġlucky":9670,"ĠArabia":9671,"Ġassumed":9672,"esty":9673,"Ġaffairs":9674,"Ġglad":9675,"ĠIndeed":9676,"ĠFA":9677,"ĠWord":9678,"Ġjoining":9679,"ifice":9680,"pread":9681,"irts":9682,"ĠSelect":9683,"Ġpopulations":9684,"aware":9685,"Ġnose":9686,"Ġcomplaints":9687,"start":9688,"Ġscoring":9689,"Thanks":9690,"Ġmining":9691,"Ġvisitors":9692,"SH":9693,"Ġdamaged":9694,"Ġcharacteristics":9695,"ĠPent":9696,"DC":9697,"Ġ83":9698,"ĠSix":9699,"rates":9700,"Ġflags":9701,"ĠBrew":9702,"dog":9703,"Mark":9704,"////":9705,"Ġexecution":9706,"Ġjoke":9707,"phones":9708,"Ġtestimony":9709,"Ġobst":9710,"QL":9711,"ĠCut":9712,"Ġstudied":9713,"ĠNintendo":9714,"icket":9715,"ĠNBC":9716,"Ġlad":9717,"ĠBra":9718,"ĠMoh":9719,"Ġkernel":9720,"Ġoverwhelming":9721,"Ġaged":9722,"Ġapplicable":9723,"ĠCond":9724,"Ġroads":9725,"ĠBlock":9726,"made":9727,"odge":9728,"Ġcommands":9729,"Ġoffices":9730,"veland":9731,"Ġtut":9732,"Ġreceiver":9733,"ĠFro":9734,"Ġshopping":9735,"ĠiP":9736,"ĠStre":9737,"ĠABC":9738,"Ġentertainment":9739,"ĠBow":9740,"orted":9741,"Mc":9742,"Ġreads":9743,"grad":9744,"ĠCollect":9745,"ĠâĪĴ":9746,"ĠCapital":9747,"ederation":9748,"Ġemployer":9749,"Ġinvolvement":9750,"Ġanxiety":9751,"alia":9752,"Ġroof":9753,"ĠAmong":9754,"ĠDemocrat":9755,"Ġstats":9756,"ĠVill":9757,"Ġconstitutional":9758,"Ġreferring":9759,"itty":9760,"Ġtackle":9761,"outube":9762,"Ġbacked":9763,"ĠHong":9764,"ĠBroad":9765,"Ġele":9766,"ĠOtt":9767,"Ġ1992":9768,"hour":9769,"achusetts":9770,"Cal":9771,"Ġdefeated":9772,"Ġ81":9773,"esp":9774,"Ġseemingly":9775,"was":9776,"ĠJenn":9777,"ĠKurd":9778,"Ġgene":9779,"Ġdiscount":9780,"Ret":9781,"ECT":9782,"();":9783,"Ġclubs":9784,"Ġsid":9785,"ĠMarsh":9786,"Check":9787,"Ġpp":9788,"ĠEag":9789,"idespread":9790,"Ġbeings":9791,"FT":9792,"Ġintroduction":9793,"ĠChange":9794,"ARD":9795,"Ġ110":9796,"adows":9797,"ierce":9798,"Ġmeal":9799,"author":9800,"ĠBang":9801,"lahoma":9802,"Ġranks":9803,"2011":9804,"????":9805,"max":9806,"Ġcollapse":9807,"Ġopens":9808,"Ġecho":9809,"Ġsoph":9810,"Ġracist":9811,"Ġenormous":9812,"Ġwaves":9813,"Ġtap":9814,"Ġcomprehensive":9815,".--":9816,"ĠRoy":9817,"Ġfarmers":9818,"Related":9819,"aired":9820,"rones":9821,"ĠCrim":9822,"Ġproportion":9823,"Ġdesigns":9824,"Ġnegotiations":9825,"Ġvirtually":9826,"ĠBatman":9827,"Ġwarn":9828,"Ġlegitimate":9829,"mate":9830,"Ġconvention":9831,",,":9832,"netic":9833,"ĠSD":9834,"Ġconsistently":9835,"Ġcompensation":9836,"Ġpunishment":9837,"Ġye":9838,"Ġtie":9839,"ĠBureau":9840,"irlf":9841,"ĠBu":9842,"ĠAren":9843,"ĠPhilipp":9844,"Ġknife":9845,"Ġmemories":9846,"ĠRoss":9847,"Ġangle":9848,"Ġ86":9849,"ĠThunder":9850,"Ġrend":9851,"ĠTour":9852,"Ġcounts":9853,"sung":9854,"ĠImp":9855,"Ġeducational":9856,"Ġaccessible":9857,"COM":9858,"Ġdrew":9859,"yer":9860,"Gl":9861,"amine":9862,"ORT":9863,"OB":9864,"IB":9865,"master":9866,"Ġtrials":9867,"ogy":9868,"har":9869,"ĠTrust":9870,"Ġpreferred":9871,"irlfriend":9872,"ĠNev":9873,"Ġbin":9874,"Ġcow":9875,"Page":9876,"Ġsignature":9877,"ĠBL":9878,"700":9879,"Ġretired":9880,"Ġbytes":9881,"Ġneighb":9882,"ĠLegend":9883,"Ġdevast":9884,"Ġsuspected":9885,"isons":9886,"ĠPokémon":9887,"scale":9888,"Ġcapabilities":9889,"Ġrevel":9890,"Ġcheese":9891,"dy":9892,"igrant":9893,"Ġfailing":9894,"bits":9895,"ĠHeroes":9896,"ĠGhost":9897,"ĠScient":9898,"Ġappointed":9899,"uri":9900,"Ġinstitution":9901,"Ġexpanded":9902,"greg":9903,"Ġmonitoring":9904,"Ġpodcast":9905,"Ġcoalition":9906,"Ġ96":9907,"Jo":9908,"Ġstolen":9909,"ĠSab":9910,"Ġstops":9911,"Ġholiday":9912,"Ġintr":9913,"Car":9914,"Black":9915,"ĠLGBT":9916,"Ġwarming":9917,"ĠAnderson":9918,"Ġ89":9919,"Ġproducer":9920,"Med":9921,"Ġaccuracy":9922,"ĠMarvel":9923,"izabeth":9924,"ĠPatrick":9925,"mony":9926,"Ġmini":9927,"acles":9928,"Ġovert":9929,"they":9930,"Ġmembership":9931,"ĠVen":9932,"Ġexch":9933,"Ġremoval":9934,"ĠDave":9935,"TY":9936,"mad":9937,"ĠFind":9938,"Ġadequ":9939,"Ġec":9940,"Ġteeth":9941,"Ġemotion":9942,"Ġperm":9943,"Ġsolely":9944,"db":9945,"Ġextraord":9946,"IGHT":9947,"cal":9948,"Ġguidelines":9949,"Ġdying":9950,"Ġsuspended":9951,"ĠPremier":9952,"ĠAnthony":9953,"elve":9954,"Ġdad":9955,"ĠEth":9956,"ĠFootball":9957,"Ġabandoned":9958,"Ġ<<":9959,"Ġmarch":9960,"Ġhorror":9961,"â̦\"":9962,"Ġchildhood":9963,"Ġcampaigns":9964,"Ġlunch":9965,"ĠAlbert":9966,"block":9967,"âĸĪâĸĪ":9968,"ounding":9969,"Ġbone":9970,"organ":9971,"aders":9972,"ĠFlash":9973,"ĠDrive":9974,"Ġtonight":9975,"Ġwars":9976,"ĠFL":9977,"Ġformation":9978,"const":9979,"News":9980,"Ġcompe":9981,"orious":9982,"ĠStaff":9983,"Ġdiscussions":9984,"ĠProtection":9985,"ĠJam":9986,"Ġcriteria":9987,"Ġinstallation":9988,"Ġaccomplish":9989,"izza":9990,"Ġpublisher":9991,"Ġrescue":9992,"ĠTry":9993,"ULL":9994,"ĠSom":9995,"ĠHop":9996,"oret":9997,"ths":9998,"ordon":9999,"Ġpocket":10000,"ĠInv":10001,"Download":10002,"ĠCrime":10003,"Ġbene":10004,"ĠGuide":10005,"ĠAssembly":10006,"Ġparameters":10007,"IE":10008,"ĠAlexander":10009,"Ġconcert":10010,"ĠSche":10011,"Ġshoes":10012,"Ġvisiting":10013,"Ġrecall":10014,"Ġbub":10015,"Ġrural":10016,"Ġconcrete":10017,"ĠRos":10018,"Next":10019,"Russ":10020,"Ġloans":10021,"ĠShield":10022,"Ġtrem":10023,"hemat":10024,"kg":10025,"ĠHarris":10026,"isition":10027,"ĠMove":10028,"ĠFC":10029,"Ġfate":10030,"ĠCho":10031,"Ġtired":10032,"Ġprincipal":10033,"hist":10034,"iences":10035,"athy":10036,"Ġsevent":10037,"Ġmood":10038,"Ġstrategic":10039,"Ġdiseases":10040,"Ġforum":10041,"Ġtempor":10042,"Ġheadquarters":10043,"Par":10044,"ige":10045,"flix":10046,"Ġguitar":10047,"Ġ94":10048,"Only":10049,"Ġreleases":10050,"roph":10051,"================================":10052,"Ġ600":10053,"ĠContinue":10054,"igate":10055,"ĠCrit":10056,"system":10057,"Ġdisabled":10058,"Ġunexpected":10059,"ithub":10060,"Ġunclear":10061,"ĠEst":10062,"Ġcontrad":10063,"Ġstrategies":10064,"ventures":10065,"Ġpassage":10066,"AME":10067,"Ġimproving":10068,"Ġreveals":10069,"Ġdecrease":10070,"ova":10071,"Ġannoy":10072,"ĠShort":10073,"ĠLibrary":10074,"Ġcyber":10075,"nell":10076,"ĠHur":10077,"ĠCB":10078,"Ġphotograp":10079,"UI":10080,"Ġsed":10081,"Ge":10082,"Ġ87":10083,"Ġdiverse":10084,"Ġencouraged":10085,"Ġconspiracy":10086,"Ġbirds":10087,"Ġoperator":10088,"Ġhandful":10089,"Ġclassified":10090,"?)":10091,"Ġdramatic":10092,"Ġinvestigators":10093,"ito":10094,"Ġwidespread":10095,"ĠRoom":10096,"----------------------------------------------------------------":10097,"Ġcollective":10098,"Ġjournalist":10099,"String":10100,"Ġtemperatures":10101,"ila":10102,"Ġguid":10103,"Ġinspect":10104,"Ġmissile":10105,"ĠMayor":10106,"Ġmanual":10107,"Ġsimultane":10108,"Ġratings":10109,"Ġsuck":10110,"Ġ97":10111,"Ġuniversal":10112,"Ġpharm":10113,"Ġdisrupt":10114,"iano":10115,"AV":10116,"Ġft":10117,"Ġstatist":10118,"olds":10119,"ĠWalker":10120,"php":10121,"Ġundert":10122,"ĠLas":10123,"ishop":10124,"ntil":10125,"reshold":10126,"ĠWhether":10127,"Ms":10128,"Ġdeny":10129,"ĠCloud":10130,"Ġprovider":10131,"Ġsurviv":10132,"ĠUpdate":10133,"has":10134,"Ġmistakes":10135,"charge":10136,"pled":10137,"rity":10138,"Ġnode":10139,"ĠMassachusetts":10140,"ools":10141,"lication":10142,"Ġfails":10143,"emale":10144,"ori":10145,"backs":10146,"Ġshirt":10147,"Ġ''":10148,"ĠNAT":10149,"Ġwaters":10150,"elson":10151,"Ġease":10152,"Ġscar":10153,"Ġcontents":10154,"mind":10155,"Ġcontribution":10156,"Ġshr":10157,"Ġhanded":10158,"Ġstability":10159,"Ġtrave":10160,"Em":10161,"Ġmirror":10162,"123":10163,"Ġweigh":10164,"Ġfiction":10165,"ouver":10166,"istant":10167,"rition":10168,"ĠFed":10169,"Ġphysically":10170,"Ġstake":10171,"ĠArticle":10172,"ĠArc":10173,"ĠLewis":10174,"ĠMind":10175,"Ġdemonstrate":10176,"Ġprofits":10177,"vision":10178,"omic":10179,"olid":10180,"Ġbattles":10181,"Ġdrives":10182,"Ġeastern":10183,"ĠSony":10184,"!!!":10185,"aration":10186,"vard":10187,"ĠGL":10188,"portation":10189,"Ġ92":10190,"Ġlawmakers":10191,"Ġprotecting":10192,"ĠEPA":10193,"Ġyeah":10194,"Ġshame":10195,"olph":10196,"even":10197,"xit":10198,"Ġattach":10199,"Ġrepresenting":10200,"Ġobs":10201,"ĠUtah":10202,"iffs":10203,"ĠFreedom":10204,"ó":10205,"AK":10206,"Ġincidents":10207,"itage":10208,"Ġviewers":10209,"cd":10210,"Ġmouse":10211,"Ġclar":10212,"Ġaccordance":10213,"Ġbot":10214,"cor":10215,"ĠSummer":10216,"held":10217,"Ġinnocent":10218,"Ġinitiative":10219,"ols":10220,"________________________________":10221,"Ġspots":10222,"pace":10223,"Ġconventional":10224,"Ġcorporations":10225,"Ġblocked":10226,"HD":10227,"attered":10228,"Ġrefers":10229,"Ġbuck":10230,"ĠDigital":10231,"120":10232,"Ġtopics":10233,"TF":10234,"Äģ":10235,"brid":10236,"reement":10237,"Ġunderlying":10238,"ĠMember":10239,"Ġinvestigating":10240,"Ġpregnancy":10241,"Ġtouchdown":10242,"ĠBand":10243,"ĠCaller":10244,"Ġinstances":10245,"PP":10246,"wa":10247,"Good":10248,"Ġ1991":10249,"ĠCold":10250,"Ġfears":10251,"Ġremarks":10252,"ĨĴ":10253,"atal":10254,"Ġmit":10255,"Ġexperiments":10256,"ipt":10257,"Color":10258,"indu":10259,"Update":10260,"Ġ93":10261,"Ag":10262,"Ġå":10263,"ancouver":10264,"Both":10265,"Ġjudges":10266,"Object":10267,"Ġstere":10268,"umbn":10269,"Ġparticipation":10270,"ĠStars":10271,"ĠJere":10272,"Ġweekly":10273,"ĠBan":10274,"Ġconversations":10275,"ĠPitt":10276,"uz":10277,"ĠIndiana":10278,"ĠKick":10279,"Ġinfection":10280,"Ġheroes":10281,"Ġsettled":10282,"Ġstrip":10283,"Ġhal":10284,"Ġdump":10285,"ĠSci":10286,"Ġles":10287,"Ġreferences":10288,"ĠURL":10289,"ĠBridge":10290,"Ġwanting":10291,"Force":10292,"Ġexclus":10293,"Meanwhile":10294,"mn":10295,"Ġgentle":10296,"maker":10297,"senal":10298,"ĠGro":10299,"ouri":10300,"ĠRain":10301,"ĠAlliance":10302,"Ġlift":10303,"ela":10304,"SD":10305,"ĠCleveland":10306,"Ġranked":10307,"Ġstadium":10308,"Ġdeadly":10309,"ä¸":10310,"Ġriding":10311,"aria":10312,"ĠArmor":10313,"Ġdocumentation":10314,"ĠGreece":10315,"reek":10316,"Ġlens":10317,"ĠSa":10318,"Ġgross":10319,"ĠEmer":10320,"agers":10321,"ĠDub":10322,"ĠRh":10323,"ĠAMD":10324,"Ġarrival":10325,"Ġdesert":10326,"Ġsupplement":10327,"ĠResp":10328,"Ġknee":10329,"Ġmargin":10330,"font":10331,"ogg":10332,"2010":10333,"ĠPir":10334,"ĠProm":10335,"ivals":10336,"Ġintake":10337,"Ġdifferently":10338,"ugs":10339,"Ġbits":10340,"cluded":10341,"Ġsearching":10342,"ĠDu":10343,"umble":10344,"Ġfunctional":10345,"ĠBaltimore":10346,"ĠCould":10347,"Ġdesired":10348,"Ġcircuit":10349,"ĠLyn":10350,"ĠGO":10351,"ĠFalse":10352,"repre":10353,"':":10354,"alties":10355,"Ġminim":10356,"Ġdrove":10357,"ĠShould":10358,"Ġhip":10359,"Ġpros":10360,"Ġutility":10361,"ĠNature":10362,"ĠMode":10363,"President":10364,"opp":10365,"rat":10366,"formance":10367,"Ġconcentration":10368,"Ġfont":10369,"ĠBud":10370,"Ġamid":10371,"Ġrevers":10372,"ĠML":10373,"Bar":10374,"Ġinteraction":10375,"Ġjurisd":10376,"Ġspells":10377,"dep":10378,"fil":10379,"Ġcivilians":10380,"utter":10381,"ĠCooper":10382,"ĠBelow":10383,"Ġentrance":10384,"Ġconvert":10385,"Ġcontroversy":10386,"owered":10387,"Ġcontrary":10388,"Ġarc":10389,"ĠExecutive":10390,"ĠOfficer":10391,"Ġpackages":10392,"Ġprogressive":10393,"width":10394,"Ġreserved":10395,"vol":10396,"ĠSamsung":10397,"Ġprinted":10398,"Ġcenters":10399,"Ġintroduce":10400,"ĠKennedy":10401,"Ġodds":10402,"Ġsurely":10403,"Ġindependence":10404,"Ġpassengers":10405,"reprene":10406,"ĠBeh":10407,"Ġloves":10408,"ĠESPN":10409,"Ġfacilit":10410,"Ġidentical":10411,"Ġdoct":10412,"Ġpartnership":10413,"conf":10414,"ĠHide":10415,"Ġconfused":10416,"ĠCow":10417,"Men":10418,"Ġwrest":10419,"ĠIraqi":10420,"Ġholes":10421,"ĠStudies":10422,"Ġpregnant":10423,"hard":10424,"Ġsignals":10425,"IX":10426,"Ġpulling":10427,"Ġgraduate":10428,"Ġnominee":10429,"Date":10430,"Ġpermitted":10431,"ĠâĤ¬":10432,"ĠOklahoma":10433,"Start":10434,"Ġauthorized":10435,"Ġalarm":10436,"ĠCos":10437,"van":10438,"Ġgenerations":10439,"cular":10440,"Ġdragon":10441,"ĠSoftware":10442,"ĠEdward":10443,"Ġcontroller":10444,"Sen":10445,"gered":10446,"ĠVik":10447,"Ġapproached":10448,"Thank":10449,"Ġcance":10450,"Ġformula":10451,"ĠSmall":10452,"Ġweakness":10453,"Ġramp":10454,"itudes":10455,"jud":10456,"Ġbrilliant":10457,"Ġaccus":10458,"source":10459,"Ġ800":10460,"ĠEvil":10461,"Sw":10462,"Ġhomeless":10463,"week":10464,"iens":10465,"rics":10466,"ĠThird":10467,"TO":10468,"Ġorganic":10469,"Ġpresentation":10470,"agh":10471,"ĠDownload":10472,"vation":10473,"Ġassembly":10474,"orable":10475,"holders":10476,"ĠBernie":10477,"ĠHelp":10478,"Ġtong":10479,"ĠFight":10480,"Ġbeach":10481,"Book":10482,"ĠLic":10483,"Ġrush":10484,"ĠRound":10485,"oup":10486,"ĠMarx":10487,"Ġcalculated":10488,"ĠDevil":10489,"ĠSarah":10490,"Ġoccasionally":10491,"Ġbullet":10492,"Available":10493,"gate":10494,"Ġ91":10495,"Ġhosp":10496,"Ġpromises":10497,"ĠHIV":10498,"ĠStadium":10499,"ĠStock":10500,"ĠCorporation":10501,"gage":10502,"NG":10503,"ĠCredit":10504,"Ġsne":10505,"ibl":10506,"Ġaccum":10507,"such":10508,"Ġterrorists":10509,"Ġconsciousness":10510,"ĠZh":10511,"Ġdrama":10512,"oola":10513,"piration":10514,"Ġlabour":10515,"ĠNin":10516,"Ġutter":10517,"Ġdemocratic":10518,"Ġassass":10519,"ilation":10520,"Ġgest":10521,"Ġabroad":10522,"Ġmetab":10523,"Ġsorts":10524,"Ġflav":10525,"UB":10526,"Ġmg":10527,"ĠNothing":10528,"ĠOd":10529,"Ġmusical":10530,"2009":10531,"Ġdrops":10532,"ocated":10533,"ateral":10534,"000000":10535,"Ġgre":10536,"Ġequality":10537,"Ġburden":10538,"Ġvig":10539,"ĠLeader":10540,"------------":10541,"Ġceremony":10542,"Ġfighter":10543,"Ġactors":10544,"Ġæ":10545,"aman":10546,"Fi":10547,"Ġalign":10548,"puter":10549,"Ġelder":10550,"ĠNSA":10551,"Ġrepresentation":10552,"ĠOntario":10553,"ITH":10554,"usalem":10555,"Ġharassment":10556,"itzer":10557,"Ġsymp":10558,"Ġboxes":10559,"ĠDR":10560,"Ġmanifest":10561,"atre":10562,"Ġ^":10563,"Ġdies":10564,"leton":10565,"Ġmissions":10566,"ethe":10567,"Ġresolve":10568,"Ġfollowers":10569,"Ġasc":10570,"Ġkm":10571,"lord":10572,"ammed":10573,"Ġsilent":10574,"ĠAssociated":10575,"Ġtiming":10576,"Ġprisoners":10577,"ĠKings":10578,"ĠFive":10579,"Ġtower":10580,"Ġapproaches":10581,"Ġprecisely":10582,"Ġbureau":10583,"ĠMother":10584,"ĠIss":10585,"Ġkeyboard":10586,"itual":10587,"Ġfunded":10588,"Ġstaying":10589,"Ġpsychological":10590,"Ġmile":10591,"ĠLeon":10592,"ĠBarb":10593,"will":10594,"Ġwider":10595,"ĠAtlantic":10596,"Ġtill":10597,"ĠRome":10598,"rot":10599,"Ġaccompan":10600,"Ġflour":10601,"aco":10602,"World":10603,"ĠExpress":10604,"ĠYu":10605,"Cor":10606,"Ġpleased":10607,"party":10608,"Ġpointing":10609,"Ġinflation":10610,"Ġroy":10611,"Ġ),":10612,"ainer":10613,"Ġwedding":10614,"ormon":10615,"Ġrequiring":10616,"Ġqualified":10617,"Ġsegment":10618,"END":10619,"Ġsizes":10620,"eals":10621,"Ġcorrupt":10622,"assador":10623,"Ġceleb":10624,"Ġdreams":10625,"ĠMess":10626,"Ġchecking":10627,"ĠVersion":10628,"Ġpreparing":10629,"Ġactively":10630,"ĠDiff":10631,"Ġlux":10632,"ĠWinter":10633,"acteria":10634,"ĠNE":10635,"Ġdeputy":10636,"Ġtransgender":10637,"Ġsummary":10638,"Ġinher":10639,"eries":10640,"char":10641,"ĠYan":10642,"Ġknock":10643,"ĠPath":10644,"Ġlip":10645,"roller":10646,"Ġimpression":10647,"Ġcelebrate":10648,"Ġslide":10649,"Ġguests":10650,"Ġclip":10651,"FS":10652,"Ġsavings":10653,"Ġcaptain":10654,"Ġlegacy":10655,"ĠDenver":10656,"Ġwounded":10657,"taboola":10658,"ACT":10659,"Ġpursue":10660,"Ġoxy":10661,"Ġq":10662,"Ġsemi":10663,"ĠNeed":10664,"ĠAffairs":10665,"Ġobsc":10666,"Ġchecked":10667,"Ġdual":10668,"Code":10669,"ĠMD":10670,"lem":10671,"ulty":10672,"Ġ©":10673,"ĠElizabeth":10674,"Ġcenturies":10675,"arded":10676,"src":10677,"Ġevident":10678,"ennis":10679,"atin":10680,"Ġunemployment":10681,"ĠMario":10682,"Ġintim":10683,"Christ":10684,"Ġbiological":10685,"Ġsoldier":10686,"ĠAdded":10687,"Ġmath":10688,"ĠGil":10689,"Ġbias":10690,"Ġdating":10691,"ĠOcean":10692,"Ġmice":10693,"Mus":10694,"hire":10695,"ĠTes":10696,"Server":10697,"limited":10698,"Size":10699,"Ġmeters":10700,"Ġrocket":10701,"essee":10702,"Ġcertificate":10703,"ĠIranian":10704,"ASS":10705,"Ġgrid":10706,"Dec":10707,"Ġrolling":10708,"commun":10709,"ĠSweden":10710,"bury":10711,"Ġtissue":10712,"Ġracism":10713,"ĠLocal":10714,"Ġmystery":10715,"Ġexamine":10716,"Ġstem":10717,"Ġsits":10718,"Ġhoped":10719,"oting":10720,"Ġdialogue":10721,"Ġpersu":10722,"Watch":10723,"lay":10724,"MAN":10725,"Ġchronic":10726,"ĠPortland":10727,"market":10728,"ĠSEC":10729,"Ġparallel":10730,"Ġscandal":10731,"Ġcarries":10732,"Ġphenomenon":10733,"human":10734,"acker":10735,"ĠOx":10736,"Ġretirement":10737,"tainment":10738,"ovie":10739,"ĠGear":10740,"Ġduties":10741,"Ġdose":10742,"Ġscroll":10743,"MB":10744,"inf":10745,"Ġsauce":10746,"Ġlandscape":10747,"reddit":10748,"ĠChampionship":10749,"ĠReddit":10750,"alid":10751,"Ġcoin":10752,"Ġovers":10753,"Ġposting":10754,"about":10755,"Ġfel":10756,"andy":10757,"Ġbold":10758,"Ġfocusing":10759,"effect":10760,"GR":10761,"Ġdeemed":10762,"Ġrecommendations":10763,"Ġstepped":10764,"Ġvoter":10765,"ĠDeep":10766,"ĠInstagram":10767,"Ġmoderate":10768,"ĠMaryland":10769,"Ġrestricted":10770,"ĠMB":10771,"ĠChall":10772,"Ġtob":10773,"Ġcir":10774,"ĠOcc":10775,"ĠEver":10776,"Ġcollaps":10777,"INFO":10778,"=-":10779,"ĠPict":10780,"ĠAccount":10781,"nc":10782,"Ġought":10783,"Ġexport":10784,"Ġdrunk":10785,"('":10786,"Ġwise":10787,"ĠMort":10788,"necess":10789,"Ġancest":10790,"ĠIncre":10791,"Ġfrequent":10792,"mir":10793,"Ġinterpretation":10794,"Ġdependent":10795,"Ġcoins":10796,"ĠBol":10797,"Video":10798,"ĠJustin":10799,"Ġfatal":10800,"Ġcooking":10801,"Ġconfusion":10802,"ipher":10803,"Ġcustody":10804,"ĠMorgan":10805,"omach":10806,"ĠGovernor":10807,"Ġrestaurants":10808,"eling":10809,"Ġacknowledged":10810,"Ġther":10811,"Ġgenes":10812,"ching":10813,"Hey":10814,"Ġtactics":10815,"ĠMexican":10816,"Ġvend":10817,"Ġhes":10818,"quer":10819,"Ġnoting":10820,"ĠCameron":10821,"Ġtargeting":10822,"rock":10823,"Ġcredits":10824,"Ġemotions":10825,"Ġrepresentatives":10826,"news":10827,"Ġlegislative":10828,"Ġremoving":10829,"Ġtweeted":10830,"ĠCarter":10831,"ĠFixed":10832,"Ġforcing":10833,"Ġspeaker":10834,"Ġmales":10835,"ĠVietnam":10836,"lined":10837,"Ġconcepts":10838,"Ġvoices":10839,"oir":10840,"ĠTrib":10841,"Whe":10842,"ĠJerusalem":10843,"ĠSant":10844,"Ġcul":10845,"Ġlady":10846,"ĠHawai":10847,"Ġarts":10848,"ĠInn":10849,"ĠMachine":10850,"ĠEmperor":10851,"Ġslot":10852,"gly":10853,"ĠProcess":10854,"III":10855,"Ġathletes":10856,"ĠTemple":10857,"ĠRepresent":10858,"Ġpresc":10859,"Ġtons":10860,"Ġgolden":10861,"Ġpunch":10862,"ĠGR":10863,"iverpool":10864,"Ġenact":10865,"Ġlobby":10866,"Ġmos":10867,"Ġpicking":10868,"Ġlifetime":10869,"Ġcognitive":10870,"Each":10871,"zo":10872,"Ġdub":10873,"Ġconsists":10874,"oln":10875,"Ġfestival":10876,"amous":10877,"Ġintellig":10878,"words":10879,"ĠSmart":10880,"Ġdele":10881,"Ġlapt":10882,"Ġmagical":10883,"ĠSin":10884,"bus":10885,"urities":10886,"ighth":10887,"ĠRuby":10888,"ĠSure":10889,"olving":10890,"Ġjun":10891,"OST":10892,"Ġimposed":10893,"Ġastron":10894,"Ġcorrel":10895,"ĠNS":10896,"ĠKit":10897,"ĠFuture":10898,"burn":10899,"Ġimmune":10900,"ocus":10901,"Ġcourses":10902,"ĠString":10903,"Ġlean":10904,"Ġghost":10905,"Ġoutcomes":10906,"Ġexpense":10907,"Ġeveryday":10908,"Ġacceptable":10909,"Ah":10910,"Ġequipped":10911,"Ġorange":10912,"FR":10913,"ĠDutch":10914,"Though":10915,"ĠRank":10916,"QU":10917,"ĠRoberts":10918,"what":10919,"rend":10920,"Ġdisappear":10921,"Ġspawn":10922,"ĠLam":10923,"ois":10924,"Ġdeserve":10925,"Ġminimal":10926,"Ġnervous":10927,"ĠWould":10928,"Ġrook":10929,"ĠVancouver":10930,"Ġresign":10931,"shire":10932,"ĠWorks":10933,"ĠBuild":10934,"Ġaffordable":10935,"ĠGary":10936,"ĠArena":10937,"Ġhanging":10938,"Ġimplications":10939,"ĠSong":10940,"Ġmaintaining":10941,"Ġguards":10942,"CON":10943,"Ġderived":10944,"Ġexecuted":10945,"Ġtheories":10946,"Ġquoted":10947,"ĠAndre":10948,"oga":10949,"seless":10950,"info":10951,"ĠBelg":10952,"Ġtears":10953,"ĠSurv":10954,"Ġbirthday":10955,"igious":10956,"immer":10957,"Ġspectrum":10958,"Ġarchitecture":10959,"Ġrecruit":10960,"arma":10961,"Table":10962,"Ġmonsters":10963,"ĠGov":10964,"Ġdestination":10965,"Ġattractive":10966,"Ġfoss":10967,"ĠMoreover":10968,"Ġpresents":10969,"THE":10970,"Ġreply":10971,"pton":10972,"Ġcum":10973,"Ġdelight":10974,"Ġaffects":10975,"Ġdonations":10976,"ĠToy":10977,"ĠHim":10978,"MENT":10979,"Ġovercome":10980,"itched":10981,"ĠFantasy":10982,"ĠHat":10983,"ĠBeast":10984,"bott":10985,"Ġinvestigations":10986,"Run":10987,"Ġhunting":10988,"di":10989,"fund":10990,"Ġsessions":10991,"estyle":10992,"Ġportray":10993,"oids":10994,"Yeah":10995,"Ġcommunicate":10996,"Ġcomedy":10997,"ĠYang":10998,"Ġbelt":10999,"ĠMarine":11000,"Ġpredicted":11001,"Play":11002,"Ġimportantly":11003,"Ġremarkable":11004,"Ġeliminate":11005,"David":11006,"Ġbind":11007,"VID":11008,"Ġadvocates":11009,"ĠGaza":11010,"imp":11011,"DB":11012,"ĠNa":11013,"ĠSimilar":11014,"IES":11015,"Ġcharity":11016,"vas":11017,"math":11018,"Ġâĸ":11019,"oker":11020,"ndum":11021,"Ġcaps":11022,"ĠHal":11023,"2000":11024,"ean":11025,"Ġfleet":11026,"Ġrecre":11027,"Right":11028,"Ġsleeping":11029,"ijing":11030,"kind":11031,"Ġdesignated":11032,"ä":11033,"Ġanimation":11034,"kee":11035,"ĠIntrodu":11036,"Ġ/>":11037,"Ġdelayed":11038,"Ġtremend":11039,"Ġcurious":11040,"Use":11041,"Ġlect":11042,"dam":11043,"Ġinnovation":11044,"ĠPoints":11045,"Ġloading":11046,"Ġdispute":11047,"ctic":11048,"irds":11049,"ĠBY":11050,"Ġnurs":11051,"ĠValue":11052,"IONS":11053,"ĠHum":11054,"Ġtemplate":11055,"mers":11056,"Ġappearances":11057,"ĠEntertainment":11058,"Ġtranslation":11059,"Ġsake":11060,"Ġbeneath":11061,"Ġinhib":11062,"Ġeuro":11063,"abetes":11064,"Ġstudying":11065,"ĠMas":11066,"Ġperceived":11067,"Ġexamined":11068,"Ġeager":11069,"Ġcoaches":11070,"Ġimper":11071,"chi":11072,"Ġproduces":11073,"\").":11074,"ĠEveryone":11075,"Ġmunicip":11076,"Ġgirlfriend":11077,"Ġhire":11078,"ĠVice":11079,"Ġsuitable":11080,"opy":11081,"Ġinequ":11082,"ĠDuke":11083,"fish":11084,"first":11085,"ĠObs":11086,"Ġinterior":11087,"ĠBruce":11088,"ĠRy":11089,"Ġanalys":11090,"Ġconsiderable":11091,"Ġforecast":11092,"Ġfert":11093,"orship":11094,"ĠDrug":11095,"ĠALL":11096,":\"":11097,"thur":11098,"ĠMail":11099,"Ġballot":11100,"Ġinstantly":11101,"ĠChannel":11102,"Ġpicks":11103,"Ġ1989":11104,"Ġtent":11105,"oli":11106,"Ġcivilian":11107,"bling":11108,"ello":11109,"bu":11110,"Ġinch":11111,"Ġlogo":11112,"Ġcooperation":11113,"Ġwalks":11114,"Ġinvestments":11115,"Ġimprison":11116,"ĠFestival":11117,"ĠKy":11118,"Ġlegally":11119,"Ġgri":11120,"charg":11121,"Sl":11122,"Ġthreatening":11123,"duction":11124,"flow":11125,"Ġdismissed":11126,"ibraries":11127,"cap":11128,"ele":11129,"ĠMcG":11130,"ĠHarvard":11131,"ĠConservative":11132,"ĠCBS":11133,"png":11134,"Ġroots":11135,"ĠHaving":11136,"umbled":11137,"ĠFun":11138,"\\/":11139,"ĠSearch":11140,"plex":11141,"Ġdiscussing":11142,"Ġcontinu":11143,"ĠTai":11144,"ĠWik":11145,"Free":11146,"fit":11147,"Ġrefuse":11148,"Ġmanaging":11149,"Ġsynd":11150,"ipedia":11151,"walk":11152,"Ġprofessionals":11153,"Ġguidance":11154,"Ġuniversities":11155,"Ġassemb":11156,"untu":11157,"Finally":11158,"ASE":11159,"ĠAuto":11160,"ĠHad":11161,"Ġanniversary":11162,"LD":11163,"ĠDur":11164,"ĠUltimate":11165,"ihad":11166,"product":11167,"Ġtransit":11168,"Ġrestore":11169,"Ġexplaining":11170,"Ġasset":11171,"Ġtransferred":11172,"Ġburst":11173,"apolis":11174,"ĠMagazine":11175,"ĠCra":11176,"ĠBR":11177,"gged":11178,"ĠHE":11179,"Mich":11180,"bet":11181,"ĠLady":11182,"ylum":11183,"erves":11184,"Ġmeets":11185,"white":11186,"Log":11187,"Ġcorresponding":11188,"Ġinsisted":11189,"GG":11190,"Ġsurrounded":11191,"Ġtens":11192,"Ġlane":11193,"Ġcoinc":11194,"home":11195,"Ġexisted":11196,"ected":11197,"ĠDouble":11198,"lamm":11199,"Ġskept":11200,"exp":11201,"Ġperception":11202,"iev":11203,"ĠBeing":11204,"oft":11205,"Ġadopt":11206,".:":11207,"];":11208,"Windows":11209,"Ġsatellite":11210,"ASH":11211,"Ġinfant":11212,"description":11213,"ĠMeanwhile":11214,"cm":11215,"oca":11216,"ĠTreat":11217,"actor":11218,"Ġtobacco":11219,"ĠNorm":11220,"emption":11221,"Ġflesh":11222,"Ġje":11223,"oop":11224,"ĠHeaven":11225,"Ġbeating":11226,"anim":11227,"Ġgathering":11228,"Ġcultiv":11229,"GO":11230,"abe":11231,"ĠJonathan":11232,"ĠSafety":11233,"Ġbadly":11234,"prot":11235,"Ġchoosing":11236,"Ġcontacted":11237,"Ġquit":11238,"Ġdistur":11239,"Ġstir":11240,"Ġtoken":11241,"Det":11242,"ĠPa":11243,"Ġfunctionality":11244,"003":11245,"some":11246,"Ġlimitations":11247,"Ġmeth":11248,"build":11249,"config":11250,"NT":11251,"rell":11252,"blem":11253,"ĠMom":11254,"Ġveterans":11255,"ĠHu":11256,"Ġtrends":11257,"arer":11258,"ĠGiven":11259,"ĠCaption":11260,"may":11261,"AST":11262,"Ġwondering":11263,"ĠClark":11264,"normal":11265,"Ġseparated":11266,"Ġdesp":11267,"stic":11268,"brew":11269,"Ġrelating":11270,"ĠNik":11271,"ĠFarm":11272,"Ġenthusi":11273,"good":11274,"deb":11275,"Ġactivist":11276,"Ġmart":11277,"Ġexplosion":11278,"ĠEconomic":11279,"Link":11280,"Ġinsight":11281,"Ġconvenient":11282,"Ġcounterpart":11283,"support":11284,"ĠVirt":11285,"agen":11286,"ĠTennessee":11287,"ĠSimon":11288,"ĠAward":11289,"OCK":11290,"ĠFigure":11291,"Ġoverseas":11292,"Ġpride":11293,"ĠCas":11294,"note":11295,"mg":11296,"Current":11297,"Ġdisplays":11298,"content":11299,"Ġtraveling":11300,"Ġhospitals":11301,"ĠFinancial":11302,"ĠPast":11303,"Ġdefendant":11304,"Ġstreaming":11305,"mble":11306,"ĠBerlin":11307,"uki":11308,"Ġdistribut":11309,"Ġantib":11310,"Ġchocolate":11311,"ĠCastle":11312,"Ġinterrupt":11313,"ĠRow":11314,"Ġconversion":11315,"Ġbugs":11316,"ĠRather":11317,"liest":11318,"LY":11319,"ĠJean":11320,"common":11321,"akh":11322,"Ġ130":11323,"otton":11324,"ĠDean":11325,"Ġamendment":11326,"Ġgameplay":11327,"ĠWarren":11328,"oda":11329,"Ġhighlights":11330,"Ġirre":11331,"ĠNATO":11332,"Ġballs":11333,"Ġdemanding":11334,"URE":11335,"ĠLuke":11336,"Figure":11337,"stop":11338,"onia":11339,"zone":11340,"izers":11341,"ĠWR":11342,"Ġawarded":11343,"Ġregulatory":11344,"ĠHart":11345,"ĠSN":11346,"pling":11347,"Ġsour":11348,"ĠPixel":11349,"usive":11350,"Ġfet":11351,"ĠSent":11352,"Ġautomatic":11353,"Ġfer":11354,"vernment":11355,"ĠKhan":11356,"TON":11357,"father":11358,"Ġextraordinary":11359,"throp":11360,"ĠPython":11361,"ĠGPU":11362,"Ġsexually":11363,"Ġdesktop":11364,"itivity":11365,"ĠAntonio":11366,"Ġorient":11367,"Ġears":11368,"obby":11369,"ouses":11370,"vertisements":11371,"Ġmanufacturers":11372,"icient":11373,"minute":11374,"Ġconviction":11375,"Ġgarden":11376,"public":11377,"Ġsatisfied":11378,"fold":11379,"OK":11380,"Ġinhab":11381,"ĠThink":11382,"Ġprogramme":11383,"Ġstomach":11384,"Ġcoordin":11385,"Ġholy":11386,"Ġthreshold":11387,"Ġrhet":11388,"Ġserial":11389,"Ġemployers":11390,"ĠEverything":11391,"rah":11392,"Ġbother":11393,"Ġbrands":11394,"Value":11395,"ĠTed":11396,"ĠPlanet":11397,"Ġpink":11398,"ĠFurthermore":11399,"sa":11400,"PE":11401,"reck":11402,"ĠUSD":11403,"otte":11404,"Ġ&&":11405,"Ġlanded":11406,"gets":11407,"Ġproducers":11408,"Ġhealthcare":11409,"Ġdominant":11410,"Ġdestro":11411,"Ġamended":11412,"chron":11413,"Ġfits":11414,"ĠSyd":11415,"ĠAuthority":11416,"ATCH":11417,"Ġfights":11418,"ĠLLC":11419,"Ġ---":11420,"ĠCorp":11421,"Ġtoxic":11422,"specific":11423,"ĠCorn":11424,"ĠChel":11425,"Ġtelephone":11426,"ĠPant":11427,"Ġmysterious":11428,"aunch":11429,"odox":11430,"media":11431,"Ġwitnesses":11432,"agu":11433,"Ġquestioned":11434,"ĠBrexit":11435,"ĠRemember":11436,"enez":11437,"Ġendorse":11438,"iatric":11439,"ĠIdent":11440,"Ġridiculous":11441,"110":11442,"Ġprayer":11443,"Ġscientist":11444,"Ġ1950":11445,"ĠAqu":11446,"Ġunderground":11447,"ĠUFC":11448,"mare":11449,"ĠLater":11450,"wich":11451,"Ġsubscrib":11452,"Ġhosts":11453,"Ġerr":11454,"Ġgrants":11455,"antom":11456,"Ġsummon":11457,"early":11458,"ĠClear":11459,"ĠPrim":11460,"Ġsuspension":11461,"Ġguaranteed":11462,"apper":11463,"Ġrice":11464,"ĠSean":11465,"ĠShin":11466,"Ġreferendum":11467,"Ġfled":11468,"rust":11469,"Ġ360":11470,"tery":11471,"Ġshocked":11472,"BR":11473,"ĠOil":11474,"ĠAllah":11475,"Ġpartly":11476,"Ġignor":11477,"Ġtransmission":11478,"Ġhomosexual":11479,"iversal":11480,"Ġhopefully":11481,"ãĤ¤":11482,"Ġlesson":11483,"Leg":11484,"Ġ..":11485,"Yet":11486,"table":11487,"appropri":11488,"rett":11489,"Ġboards":11490,"Ġincorrect":11491,"Ġbacteria":11492,"aru":11493,"amac":11494,"Ġsnap":11495,".'\"":11496,"Ġparad":11497,"tem":11498,"heart":11499,"Ġavailability":11500,"Ġwisdom":11501,"Ġ(+":11502,"Ġpriest":11503,"ĠÂłĠÂł":11504,"Open":11505,"Ġspan":11506,"Ġparameter":11507,"Ġconvince":11508,"Ġ(%)":11509,"rac":11510,"Ġfo":11511,"Ġsafely":11512,"Ġconverted":11513,"ĠOlympic":11514,"Ġreserve":11515,"Ġhealing":11516,"ĠMine":11517,"Max":11518,"Ġinherent":11519,"ĠGraham":11520,"Ġintegrated":11521,"Dem":11522,"Ġpipeline":11523,"Ġapplying":11524,"Ġembed":11525,"ĠCharlie":11526,"Ġcave":11527,"2008":11528,"Ġconsensus":11529,"Ġrewards":11530,"Pal":11531,"ĠHTML":11532,"Ġpopularity":11533,"looking":11534,"ĠSword":11535,"ĠArts":11536,"')":11537,"Ġelectron":11538,"clusions":11539,"Ġintegrity":11540,"Ġexclusively":11541,"Ġgrace":11542,"Ġtorture":11543,"Ġburned":11544,"two":11545,"Ġ180":11546,"Produ":11547,"Ġentreprene":11548,"raphics":11549,"Ġgym":11550,"ricane":11551,"ĠTam":11552,"Ġadministrative":11553,"Ġmanufacturer":11554,"Ġvel":11555,"ĠNi":11556,"Ġisolated":11557,"ĠMedicine":11558,"Ġbackup":11559,"Ġpromoting":11560,"Ġcommander":11561,"Ġflee":11562,"ĠRussell":11563,"Ġforgotten":11564,"ĠMissouri":11565,"Ġresidence":11566,"mons":11567,"Ġresemb":11568,"Ġwand":11569,"Ġmeaningful":11570,"PT":11571,"Ġbol":11572,"Ġhelic":11573,"Ġwealthy":11574,"Ġrifle":11575,"strong":11576,"rowing":11577,"plan":11578,"asury":11579,"â̦.":11580,"Ġexpanding":11581,"ĠHamilton":11582,"Ġreceives":11583,"SI":11584,"eatures":11585,"ĠAnim":11586,"REE":11587,"Put":11588,"Ġbriefly":11589,"rive":11590,"Ġstimul":11591,"Ġ``(":11592,"Ġ__":11593,"Ġchip":11594,"Ġhaz":11595,"Ġprize":11596,"ĠThings":11597,"ACE":11598,"ulin":11599,"dict":11600,"oku":11601,"Ġassociate":11602,"ockets":11603,"youtube":11604,"Story":11605,"ategory":11606,"Ġmild":11607,"ailing":11608,"ĠYe":11609,"Orig":11610,"ĠKa":11611,"orig":11612,"Ġpropaganda":11613,"Ġanonymous":11614,"Ġstruggled":11615,"Ġoutrage":11616,"ATED":11617,"ĠBeijing":11618,"rary":11619,"Ġleather":11620,"Ġworlds":11621,"Ġbroader":11622,"125":11623,"idal":11624,"ĠBetter":11625,"Ġtear":11626,"Ext":11627,"Ġproposals":11628,"Ġiter":11629,"ĠSquad":11630,"Ġvolunt":11631,"mi":11632,"Did":11633,"ĠPu":11634,"pin":11635,"Ġspeakers":11636,"Ġborders":11637,"Ġfigured":11638,"='":11639,"Ġsimultaneously":11640,"aeda":11641,"Ġcharging":11642,"Ġurged":11643,"Ġconj":11644,"256":11645,"ĠGordon":11646,"merce":11647,"Ġdocumentary":11648,"Share":11649,"itol":11650,"ONE":11651,"ĠGarden":11652,"hatt":11653,"ĠThompson":11654,"aneous":11655,"apore":11656,"Ġtanks":11657,"Ġlessons":11658,"track":11659,"Ġoutstanding":11660,"Ġvolunteers":11661,"Ġspray":11662,"Ġmanagers":11663,"large":11664,"Ġcamps":11665,"Ġartificial":11666,"ĠRu":11667,"Ġbags":11668,"thal":11669,"Ġcompatible":11670,"ĠBlade":11671,"Ġfed":11672,"Ġargues":11673,"FI":11674,"Ġunfair":11675,"Ġcorn":11676,"Ġoffset":11677,"Ġdirections":11678,"Ġdisappointed":11679,"ĠConvention":11680,"Ġviewing":11681,"ME":11682,"ocity":11683,"Ġtowns":11684,"Ġlayers":11685,"Ġrolled":11686,"Ġjumped":11687,"Ġattribute":11688,"Ġunnecess":11689,"incoln":11690,"Ġsuppose":11691,"ĠNether":11692,"cha":11693,"Ġburied":11694,"Ġsixth":11695,"Ben":11696,"ressing":11697,"OUR":11698,"Ġwound":11699,"Ġcycl":11700,"Ġmechanisms":11701,"Ġcongressional":11702,"ĠElement":11703,"Ġagreements":11704,"Ġdecor":11705,"Ġclosest":11706,"ĠMit":11707,"Google":11708,"}}":11709,"Ġmixture":11710,"Ġfluid":11711,"Sign":11712,"ĠScholar":11713,"Ġpist":11714,"asket":11715,"abling":11716,"Ġracing":11717,"hero":11718,"riel":11719,"assy":11720,"Ġcheaper":11721,"ben":11722,"Ġvertical":11723,"amacare":11724,"ĠReading":11725,"gments":11726,"Ġhelicop":11727,"Ġsacrifice":11728,"aya":11729,"paren":11730,"VA":11731,"ĠLes":11732,"ĠStudio":11733,"Ġviolations":11734,"ĠAnna":11735,"acer":11736,"é¾":11737,"ĠRat":11738,"ĠBeck":11739,"ĠDick":11740,"ĠACT":11741,"Ġcomposition":11742,"Ġtexture":11743,"ĠOwn":11744,"Ġsmartphone":11745,"ĠNA":11746,"Ġforb":11747,"import":11748,"Ġdefending":11749,"ilst":11750,"rer":11751,"Ġoh":11752,"ĠJeremy":11753,"Ġbanking":11754,"ceptions":11755,"Ġrespective":11756,"/.":11757,"Ġdrinks":11758,"ĠWi":11759,"Ġbands":11760,"ĠLiverpool":11761,"Ġgrip":11762,"ĠBuy":11763,"Ġopenly":11764,"Ġreviewed":11765,"pert":11766,"Ġverify":11767,"ĠCole":11768,"ĠWales":11769,"MO":11770,"Ġunpre":11771,"Ġshelter":11772,"ĠImperial":11773,"Ġgui":11774,"ĠDak":11775,"Ġsuggestions":11776,"Ġexplicitly":11777,"Ġslave":11778,"Ġblockchain":11779,"Ġcompeting":11780,"Ġpromising":11781,"SON":11782,"Ġsoccer":11783,"Ġconstitution":11784,"429":11785,"Ġdistract":11786,"ĠUser":11787,"esides":11788,"ĠMethod":11789,"ĠTokyo":11790,"Ġaccompanied":11791,"Client":11792,"sur":11793,"alog":11794,"Ġidentification":11795,"Ġinvasion":11796,"asma":11797,"Ġindustries":11798,"ppers":11799,"Ġsubtle":11800,"ĠUnit":11801,"natural":11802,"Ġsurvived":11803,"Ġflaw":11804,"ĺħ":11805,"ĠHoll":11806,"Ġdeficit":11807,"Ġtutorial":11808,"ĠChance":11809,"Ġarguing":11810,"Ġcontemporary":11811,"Ġintegration":11812,"forward":11813,"Ġtum":11814,"itis":11815,"Ġhiding":11816,"ĠDomin":11817,"ĠTan":11818,"ĠBuilding":11819,"ĠVin":11820,"Ġspokesperson":11821,"ĠNotes":11822,"Ġemerging":11823,"Ġpreparation":11824,"Ġprost":11825,"Ġsuspects":11826,"Ġautonom":11827,"Description":11828,"Ġdealt":11829,"ĠPear":11830,"Ġsteady":11831,"Ġdecreased":11832,"Ġsovere":11833,"ĠClin":11834,"Ġgradually":11835,"orses":11836,"ĠWAR":11837,"Serv":11838,"ãĤ¢":11839,"hr":11840,"Ġdirty":11841,"ĠBarn":11842,"ĠBC":11843,"Ġdil":11844,"Ġcalendar":11845,"Ġcompliance":11846,"Ġchamber":11847,"bb":11848,"Ġpassenger":11849,"ateful":11850,"ĠTitle":11851,"ĠSydney":11852,"ĠGot":11853,"Ġdarkness":11854,"Ġdefect":11855,"Ġpacked":11856,"assion":11857,"Ġgods":11858,"Ġharsh":11859,"ICK":11860,"leans":11861,"Ġalgorithm":11862,"Ġoxygen":11863,"Ġvisits":11864,"Ġblade":11865,"Ġkilomet":11866,"ĠKentucky":11867,"Ġkiller":11868,"Pack":11869,"enny":11870,"Ġdivine":11871,"Ġnomination":11872,"being":11873,"Ġengines":11874,"Ġcats":11875,"Ġbuffer":11876,"ĠPhill":11877,"Ġtraff":11878,"AGE":11879,"Ġtongue":11880,"Ġradiation":11881,"erer":11882,"mem":11883,"ĠExplicit":11884,"é¾į":11885,"Ġcouples":11886,"Ġphysics":11887,"ĠMcK":11888,"Ġpolitically":11889,"awks":11890,"ĠBloom":11891,"Ġworship":11892,"eger":11893,"uter":11894,"ĠFO":11895,"Ġmathemat":11896,"Ġsentenced":11897,"Ġdisk":11898,"ĠMarg":11899,"Ġ/*":11900,"PI":11901,"Ġoptional":11902,"Ġbabies":11903,"Ġseeds":11904,"ĠScottish":11905,"Ġthy":11906,"]]":11907,"ĠHitler":11908,"PH":11909,"ngth":11910,"Ġrecovered":11911,"inge":11912,"Ġpowder":11913,"Ġlips":11914,"Ġdesigner":11915,"Ġdisorders":11916,"Ġcourage":11917,"Ġchaos":11918,"\"},{\"":11919,"Ġcarrier":11920,"bably":11921,"High":11922,"ĠRT":11923,"esity":11924,"len":11925,"Ġroutes":11926,"uating":11927,"Fil":11928,"NOT":11929,"wall":11930,"sburgh":11931,"Ġengaging":11932,"ĠJavaScript":11933,"orer":11934,"lihood":11935,"Ġunions":11936,"ĠFederation":11937,"ĠTesla":11938,"Ġcompletion":11939,"ĠTa":11940,"Ġprivilege":11941,"ĠOrange":11942,"Ġneur":11943,"parency":11944,"Ġbones":11945,"Ġtitled":11946,"Ġprosecutors":11947,"ĠME":11948,"Ġengineer":11949,"ĠUniverse":11950,"ĠHig":11951,"nie":11952,"oard":11953,"Ġhearts":11954,"ĠGre":11955,"ussion":11956,"Ġministry":11957,"Ġpenet":11958,"ĠNut":11959,"ĠOw":11960,"ĠXP":11961,"instein":11962,"Ġbulk":11963,"System":11964,"icism":11965,"ĠMarketable":11966,"Ġpreval":11967,"Ġposter":11968,"Ġattending":11969,"urable":11970,"Ġlicensed":11971,"ĠGh":11972,"etry":11973,"ĠTradable":11974,"Ġblast":11975,"à¤":11976,"ĠTitan":11977,"elled":11978,"die":11979,"Have":11980,"ĠFlame":11981,"Ġprofound":11982,"Ġparticipating":11983,"Ġanime":11984,"ĠEss":11985,"Ġspecify":11986,"Ġregarded":11987,"ĠSpell":11988,"Ġsons":11989,"owned":11990,"Ġmerc":11991,"Ġexperimental":11992,"lando":11993,"hs":11994,"ĠDungeon":11995,"inos":11996,"Ġcomply":11997,"ĠSystems":11998,"arth":11999,"Ġseized":12000,"local":12001,"ĠGirls":12002,"udo":12003,"oned":12004,"ĠFle":12005,"Ġconstructed":12006,"Ġhosted":12007,"Ġscared":12008,"actic":12009,"ĠIslands":12010,"ĠMORE":12011,"Ġbless":12012,"Ġblocking":12013,"Ġchips":12014,"Ġevac":12015,"Ps":12016,"Ġcorporation":12017,"Ġox":12018,"Ġlighting":12019,"Ġneighbors":12020,"ĠUb":12021,"aro":12022,"Ġbeef":12023,"ĠUber":12024,"Facebook":12025,"armed":12026,"itate":12027,"ĠRating":12028,"ĠQuick":12029,"Ġoccupied":12030,"Ġaims":12031,"ĠAdditionally":12032,"ĠInterest":12033,"Ġdramatically":12034,"Ġheal":12035,"Ġpainting":12036,"Ġengineers":12037,"MM":12038,"ĠMust":12039,"Ġquantity":12040,"Paul":12041,"Ġearnings":12042,"ĠPosts":12043,"stra":12044,"ãĥ¼ãĥ":12045,"Ġstance":12046,"Ġdropping":12047,"script":12048,"Ġdressed":12049,"Make":12050,"Ġjustify":12051,"ĠLtd":12052,"Ġprompted":12053,"Ġscrut":12054,"Ġspeeds":12055,"ĠGiants":12056,"omer":12057,"ĠEditor":12058,"Ġdescribing":12059,"ĠLie":12060,"mented":12061,"Ġnowhere":12062,"ocaly":12063,"Ġinstruction":12064,"fortable":12065,"Ġentities":12066,"Ġcm":12067,"ĠNatural":12068,"Ġinquiry":12069,"Ġpressed":12070,"izont":12071,"forced":12072,"Ġraises":12073,"ĠNetflix":12074,"ĠSide":12075,"Ġouter":12076,"Ġamongst":12077,"ims":12078,"owski":12079,"Ġclimb":12080,"never":12081,"Ġcombine":12082,"ding":12083,"Ġcompr":12084,"Ġsignificance":12085,"Ġremembered":12086,"ĠNevada":12087,"ĠTel":12088,"ĠScar":12089,"ĠWarriors":12090,"ĠJane":12091,"Ġcoup":12092,"bas":12093,"Ġterminal":12094,",-":12095,"OH":12096,"Ġtension":12097,"Ġwings":12098,"ĠMyster":12099,"����":12100,"ĠUnlike":12101,"valid":12102,"vironments":12103,"ĠAli":12104,"Ġnaked":12105,"books":12106,"ĠMun":12107,"ĠGulf":12108,"Ġdensity":12109,"Ġdimin":12110,"Ġdesperate":12111,"Ġpresidency":12112,"Ġ1986":12113,"hy":12114,"IND":12115,"Ġunlock":12116,"imens":12117,"Ġhandled":12118,"ĠEb":12119,"Ġdisappeared":12120,"Ġgenre":12121,"Ġ1988":12122,"Ġdetermination":12123,"Stream":12124,"iko":12125,"apters":12126,"Ġacknowledge":12127,"Jan":12128,"Ġcapitalism":12129,"Pat":12130,"Ġ2020":12131,"Ġpainful":12132,"Ġcurve":12133,"Ġbombs":12134,"storm":12135,"ĠMetal":12136,"encer":12137,"ĠFig":12138,"ĠAaron":12139,"anches":12140,"Ġinspiration":12141,"Ġexhaust":12142,"tains":12143,"ashi":12144,"Ġdescript":12145,"Ġritual":12146,"ĠChelsea":12147,"Ġpromotion":12148,"ĠHung":12149,"ĠWard":12150,"iva":12151,"ĠET":12152,"Ġtoss":12153,"allow":12154,"ĠFrancis":12155,"Dep":12156,"Ġhappiness":12157,"ĠGlass":12158,"Ġbeta":12159,"Ġstrengthen":12160,"NE":12161,"oa":12162,"Ġbuttons":12163,"ĠMurray":12164,"Ġkicked":12165,"Quest":12166,"ĠTalk":12167,"ĠSeveral":12168,"ĠZero":12169,"Ġdrone":12170,"ulk":12171,"Ġcam":12172,"ĠMobile":12173,"Ġpreventing":12174,"Ġretro":12175,"ĠAx":12176,"Ġcruel":12177,"Ġfloat":12178,".),":12179,"Ġfiling":12180,"ĠGrant":12181,"ĠBor":12182,"Ġrib":12183,"Ġchampionship":12184,"ĠMerc":12185,"Ġstyles":12186,"Ġcake":12187,"Ġbuilds":12188,"ĠSelf":12189,"iox":12190,"Ġepic":12191,"oyd":12192,"Bel":12193,"ĠStew":12194,".(":12195,"ahu":12196,"ĠBeyond":12197,"Ġouts":12198,"Ġsolo":12199,"ĠTree":12200,"Ġpreserve":12201,"Ġtub":12202,"ARE":12203,"roc":12204,"ĠImpro":12205,"ĠWright":12206,"Ġbund":12207,"Ġtraged":12208,"Ġoccasional":12209,"bian":12210,"Second":12211,"rons":12212,"Ġinteractions":12213,"formed":12214,"sing":12215,"Ġowns":12216,"Ġhockey":12217,"General":12218,"Ġlogical":12219,"Ġexpend":12220,"Ġescal":12221,"ĠGriff":12222,"ĠCrown":12223,"ĠReserve":12224,"Ġstopping":12225,"Ġexcuse":12226,"second":12227,"Ġoperated":12228,"Ġreaches":12229,"ĠMalays":12230,"Ġpollution":12231,"ĠBrooklyn":12232,"Ġdelete":12233,"Ġhash":12234,"Block":12235,"aha":12236,"â̳":12237,"Ġshorter":12238,"piece":12239,">>>":13163,"ĠMormon":13164,"tor":13165,"Ġparticles":13166,"ĠBart":13167,"ryption":13168,"Ġadmin":13169,"Ġsquee":13170,"VIDIA":13171,"Ġcreator":13172,"iameter":13173,"icular":13174,"NBC":13175,"Ġgrabbed":13176,"Ġnodd":13177,"Ġrated":13178,"Ġrotation":13179,"Ġgrasp":13180,"Ġexcessive":13181,"ĠEC":13182,"ĠWhit":13183,"Ġinventory":13184,"aults":13185,"ĠFB":13186,"Ġecosystem":13187,"Ġbillions":13188,"Ġventure":13189,"named":13190,"Ġdefender":13191,"oute":13192,"Instead":13193,"irable":13194,"War":13195,"Ġassumption":13196,"Ġbite":13197,"Ġearthqu":13198,"tail":13199,"space":13200,"Ġgifts":13201,"boys":13202,"Ġinevitable":13203,"Ġstructural":13204,"Ġbeneficial":13205,"Ġcompelling":13206,"hole":13207,"ervation":13208,"Ġcoat":13209,"oj":13210,"incarn":13211,"ĠYears":13212,"Ġdetermining":13213,"Ġrhetoric":13214,"Ġboundaries":13215,"Ġwhites":13216,"Ant":13217,"addy":13218,")-":13219,"raham":13220,"etermin":13221,"Ġharvest":13222,"ĠConc":13223,"Ġlaptop":13224,"ĠMatch":13225,"Ġenjoying":13226,"cca":13227,"ollar":13228,"Ġtrips":13229,"Ġaddiction":13230,"ĠSak":13231,"Ġpowered":13232,"Ġcous":13233,"ĠRussians":13234,"iere":13235,"Ġretrie":13236,"quality":13237,"Ġdiffer":13238,"Ġkingdom":13239,"ĠLaur":13240,"ĠCapitol":13241,"Ġconclusions":13242,"ĠAltern":13243,"ĠNav":13244,"Ġtransparent":13245,"BER":13246,"Group":13247,"ĠComplete":13248,"Ġinfer":13249,"Ġintrig":13250,"Ġinsane":13251,"RO":13252,"ophob":13253,"isen":13254,"qual":13255,"Michael":13256,"Ġmuseum":13257,"ĠPope":13258,"Ġreset":13259,"rative":13260,"five":13261,"Ġaggreg":13262,"ittees":13263,"ository":13264,"Ġcarb":13265,"ĠRecord":13266,"Ġdecides":13267,"ĠFix":13268,"Ġexceptions":13269,"ĠCommissioner":13270,"uns":13271,"ĠEnvironmental":13272,"Ġlegendary":13273,"istence":13274,"Ġtunnel":13275,"km":13276,"Ġinsult":13277,"Ġtroll":13278,"Ġshake":13279,"Ġdetention":13280,"ques":13281,"ĠChrome":13282,"ĠFiles":13283,"Ġsubt":13284,"Ġprospects":13285,"Ġprol":13286,"render":13287,"proof":13288,"Ġperformances":13289,"Str":13290,"Ġhref":13291,"ername":13292,"Ġachievement":13293,"Ġfut":13294,"Full":13295,"ĠLeban":13296,"google":13297,"ãĥĪ":13298,"ampa":13299,"Maybe":13300,"Ġprojected":13301,"ĠEmb":13302,"Ġcolleg":13303,"Ġawards":13304,"ĠâĶ":13305,"Gold":13306,"ĠBlake":13307,"ĠRaj":13308,"ifting":13309,"Ġpending":13310,"Ġinstinct":13311,"Ġdevelopments":13312,"Connect":13313,"ĠMand":13314,"ĠWITH":13315,"ĠPhilippines":13316,"profile":13317,"Ġaltogether":13318,"ĠBund":13319,"ĠTD":13320,"oooo":13321,"amped":13322,"iph":13323,"Ġsteam":13324,"Ġoldest":13325,"Ġdetection":13326,"ulpt":13327,"Ġç":13328,"ĠWayne":13329,"2006":13330,"fa":13331,"Ġcircles":13332,"ĠFu":13333,"Ġdonors":13334,"appropriate":13335,"ĠDakota":13336,"jamin":13337,"Ġmotivated":13338,"Ġpurchases":13339,"ĠLouisiana":13340,"ĠSpl":13341,"Ġglobe":13342,"Ġ105":13343,"zip":13344,"call":13345,"Ġdepartments":13346,"Ġsustainable":13347,"105":13348,"ĠOP":13349,"ifiers":13350,"Ġprevented":13351,"Ġincomp":13352,"ĠCommander":13353,"Ġdominated":13354,"Ġ»":13355,"Ġinvested":13356,"Ġcomplexity":13357,"Ġincl":13358,"Ġensuring":13359,"Ġrealm":13360,"ync":13361,"ĠIndependent":13362,"rained":13363,"ĠJen":13364,"ĠFlight":13365,"Ġathe":13366,"Ġspeculation":13367,"ĠTE":13368,"ocate":13369,"tic":13370,"Ġplaint":13371,"herry":13372,"Ġtoy":13373,"Ġ111":13374,"Ġplates":13375,"status":13376,"ĠIsa":13377,"Ġdevoted":13378,"Cop":13379,"ĠES":13380,"255":13381,"urrency":13382,"Main":13383,"Ġslaves":13384,"Ġpepper":13385,"Ġquotes":13386,"Ġceiling":13387,"ĠFish":13388,"Ġtransformation":13389,"Ġfraction":13390,"Ġadvantages":13391,"Ġtoile":13392,"Ġstunning":13393,"Ġmoist":13394,"breaking":13395,"si":13396,"ĠLocation":13397,"ĠMedium":13398,"Ġtexts":13399,"Ġugly":13400,"Ġbio":13401,".âĢĶ":13402,"ĠBased":13403,"Ġtrains":13404,"ĠWing":13405,"ĠAncient":13406,"ĠRecords":13407,"ĠHope":13408,"Special":13409,"adesh":13410,"obi":13411,"[/":13412,"Ġtemporarily":13413,"Ver":13414,"hu":13415,"oser":13416,"Ġovernight":13417,"Ġmamm":13418,"ĠTreasury":13419,"ĠVenezuel":13420,"ĠMega":13421,"Ġtar":13422,"Ġexpects":13423,"black":13424,"orph":13425,"\\\\\\\\":13426,"Ġacceptance":13427,"Ġradar":13428,"sis":13429,"Ġjunior":13430,"Ġframes":13431,"Ġobservation":13432,"acies":13433,"Power":13434,"ĠAdvanced":13435,"Mag":13436,"ologically":13437,"ĠMechan":13438,"Ġsentences":13439,"Ġanalysts":13440,"aughters":13441,"forcement":13442,"Ġvague":13443,"Ġclause":13444,"Ġdirectors":13445,"Ġevaluate":13446,"Ġcabinet":13447,"Matt":13448,"ĠClassic":13449,"Ang":13450,"Ġcler":13451,"ĠBuck":13452,"Ġresearcher":13453,"Ġ160":13454,"Ġpoorly":13455,"Ġexperiencing":13456,"ĠPed":13457,"ĠManhattan":13458,"Ġfreed":13459,"Ġthemes":13460,"advant":13461,"Ġnin":13462,"Ġpraise":13463,"104":13464,"ĠLibya":13465,"best":13466,"Ġtrusted":13467,"Ġcease":13468,"Ġdign":13469,"Direct":13470,"Ġbombing":13471,"Ġmigration":13472,"ĠSciences":13473,"Ġmunicipal":13474,"ĠAverage":13475,"Ġglory":13476,"Ġrevealing":13477,"Ġarena":13478,"Ġuncertainty":13479,"Ġbattlefield":13480,"iao":13481,"God":13482,"Ġcinem":13483,"rape":13484,"elle":13485,"apons":13486,"Ġlisting":13487,"Ġwaited":13488,"Ġspotted":13489,"keley":13490,"ĠAudio":13491,"eor":13492,"arding":13493,"idding":13494,"igma":13495,"ĠNeg":13496,"Ġlone":13497,"Ġ----":13498,"exe":13499,"deg":13500,"Ġtransf":13501,"Ġwash":13502,"Ġslavery":13503,"Ġexploring":13504,"ĠWW":13505,"atson":13506,"Ġencl":13507,"lies":13508,"ĠCreek":13509,"Ġwooden":13510,"Manager":13511,"ĠBrand":13512,"ummy":13513,"ĠArthur":13514,"Ġbureaucr":13515,"Ġblend":13516,"arians":13517,"Further":13518,"Ġsupposedly":13519,"Ġwinds":13520,"Ġ1979":13521,"Ġgravity":13522,"Ġanalyses":13523,"ĠTravel":13524,"ĠVeter":13525,"Ġdumb":13526,"Ġalternate":13527,"gal":13528,"Ġconsumed":13529,"Ġeffectiveness":13530,".''":13531,"Ġpaths":13532,"onda":13533,"LA":13534,"ĠStrong":13535,"Ġenables":13536,"Ġescaped":13537,"Ġ\"\"":13538,"Ġ112":13539,"Ġ1983":13540,"Ġsmiled":13541,"Ġtendency":13542,"Fire":13543,"Ġpars":13544,"ĠRoc":13545,"Ġlake":13546,"Ġfitness":13547,"ĠAth":13548,"ĠHorn":13549,"Ġhier":13550,"Ġimpose":13551,"mother":13552,"Ġpension":13553,"icut":13554,"borne":13555,"iciary":13556,"._":13557,"ĠSU":13558,"Ġpolar":13559,"isy":13560,"engu":13561,"itialized":13562,"ATA":13563,"write":13564,"Ġexercises":13565,"ĠDiamond":13566,"otypes":13567,"Ġharmful":13568,"onz":13569,"Ġprinting":13570,"story":13571,"Ġexpertise":13572,"ĠGer":13573,"Ġtragedy":13574,"ĠFly":13575,"Ġdivid":13576,"ampire":13577,"stock":13578,"Mem":13579,"Ġreign":13580,"Ġunve":13581,"Ġamend":13582,"ĠProphet":13583,"Ġmutual":13584,"ĠFac":13585,"Ġreplacing":13586,"Har":13587,"ĠCircuit":13588,"Ġthroat":13589,"ĠShot":13590,"Ġbatteries":13591,"Ġtoll":13592,"Ġaddressing":13593,"ĠMedicaid":13594,"Ġpupp":13595,"ĠNar":13596,"olk":13597,"Ġequity":13598,"MR":13599,"ĠHispan":13600,"ĠLarge":13601,"mid":13602,"Dev":13603,"Ġexped":13604,"Ġdemo":13605,"ĠMarshall":13606,"ergus":13607,"Ġfiber":13608,"Ġdivorce":13609,"ĠCreate":13610,"Ġslower":13611,"ĠParker":13612,"ĠStudent":13613,"ĠTraining":13614,"Return":13615,"ĠTru":13616,"Ġcub":13617,"ĠReached":13618,"Ġpanic":13619,"Ġquarters":13620,"Ġrect":13621,"Ġtreating":13622,"Ġrats":13623,"ĠChristianity":13624,"oler":13625,"Ġsacred":13626,"Ġdeclare":13627,"ulative":13628,"eting":13629,"Ġdelivering":13630,"estone":13631,"Ġtel":13632,"ĠLarry":13633,"Ġmeta":13634,"accept":13635,"artz":13636,"ĠRoger":13637,"handed":13638,"Ġheader":13639,"Ġtrapped":13640,"ĠCentury":13641,"Ġknocked":13642,"ĠOxford":13643,"Ġsurvivors":13644,"bot":13645,"Ġdemonstration":13646,"Ġdirt":13647,"Ġassists":13648,"OME":13649,"ĠDraft":13650,"ortunate":13651,"folio":13652,"pered":13653,"usters":13654,"gt":13655,"ĠLock":13656,"Ġjudicial":13657,"verted":13658,"Ġsecured":13659,"outing":13660,"ĠBooks":13661,"Ġhosting":13662,"Ġlifted":13663,"length":13664,"Ġjer":13665,"Ġwheels":13666,"ĠRange":13667,"umbnails":13668,"Ġdiagnosis":13669,"tech":13670,"ĠStewart":13671,"ĠPract":13672,"Ġnationwide":13673,"Ġdear":13674,"Ġobligations":13675,"Ġgrows":13676,"Ġmandatory":13677,"Ġsuspicious":13678,"!'":13679,"Apr":13680,"Great":13681,"Ġmortgage":13682,"Ġprosecutor":13683,"Ġeditorial":13684,"ĠKr":13685,"Ġprocessed":13686,"ungle":13687,"Ġflexibility":13688,"Earlier":13689,"ĠCart":13690,"ĠSug":13691,"Ġfocuses":13692,"Ġstartup":13693,"Ġbreach":13694,"ĠTob":13695,"cycle":13696,"ãĢĮ":13697,"rose":13698,"Ġbizarre":13699,"ãĢį":13700,"Ġvegetables":13701,"$$":13702,"Ġretreat":13703,"oshi":13704,"ĠShop":13705,"ĠGround":13706,"ĠStop":13707,"ĠHawaii":13708,"ĠAy":13709,"Perhaps":13710,"ĠBeaut":13711,"uffer":13712,"enna":13713,"Ġproductivity":13714,"Fixed":13715,"control":13716,"Ġabsent":13717,"ĠCampaign":13718,"Green":13719,"Ġidentifying":13720,"Ġregret":13721,"Ġpromoted":13722,"ĠSeven":13723,"Ġeru":13724,"neath":13725,"aughed":13726,"ĠPin":13727,"ĠLiving":13728,"Cost":13729,"omatic":13730,"mega":13731,"ĠNig":13732,"ocy":13733,"Ġinbox":13734,"Ġempire":13735,"Ġhorizont":13736,"Ġbranches":13737,"Ġmetaph":13738,"Active":13739,"edi":13740,"ĠFilm":13741,"ĠSomething":13742,"Ġmods":13743,"incial":13744,"ĠOriginal":13745,"Gen":13746,"Ġspirits":13747,"Ġearning":13748,"Hist":13749,"Ġriders":13750,"Ġsacrific":13751,"MT":13752,"ĠVA":13753,"ĠSalt":13754,"Ġoccupation":13755,"ĠMi":13756,"Ġdisg":13757,"lict":13758,"Ġnit":13759,"Ġnodes":13760,"eem":13761,"ĠPier":13762,"Ġhatred":13763,"psy":13764,"ãĥī":13765,"Ġtheater":13766,"Ġsophisticated":13767,"Ġdefended":13768,"Ġbesides":13769,"Ġthoroughly":13770,"ĠMedicare":13771,"Ġblamed":13772,"arently":13773,"Ġcrying":13774,"FOR":13775,"priv":13776,"Ġsinging":13777,"ĠIl":13778,"Ġcute":13779,"oided":13780,"olitical":13781,"ĠNeuro":13782,"å¤":13783,"Ġdonation":13784,"ĠEagles":13785,"ĠGive":13786,"Tom":13787,"Ġsubstantially":13788,"ĠLicense":13789,"ĠJa":13790,"Ġgrey":13791,"ĠAnimal":13792,"ĠER":13793,"ĠUnd":13794,"Ġkeen":13795,"Ġconclude":13796,"ĠMississippi":13797,"Engine":13798,"ĠStudios":13799,"Press":13800,"overs":13801,"llers":13802,"Ġ350":13803,"ĠRangers":13804,"Ġrou":13805,"erto":13806,"Ep":13807,"issa":13808,"ivan":13809,"Ġseal":13810,"ĠRegist":13811,"display":13812,"Ġweaken":13813,"uum":13814,"ĠCommons":13815,"ĠSay":13816,"Ġcultures":13817,"Ġlaughed":13818,"Ġslip":13819,"Ġtreatments":13820,"izable":13821,"mart":13822,"ĠRice":13823,"Ġbeast":13824,"Ġobesity":13825,"ĠLaure":13826,"iga":13827,"Which":13828,"holder":13829,"Ġelderly":13830,"Ġpays":13831,"Ġcomplained":13832,"Ġcrop":13833,"Ġproc":13834,"Ġexplosive":13835,"ĠFan":13836,"ĠArsenal":13837,"Author":13838,"eful":13839,"Ġmeals":13840,"Ġ(-":13841,"idays":13842,"Ġimagination":13843,"Ġannually":13844,"Ġms":13845,"asures":13846,"Head":13847,"ikh":13848,"matic":13849,"Ġboyfriend":13850,"ĠComputer":13851,"Ġbump":13852,"Ġsurge":13853,"ĠCraig":13854,"ĠKirk":13855,"Del":13856,"mediate":13857,"Ġscenarios":13858,"ĠMut":13859,"ĠStream":13860,"Ġcompetitors":13861,"ÙĦ":13862,"ĠStanford":13863,"ĠResources":13864,"azed":13865,"bage":13866,"Ġorganis":13867,"ĠRelease":13868,"Ġseparately":13869,"Ġhabits":13870,"Ġmeasurements":13871,"ĠClose":13872,"Ġaccompany":13873,"Ġgly":13874,"Ġtang":13875,"ĠRou":13876,"Ġplugin":13877,"Ġconvey":13878,"ĠChallenge":13879,"oots":13880,"jan":13881,"Ġcurs":13882,"ĠRelations":13883,"keeper":13884,"Ġapproaching":13885,"ping":13886,"Speaking":13887,"Ġarrangement":13888,"ĠVI":13889,"arettes":13890,"Ġaffecting":13891,"Ġpermits":13892,"because":13893,"Ġuseless":13894,"ĠHus":13895,"!!!!":13896,"Ġdestroying":13897,"Unfortunately":13898,"Ġfascinating":13899,"Sem":13900,"Ġelectoral":13901,"Ġtransparency":13902,"ĠChaos":13903,"Ġvolunteer":13904,"Ġstatistical":13905,"Ġactivated":13906,"rox":13907,"Web":13908,"HE":13909,"ĠHampshire":13910,"isive":13911,"Map":13912,"Ġtrash":13913,"ĠLawrence":13914,"stick":13915,"Cr":13916,"Ġrings":13917,"EXT":13918,"Ġoperational":13919,"opes":13920,"Does":13921,"ĠEvans":13922,"Ġwitnessed":13923,"Port":13924,"Ġlaunching":13925,"econom":13926,"wear":13927,"ĠParticip":13928,"umm":13929,"cules":13930,"ĠRAM":13931,"ĠTun":13932,"Ġassured":13933,"Ġbinary":13934,"Ġbetray":13935,"Ġexploration":13936,"ĠFel":13937,"Ġadmission":13938,"itated":13939,"Sy":13940,"Ġavoided":13941,"ĠSimulator":13942,"Ġcelebrated":13943,"ĠElectric":13944,"¥ŀ":13945,"Ġcluster":13946,"itzerland":13947,"health":13948,"Line":13949,"ĠNash":13950,"aton":13951,"Ġspare":13952,"Ġenterprise":13953,"ĠDIS":13954,"cludes":13955,"Ġflights":13956,"Ġregards":13957,"ĠÃĹ":13958,"half":13959,"Ġtrucks":13960,"Ġcontacts":13961,"Ġuncons":13962,"ĠClimate":13963,"Ġimmense":13964,"NEW":13965,"occ":13966,"ective":13967,"Ġembod":13968,"Ġpatrol":13969,"Ġbeside":13970,"Ġviable":13971,"Ġcreep":13972,"Ġtriggered":13973,"verning":13974,"Ġcomparable":13975,"ql":13976,"Ġgaining":13977,"asses":13978,"Ġ();":13979,"ĠGrey":13980,"ĠMLS":13981,"sized":13982,"Ġprosper":13983,"\"?":13984,"Ġpolling":13985,"Ġshar":13986,"ĠRC":13987,"Ġfirearm":13988,"orient":13989,"Ġfence":13990,"Ġvariations":13991,"giving":13992,"ĠPi":13993,"ospel":13994,"Ġpledge":13995,"Ġcure":13996,"Ġspy":13997,"Ġviolated":13998,"Ġrushed":13999,"Ġstroke":14000,"ĠBlog":14001,"sels":14002,"ĠEc":14003,",''":14004,"Ġpale":14005,"ĠCollins":14006,"terror":14007,"ĠCanadians":14008,"Ġtune":14009,"Ġlaboratory":14010,"Ġnons":14011,"tarian":14012,"Ġdisability":14013,"ĠGam":14014,"Ġsinger":14015,"alg":14016,"ĠSenior":14017,"Ġtraded":14018,"ĠWarrior":14019,"Ġinfring":14020,"ĠFranklin":14021,"Ġstrain":14022,"ĠSwedish":14023,"Ġseventh":14024,"ĠBenn":14025,"ĠTell":14026,"Ġsyndrome":14027,"Ġwondered":14028,"iden":14029,"++++":14030,"igo":14031,"Ġpurple":14032,"Ġjournalism":14033,"Ġrebel":14034,"Ġfu":14035,"blog":14036,"Ġinvite":14037,"rencies":14038,"ĠContact":14039,"Israel":14040,"ĠContent":14041,"Ġcheer":14042,"Ġbedroom":14043,"ĠEngineering":14044,"ĠQueens":14045,"Ġdwell":14046,"ĠPlayStation":14047,"ĠDim":14048,"ĠColon":14049,"lr":14050,"Ġoperates":14051,"Ġmotivation":14052,"USA":14053,"astered":14054,"Core":14055,"ĠTruth":14056,"olo":14057,"OSE":14058,"ĠMemory":14059,"Ġpredec":14060,"Ġanarch":14061,"Ġ1920":14062,"ĠYam":14063,"è":14064,"bid":14065,"Ġgrateful":14066,"Ġexcitement":14067,"Ġtreasure":14068,"Ġlongest":14069,"ctive":14070,"Ġdeserves":14071,"Ġreserves":14072,"Ġcops":14073,"ĠOttawa":14074,"ĠEgyptian":14075,"anked":14076,"Ġartif":14077,"Ġhypothesis":14078,":/":14079,"Ġpurchasing":14080,"Ġlovely":14081,"HP":14082,"Ġdivide":14083,"Ġstrictly":14084,"Ġquestioning":14085,"Ġtaxpayers":14086,"ĠJoy":14087,"Ġrolls":14088,"ĠHeavy":14089,"Ġports":14090,"Ġmagnetic":14091,"Ġinflamm":14092,"Ġbrush":14093,"tics":14094,"âĪĴ":14095,"Ġbottles":14096,"ppy":14097,"Ġpadd":14098,"ãĤ¯":14099,"million":14100,"Ġdevastating":14101,"Ġcompiled":14102,"Ġmedication":14103,"Ġtwelve":14104,"ĠPerry":14105,"Space":14106,"imb":14107,"your":14108,"Ġleaked":14109,"ĠTar":14110,"Ġunity":14111,"Ġinfected":14112,"Ġtraveled":14113,"IDE":14114,"ĠMcDonald":14115,"txt":14116,"ĠPrinc":14117,"Ġinterven":14118,"ĠTaiwan":14119,"ĠPow":14120,"Ġbearing":14121,"ĠThread":14122,"Ġzones":14123,"izards":14124,"unks":14125,"Chapter":14126,"llor":14127,"Ġ·":14128,"Ġwounds":14129,"Ġdiscretion":14130,"Ġsucceeded":14131,"iking":14132,"Ġiconic":14133,"Call":14134,"Ġscreening":14135,"ĠMis":14136,"icts":14137,"Ġministers":14138,"Ġseparation":14139,"Player":14140,"Ġbip":14141,"Ġbeloved":14142,"Ġcounting":14143,"ĠEye":14144,"around":14145,"inging":14146,"Ġtablet":14147,"Ġoffence":14148,"inance":14149,"have":14150,"ĠInfo":14151,"ĠNinja":14152,"Ġprotective":14153,"ĠCass":14154,"Mac":14155,"ĠQuality":14156,"North":14157,"Ġic":14158,"ĠCuba":14159,"ĠChronicle":14160,"ĠProperty":14161,"Ġfastest":14162,"otos":14163,"ĠGerm":14164,"OWN":14165,"Ġboom":14166,"ĠStanley":14167,"erguson":14168,"Ġclever":14169,"Ġenters":14170,"mode":14171,"terior":14172,"ĠSens":14173,"Ġlinear":14174,"ARK":14175,"Ġcomparing":14176,"Ġpurely":14177,"Ġsafer":14178,"ĠPotter":14179,"Ġcups":14180,"RT":14181,"Ġgluc":14182,"Ġattributed":14183,"Ġdupl":14184,"ĠPap":14185,"Ġprecious":14186,"Ġpa":14187,"ictionary":14188,"ĠTig":14189,"ĠToo":14190,"olutions":14191,"stan":14192,"Ġrobots":14193,"Ġlobb":14194,"Ġstatute":14195,"Ġprevention":14196,"western":14197,"160":14198,"ĠActive":14199,"ĠMaria":14200,"hal":14201,"None":14202,"ellar":14203,"ĠKB":14204,"ĠPartners":14205,"ĠSingle":14206,"ĠFollowing":14207,"ango":14208,"acious":14209,"Ġthou":14210,"Ġkg":14211,"Ġinfluential":14212,"ĠFriends":14213,"Sur":14214,"ainted":14215,"Ġforums":14216,"Ġstarter":14217,"Ġcitizenship":14218,"ĠElection":14219,"onge":14220,"otation":14221,"osph":14222,";;;;":14223,"utical":14224,"pur":14225,"eren":14226,"Ġaccusations":14227,"bitious":14228,"abbit":14229,"ĠOrd":14230,"Posted":14231,"irk":14232,"Ġsensitivity":14233,"iche":14234,"ĠAmy":14235,"ĠFab":14236,"Ġsummit":14237,"Ġpedest":14238,"Ġrubber":14239,"Ġagricultural":14240,"Ġcancel":14241,"AE":14242,"Ġinaug":14243,"Ġcontam":14244,"Ġfirmly":14245,"iw":14246,"stage":14247,"ĠKan":14248,"Ġtier":14249,"Ġinvention":14250,"Ġtranslated":14251,"ĠRules":14252,"Box":14253,"Twitter":14254,"IDS":14255,"Ġpizza":14256,"Ġdebug":14257,"ĠDrop":14258,"vs":14259,"Ġhorses":14260,"big":14261,"Ġboring":14262,"Ġhood":14263,"ĠMcCain":14264,"atched":14265,"ĠBros":14266,"Ġskip":14267,"Ġessay":14268,"stat":14269,"ĠLegends":14270,"Ġammunition":14271,"auc":14272,"Ġshooter":14273,"Ġunh":14274,"Ġsupplied":14275,"Ġgeneric":14276,"ĠSK":14277,"iban":14278,"yrics":14279,"Ġ255":14280,"Ġclimbing":14281,"Former":14282,"Ġflip":14283,"Ġjumping":14284,"Ġfrustration":14285,"ĠTerry":14286,"Ġneighborhoods":14287,"Ġmedian":14288,"bean":14289,"Ġbrains":14290,"Following":14291,"Ġshaped":14292,"Ġdraws":14293,"Ġaltered":14294,"Jack":14295,"Ġrecipes":14296,"Ġskilled":14297,"wealth":14298,"achi":14299,"election":14300,"Ġbehaviors":14301,"deals":14302,"ĠUntil":14303,"Fe":14304,"Ġdeclaration":14305,"marks":14306,"ĠBetween":14307,"celona":14308,"Ġreson":14309,"Ġbubble":14310,"Among":14311,"Ġimperial":14312,"GS":14313,"Ġfeminist":14314,"2005":14315,"ĠKyle":14316,"Ġaccounting":14317,"ĠTele":14318,"ĠTyr":14319,"Ġconnecting":14320,"Ġrehab":14321,"ĠPred":14322,"sim":14323,"Ġmeantime":14324,"Ġphysician":14325,"MW":14326,"ĠCampbell":14327,"ĠBrandon":14328,"Ġcontributing":14329,"ĠRule":14330,"ĠWeight":14331,"ĠNap":14332,"Ġinteractive":14333,"Ġvag":14334,"Ġhelmet":14335,"ĠComb":14336,"four":14337,"Ġshipped":14338,"Ġcompleting":14339,"ĠPD":14340,"PDATE":14341,"Ġspreading":14342,"Ġscary":14343,"erving":14344,"ĠGas":14345,"Ġfrank":14346,"school":14347,"Ġromantic":14348,"Ġstabil":14349,"Rob":14350,"Ġaccurately":14351,"Ġacute":14352,"ĠHann":14353,"Ġsymbols":14354,"Ġcivilization":14355,"ĠAW":14356,"Ġlightning":14357,"Ġconsiders":14358,"Ġvenue":14359,"Ġ×":14360,"Ġoven":14361,"ĠSF":14362,"his":14363,"Ġnu":14364,"ĠLearn":14365,"Ġpeoples":14366,"Ġstd":14367,"Ġslee":14368,"Ġslic":14369,"ĠStatistics":14370,"Ġcorners":14371,"ĠBaker":14372,"Ġ:)":14373,"mentation":14374,"olver":14375,"Ġlaughing":14376,"ĠTodd":14377,"onde":14378,"ĠHills":14379,"Ġnuts":14380,"ĠWoman":14381,"plane":14382,"Ġliver":14383,"ĠInside":14384,"Sorry":14385,"Ġagrees":14386,"Ġfundament":14387,"ĠFisher":14388,"Ġauction":14389,"Ġthreads":14390,"glas":14391,"ĠBasic":14392,"ĠNat":14393,"Ġlacking":14394,"Ġcelebration":14395,"ju":14396,"Ġsilly":14397,"Euro":14398,"Ġtatt":14399,"ighty":14400,"controlled":14401,"Test":14402,"ĠSingh":14403,"Ġrage":14404,"Ġrhyth":14405,"offic":14406,"ĠPhantom":14407,"Ġheadlines":14408,"Ġresponding":14409,"ĠMorning":14410,"Ġvitamin":14411,"Ġboots":14412,"ĠSite":14413,"alin":14414,"pi":14415,"Ġviral":14416,"ĠUC":14417,"DER":14418,"ĠSex":14419,"Ġstocks":14420,"current":14421,"Ġchurches":14422,"ĠRare":14423,"ĠMurphy":14424,"Ġdenial":14425,"ĠGaming":14426,"Ġtoug":14427,"Ġnick":14428,"Ġmakers":14429,"ĠRonald":14430,"Ġgenerous":14431,"ĠDoc":14432,"ĠMorris":14433,"Ġtransformed":14434,"ĠNormal":14435,"Ġ104":14436,"ĠKickstarter":14437,"ĠUpon":14438,"Online":14439,"ĠIRS":14440,"Ġwrap":14441,"Ġloving":14442,"Ġarrives":14443,"ĠDue":14444,"Ġheter":14445,"ĠMade":14446,"Ġrental":14447,"Ġbelongs":14448,"Ġattorneys":14449,"Ġcrops":14450,"Ġmatched":14451,"ulum":14452,"oline":14453,"109":14454,"Ġdispar":14455,"Ġbuyers":14456,"ĠCambridge":14457,"Ġethics":14458,"roups":14459,"Ġjustified":14460,"Ġmarginal":14461,"Ġrespected":14462,"winning":14463,"Ġnodded":14464,"ĠSerge":14465,"ĠFormer":14466,"Craft":14467,"################":14468,"ĠWarner":14469,"Ġdash":14470,"ete":14471,"Ġentert":14472,"ĠEscape":14473,"outheast":14474,"Ġknees":14475,"ĠBomb":14476,"Ġrug":14477,"Pass":14478,"Ġattitudes":14479,"government":14480,"ĠPrior":14481,"Ġqualities":14482,"Ġnotification":14483,"ĠPhone":14484,"lie":14485,"Ġanticipated":14486,"ĠCombat":14487,"ĠBarry":14488,"Ġ1982":14489,"Users":14490,"oner":14491,"Ġcomputing":14492,"ĠConnecticut":14493,"Ġlesser":14494,"Ġpeers":14495,"ĠCu":14496,"Ġtechnically":14497,"Ġsubmission":14498,"ĠUniversal":14499,"Ġmanually":14500,"ourge":14501,"Ġrespondents":14502,"ĠBTC":14503,"ĠHost":14504,"Ġfare":14505,"ĠBird":14506,"Ġreceipt":14507,"also":14508,"Ġjack":14509,"Ġagriculture":14510,"Ġskull":14511,"Ġ!=":14512,"Ġpassive":14513,"ĠCI":14514,"Ġsocieties":14515,"Ġreminded":14516,"Ġinterference":14517,"Buy":14518,"Ġâľ":14519,"gon":14520,"Ġscrutiny":14521,"ĠWitch":14522,"Ġconducting":14523,"Ġãĥ":14524,"Ġexchanges":14525,"ĠMitchell":14526,"Ġinhabit":14527,"Ġtwist":14528,"BD":14529,"Ġwherever":14530,"groupon":14531,"Ġjokes":14532,"ĠBenjamin":14533,"ĠRandom":14534,"frame":14535,"ĠLions":14536,"Ġhighlighted":14537,"ĠArkansas":14538,"Ent":14539,"Ġpile":14540,"Ġprelim":14541,"gs":14542,"minded":14543,"Ġfelony":14544,"ĠGA":14545,"ĠLuck":14546,"Ġpractically":14547,"ĠBos":14548,"Ġactress":14549,"Dam":14550,"ĠBou":14551,"Ġvisa":14552,"Ġembedded":14553,"Ġhybrid":14554,"Ġearliest":14555,"Ġsooner":14556,"social":14557,"ĠHA":14558,"Ġsteep":14559,"Ġdisadvant":14560,"Ġexploit":14561,"ĠEgg":14562,"ĠUltra":14563,"Ġnecessity":14564,"Local":14565,"iege":14566,"Ġdated":14567,"Ġmasses":14568,"Ġsubscription":14569,"pless":14570,"Ġanonym":14571,"Ġpresumably":14572,"Blue":14573,"Their":14574,"asketball":14575,"ĠPhilip":14576,"Ġcomed":14577,"loaded":14578,"rane":14579,"Ġreflection":14580,"China":14581,"Ġextends":14582,"Ġforming":14583,"Ġunders":14584,"2001":14585,"Ġgrat":14586,"Ġconcentrations":14587,"Ġinsulin":14588,"Ġsecular":14589,"Ġwhilst":14590,"Ġwinners":14591,"Advertisements":14592,"Ġdeliberately":14593,"ĠWorking":14594,"Ġsink":14595,"etics":14596,"dale":14597,"Ġmandate":14598,"Ġgram":14599,"Ġvacation":14600,"Ġwarnings":14601,"ripp":14602,"ĠTHAT":14603,"Ġcommentary":14604,"Ġintu":14605,"Ġaest":14606,"Ġreasoning":14607,"Ġbreakdown":14608,"ĠZombie":14609,"Ġ-->":14610,"ĠPolitical":14611,"cott":14612,"Ġthrust":14613,"Ġtechnological":14614,"Ġdeciding":14615,"Ġtrafficking":14616,"Long":14617,"Welcome":14618,"prising":14619,"ĠCommunications":14620,"Ġendors":14621,"Ġswift":14622,"Ġmetabol":14623,"coins":14624,"resa":14625,"ĠHTTP":14626,"Ġenroll":14627,"ĠHappy":14628,"usr":14629,"intage":14630,"Ġ[\"":14631,"uably":14632,"ĠMaterial":14633,"Ġrepeal":14634,"Sept":14635,"kh":14636,"ĠModi":14637,"Ġunderneath":14638,"ĠIL":14639,"shore":14640,"Ġdiagnosed":14641,"aceutical":14642,"Ġshower":14643,"aux":14644,"ĠSwitch":14645,"ĠStrength":14646,"Ġjihad":14647,"national":14648,"Ġtrauma":14649,"ussy":14650,"oni":14651,"Ġconsolid":14652,"Ġcalories":14653,"ĠFlynn":14654,"agged":14655,"168":14656,"ĠPink":14657,"Ġfulfill":14658,"Ġchains":14659,"Ġnotably":14660,"ĠAV":14661,"Life":14662,"ĠChuck":14663,"mus":14664,"ĠUrban":14665,"ĠHend":14666,"Ġdeposit":14667,"ĠSad":14668,"Ġaffair":14669,"ORK":14670,"ieval":14671,"ĠFDA":14672,"Ġtrop":14673,"ĠOverall":14674,"Ġvirtue":14675,"Ġsatisfaction":14676,"aund":14677,"Ġlun":14678,"ĠSwitzerland":14679,"ĠOperation":14680,"process":14681,"Ġshook":14682,"Ġcounties":14683,"leased":14684,"ĠCharlotte":14685,"112":14686,"Ġtranscript":14687,"Ġredd":14688,"push":14689,"ĠHey":14690,"ĠAnalysis":14691,"[\"":14692,"Ġalternatives":14693,"ardless":14694,"Ġeleph":14695,"Ġprejud":14696,"ĠLeaf":14697,"Having":14698,"ĠHub":14699,"Ġexpressions":14700,"ĠVolume":14701,"Ġshocking":14702,"ĠReds":14703,"Ġreadily":14704,"Ġplanets":14705,"adata":14706,"Ġcollapsed":14707,"ĠMadrid":14708,"Ġirrit":14709,"ipper":14710,"ĠEnc":14711,"ĠWire":14712,"Ġbuzz":14713,"ĠGP":14714,"asha":14715,"Ġaccidentally":14716,"uru":14717,"Ġfrustrated":14718,"ĠSA":14719,"Ġhungry":14720,"ĠHuff":14721,"Ġlabels":14722,"anto":14723,"ĠEP":14724,"Ġbarriers":14725,")|":14726,"ĠBerkeley":14727,"ĠJets":14728,"Ġpairs":14729,"ĠLan":14730,"James":14731,"ĠBear":14732,"Ġhumor":14733,"ĠLiberty":14734,"Ġmagnitude":14735,"Ġaging":14736,"ĠMason":14737,"Ġfriendship":14738,"umbling":14739,"Ġemerge":14740,"Ġnewspapers":14741,"Ġambitious":14742,"ĠRichards":14743,"aternal":14744,"Ġ1981":14745,"Ġcookies":14746,"Ġsculpt":14747,"Ġpursuit":14748,"Location":14749,"Ġscripts":14750,"pc":14751,"Ġarrangements":14752,"Ġdiameter":14753,"Ġloses":14754,"amation":14755,"Ġliqu":14756,"ĠJake":14757,"arette":14758,"Ġunderstands":14759,"ĠZen":14760,"vm":14761,"Ġapprove":14762,"Ġwip":14763,"Ġultra":14764,"Ġintend":14765,"ĠDI":14766,"ascular":14767,"Ġstays":14768,"ĠKor":14769,"ĠKl":14770,"Ġinvesting":14771,"La":14772,"Ġbelieving":14773,"bad":14774,"mouth":14775,"Ġtaxpayer":14776,"ãĥĥ":14777,"ĠQuebec":14778,"Ġlap":14779,"ĠSwiss":14780,"drop":14781,"Ġdrain":14782,"iri":14783,"etc":14784,"ften":14785,"ĠNex":14786,"Ġstraw":14787,"Ġscreaming":14788,"Ġcounted":14789,"Ġdamaging":14790,"Ġambassador":14791,"century":14792,"Ġprox":14793,"Ġarrests":14794,"uv":14795,"ilateral":14796,"ĠCharg":14797,"Ġprescribed":14798,"Ġindependently":14799,"Ġfierce":14800,"ĠBaby":14801,"Ġbrave":14802,"Ġsuits":14803,"=>":14804,"Ġbaseline":14805,"ĠRate":14806,"Ġislands":14807,"Ġ((":14808,"green":14809,"ixels":14810,"Ġnamely":14811,"ĠVillage":14812,"than":14813,"amy":14814,"Version":14815,"gmail":14816,"entials":14817,"ĠSud":14818,"ĠMelbourne":14819,"Ġarriving":14820,"Ġquantum":14821,"eff":14822,"ropolitan":14823,"Tri":14824,"Ġfuneral":14825,"ĠIR":14826,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":14827,"ĠCob":14828,"itably":14829,"Ġturb":14830,"Ġcombo":14831,"Review":14832,"Ġdeployment":14833,"uity":14834,"ĠBott":14835,"Ġinvisible":14836,"Ġrendering":14837,"Ġunlocked":14838,"Ġaqu":14839,"ĠVladimir":14840,"Ġpad":14841,"ĠBrain":14842,"ĠLegacy":14843,"dragon":14844,"ĠKurdish":14845,"Ġsounded":14846,"Ġdetained":14847,"ĠDM":14848,"gary":14849,"Ġdaughters":14850,"Ġdisturbing":14851,"uka":14852,"ĠParad":14853,"Ġtast":14854,"Ġunfortunate":14855,"Ġul":14856,"emin":14857,"Ġattendance":14858,"trl":14859,"Ġparks":14860,"ĠMemorial":14861,"ĠAlice":14862,"othy":14863,"guard":14864,"ĠDise":14865,"ĠShan":14866,"ĠForum":14867,"Rich":14868,"Ġshifted":14869,"uez":14870,"Ġlighter":14871,"ĠMagn":14872,"Ġcod":14873,"Sch":14874,"hammad":14875,"Pub":14876,"350":14877,"ĠPokemon":14878,"Ġprototype":14879,"Ġunre":14880,"Base":14881,"ĠStudents":14882,"ĠReply":14883,"ĠCommunist":14884,"Ġgau":14885,"ĠTyler":14886,"IZ":14887,"Ġparticipated":14888,"Ġsuprem":14889,"ĠDetails":14890,"Ġvessels":14891,"rod":14892,"Ġtribe":14893,"keep":14894,"Ġassumptions":14895,"Ġpound":14896,"Ġcrude":14897,"ĠAvailable":14898,"Ġswimming":14899,"Ġinclusion":14900,"Ġadvances":14901,"culation":14902,"Ġconservation":14903,"Ġoverd":14904,"ĠBuffalo":14905,"Article":14906,"edge":14907,"Ġawa":14908,"ĠMadison":14909,"Ġsidew":14910,"Ġcatast":14911,"ĠKrist":14912,"ucle":14913,"ĠHighway":14914,"ĠTerror":14915,"Ġactivation":14916,"Ġunconscious":14917,"ĠSatan":14918,"ĠSusan":14919,"illery":14920,"Ġarranged":14921,"iop":14922,"Ġrumors":14923,"urring":14924,"think":14925,"ĠKeith":14926,"ĠKind":14927,"Ġavoiding":14928,"byn":14929,"nut":14930,"ĠSpeaker":14931,"rus":14932,"names":14933,"Ġguilt":14934,"ĠOlympics":14935,"Ġsail":14936,"ĠMes":14937,"levant":14938,"ĠColumbus":14939,"aft":14940,"City":14941,"South":14942,"ĠHarvey":14943,"ĠPun":14944,"Several":14945,"Ġmentally":14946,"Ġimpress":14947,"mount":14948,"ĠUbuntu":14949,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":14950,"ĠSuperman":14951,"ĠMPs":14952,"Ġintentions":14953,"ĠRacing":14954,"Ġlikelihood":14955,"Ġ240":14956,"Total":14957,"Ġtoys":14958,"ĠWatson":14959,"Ġurge":14960,"Lear":14961,"ĠPaper":14962,"Ġoccurring":14963,"ĠBeng":14964,"ĠCert":14965,"Ġstones":14966,"Tim":14967,"ĠTwin":14968,"zb":14969,"ĠDynam":14970,"Ġpolitician":14971,"kens":14972,"ĠEnterprise":14973,"UTERS":14974,"Ġabol":14975,"Ġrefresh":14976,"Ġarbitrary":14977,"pection":14978,"Ġtroubles":14979,"Ġ});":14980,"tv":14981,"Ġpilots":14982,"Ġdistribute":14983,"Ġaudit":14984,"Ġpause":14985,"original":14986,"Ġrivals":14987,"£":14988,"Fig":14989,"TL":14990,"abil":14991,"rying":14992,"Lin":14993,"ioned":14994,"lon":14995,"Ġfancy":14996,"Ġcrashed":14997,"Ġtract":14998,"Ġshed":14999,"Ġconsume":15000,"Based":15001,"download":15002,"init":15003,"Ġvoltage":15004,"Introdu":15005,"Ġcondemned":15006,"ĠFinance":15007,"respect":15008,"Ġexcluded":15009,"Ġestablishing":15010,"heric":15011,"Ġheritage":15012,"Ġspectacular":15013,"Ġunst":15014,"ĠSnowden":15015,"ĠLane":15016,"San":15017,"Ġprotections":15018,"struction":15019,"incinn":15020,"Ġmacro":15021,"Custom":15022,"iosity":15023,"Ġesp":15024,"Ġfunctioning":15025,"Ġmush":15026,"Ġpuzzle":15027,"Ġethical":15028,"Mal":15029,"Ġgoverning":15030,"ĠFerguson":15031,"Ġrestored":15032,"Ġstressed":15033,"ĠCounter":15034,"ĠKas":15035,"clip":15036,"ANS":15037,"Ġseiz":15038,"UK":15039,"byss":15040,"oldown":15041,"api":15042,"Ġpermanently":15043,"ounters":15044,"West":15045,"Through":15046,"Light":15047,"atoes":15048,"Ġneat":15049,"Ġcord":15050,"urer":15051,"Ġseverely":15052,"ĠAven":15053,"Ġinterrog":15054,"Ġtriple":15055,"Given":15056,"Number":15057,"Ġarise":15058,"Ġsher":15059,"plant":15060,"Ġflower":15061,"ĠCou":15062,"Ġate":15063,"Ġnewer":15064,"bul":15065,"Ġmeanwhile":15066,"ĠLair":15067,"Ġadjustment":15068,"ĠCopyright":15069,"Ġdivers":15070,"iological":15071,"Ġgamers":15072,"oat":15073,"Ġhistorically":15074,"Ġanalog":15075,"Ġlongtime":15076,"Ġprescription":15077,"ĠMist":15078,"ĠHyper":15079,"ĠMaine":15080,"ĠDeity":15081,"Ġmultipl":15082,"ĠReincarn":15083,"ĠHyd":15084,"ĠPic":15085,"Sil":15086,"rants":15087,"ĠCris":15088,".;":15089,"({":15090,"ependence":15091,"Ġrecy":15092,"ateur":15093,"Ġquad":15094,"Ġglob":15095,"Ġconced":15096,"team":15097,"Ġcapitalist":15098,"ĠLot":15099,"Ġroyal":15100,"ĠCyber":15101,"Ġblacks":15102,"metic":15103,"riv":15104,"ĠDanny":15105,"Ġspo":15106,"ĠRO":15107,"Ġanimated":15108,"rypted":15109,"ĠDeputy":15110,"Ġrendered":15111,"FE":15112,"Ġstreak":15113,"Ġclouds":15114,"ĠDoug":15115,"~~~~~~~~":15116,"Ġdiscour":15117,"ĠVeh":15118,"Ġpsychology":15119,"ĠJourney":15120,"Ġcrystal":15121,"ĠFrost":15122,"Ġsuspicion":15123,"Ġrelate":15124,"orus":15125,"ĠCrypt":15126,"ĠNVIDIA":15127,"comed":15128,"uting":15129,"incinnati":15130,"Ġvulnerability":15131,"ostic":15132,"Ġisolation":15133,"Ġcooling":15134,"ĠCoalition":15135,"Ġ119":15136,"Four":15137,"ĠDeal":15138,"Ġâī":15139,"semble":15140,"rament":15141,"ĠBarcelona":15142,"Ġ102":15143,"Ġcocaine":15144,"ocalypse":15145,"Feb":15146,"ogenic":15147,"Ġmutation":15148,"Ġcryptoc":15149,"ĠKel":15150,"ĠGit":15151,"ais":15152,"Ġsisters":15153,"ANK":15154,"Ġactivate":15155,"Ter":15156,"Ġdread":15157,"ylon":15158,"Ġpropri":15159,"Aust":15160,"ĠDefault":15161,"Ġoutdoor":15162,"Ġsheer":15163,"ceive":15164,"Ġgently":15165,"о":15166,"Program":15167,"ĠâĨĴ":15168,"Ġvegan":15169,"ĠCrus":15170,"Ġresponsibilities":15171,"ĠHR":15172,"OLD":15173,"Ġprevents":15174,"Ġstiff":15175,"ĠWere":15176,"Ġathletic":15177,"ĠScore":15178,"Ġ):":15179,"Ġcolumns":15180,"ĠLoc":15181,"available":15182,"ĠFram":15183,"ĠSessions":15184,"Ġcompanion":15185,"Ġpacks":15186,"140":15187,"ĠKnights":15188,"Ġfart":15189,"Ġstreams":15190,"Ġshore":15191,"Ġappeals":15192,"ĠPerformance":15193,"haul":15194,"ĠStra":15195,"ĠNag":15196,"103":15197,"ĠTransportation":15198,"BB":15199,"Ev":15200,"zan":15201,"Public":15202,"Ġtwin":15203,"ulsion":15204,"Mult":15205,"Ġelectro":15206,"Ġstatue":15207,"ationally":15208,"ĠNort":15209,"Ġinspection":15210,"/*":15211,"igue":15212,"Ġcompassion":15213,"ĠTales":15214,"ĠStein":15215,"ĠScreen":15216,"ĠBug":15217,"ĠLion":15218,"girl":15219,"Ġwithdrawal":15220,"Ġobjectives":15221,"Ġbloody":15222,"Ġpreliminary":15223,"Ġjacket":15224,"Ġdimensions":15225,"ĠCool":15226,"ĠOccup":15227,"Ġwreck":15228,"Ġdoubled":15229,"anking":15230,"Ġ1975":15231,"Ġglasses":15232,"ĠWang":15233,"prov":15234,"Path":15235,"connected":15236,"ĠMulti":15237,"ĠNorway":15238,"agonist":15239,"Ġfeared":15240,"Ġtouching":15241,"Ġarguably":15242,"¯¯¯¯¯¯¯¯":15243,"ĠNCAA":15244,"chem":15245,"Ġspat":15246,"ĠWWE":15247,"ĠCel":15248,"igger":15249,"Ġattacker":15250,"ĠJoin":15251,"object":15252,"etta":15253,"Ġeliminated":15254,"det":15255,"Ġdestruct":15256,"ĠLucas":15257,"ctuary":15258,"180":15259,"ĠBrady":15260,"ĠBlues":15261,"Bay":15262,"aukee":15263,"Ġtimeline":15264,"Ġdelegates":15265,"written":15266,"ufficient":15267,"Ġshapes":15268,"Copyright":15269,"ouble":15270,"service":15271,"Ġpione":15272,"Ġcolleges":15273,"Ġrows":15274,"Ġspite":15275,"Ġassessed":15276,"360":15277,"Ġlease":15278,"Ġconfidential":15279,"cker":15280,"ĠManning":15281,"ĠVoice":15282,"Ġsealed":15283,"Ġcalculate":15284,"NO":15285,"ĠAssistant":15286,"Ġteenager":15287,"ulent":15288,"atherine":15289,"Ġmock":15290,"Ġdiamond":15291,"Ġfest":15292,"Ġswitched":15293,"Ġresume":15294,"ĠPuerto":15295,"Ġlanes":15296,"iration":15297,"ĠSimilarly":15298,"Ġrod":15299,"ĠSel":15300,"ĠPalace":15301,"ĠLimited":15302,"eous":15303,"Ġvariant":15304,"Ġward":15305,"Ġ))":15306,"Show":15307,"OOK":15308,"Alex":15309,"ĠNep":15310,"bris":15311,"ĠWikipedia":15312,"Ġexceptional":15313,"Ġmanages":15314,"ĠDraw":15315,"Again":15316,"Ġcopper":15317,"utt":15318,"Ġexports":15319,"Ġportfolio":15320,"Ġelevated":15321,"Rated":15322,"ĠOtherwise":15323,"ĠTact":15324,"ĠShel":15325,"ĠTX":15326,"\"âĢĶ":15327,"Ġresur":15328,"ĠWa":15329,"venant":15330,"Ġmonetary":15331,"people":15332,"Email":15333,"Ġfifty":15334,"ĠSweet":15335,"ĠMalaysia":15336,"Ġconfusing":15337,"ĠRio":15338,"uda":15339,"utenant":15340,"\");":15341,"Ġpraised":15342,"Ġvolumes":15343,"turn":15344,"Ġmature":15345,"Ġnonprofit":15346,"Ġpassionate":15347,"ĠPrivate":15348,"Ġ103":15349,"Ġdescend":15350,"ç¥ŀ":15351,"uffy":15352,"headed":15353,"Whether":15354,"rien":15355,"zech":15356,"beit":15357,"Ġchrom":15358,"ĠMcM":15359,"Ġdancing":15360,"Ġeleg":15361,"ĠNoticed":15362,"115":15363,"Ġadvocacy":15364,"ENTS":15365,"ambling":15366,"ĠMinor":15367,"ĠFinn":15368,"Ġpriorities":15369,"Ġthereof":15370,"ĠStage":15371,"ĠRogers":15372,"Ġsubstitute":15373,"ĠJar":15374,"ĠJefferson":15375,"Ġlightly":15376,"102":15377,"ĠLisa":15378,"uits":15379,"ysical":15380,"Ġshifts":15381,"Ġdrones":15382,"Ġworkplace":15383,"Ġresid":15384,"ensed":15385,"ahn":15386,"Ġpreferences":15387,"server":15388,"Ġdebates":15389,"doc":15390,"ĠGods":15391,"Ġhelicopter":15392,"Ġhonour":15393,"Ġconsiderably":15394,"eded":15395,"ĠFemale":15396,"ĠAnne":15397,"Ġreun":15398,"ĠFace":15399,"ĠHallow":15400,"ĠBudget":15401,"Ġcondemn":15402,"Ġtender":15403,"Prof":15404,"ocratic":15405,"ĠTurner":15406,"ĠAgric":15407,"Ġ1976":15408,"Ġapt":15409,"disc":15410,"ĠFighter":15411,"ĠAur":15412,"Ġgarbage":15413,"input":15414,"ĠKarl":15415,"ĠOliver":15416,"ĠLanguage":15417,"kn":15418,"Non":15419,"ĠClar":15420,"Ġtraditions":15421,"Ġadvertisement":15422,"ĠSor":15423,"Ġarchive":15424,"Ġvillages":15425,"750":15426,"Ġimplementing":15427,"waukee":15428,"Ġdietary":15429,"Ġswitching":15430,"Republic":15431,"Ġvelocity":15432,"Ġcit":15433,"ĠAwards":15434,"Ġfinancing":15435,"Ġlasted":15436,")]":15437,"Ġreminder":15438,"Person":15439,"Ġprecision":15440,"Ġdesigners":15441,"ĠFried":15442,"ĠBorder":15443,"Ġtragic":15444,"Ġwield":15445,"Ġinitiatives":15446,"ĠTank":15447,"wer":15448,"Ġjoins":15449,"Ro":15450,"inery":15451,"Ġarrow":15452,"Ġgenerating":15453,"founder":15454,"Ġsearches":15455,"Ġrandomly":15456,"Access":15457,"Ġbatch":15458,"Ġposed":15459,"lat":15460,"Ġpursuing":15461,"asa":15462,"Ġtestified":15463,"forming":15464,"ĠShar":15465,"wiki":15466,"ĠEither":15467,"Sometimes":15468,"Ġsenators":15469,"ĠJohnny":15470,"ĠTaliban":15471,"ĠGPS":15472,"\":\"/":15473,"ãģ®å":15474,"Ġanalyzed":15475,"ĠRubio":15476,"ĠMovement":15477,"opard":15478,"iii":15479,"Stand":15480,"fight":15481,"Ġignoring":15482,"iang":15483,"ĠGN":15484,"soever":15485,"ĠSTAT":15486,"Ġrefusing":15487,"Ġsweat":15488,"Ġbay":15489,"PORT":15490,"irmed":15491,"aky":15492,"Ġdispro":15493,"Ġlabeled":15494,"Ġ108":15495,"Hello":15496,"Ġpleasant":15497,"aba":15498,"Ġtriumph":15499,"Ġaboard":15500,"Ġincom":15501,"ĠCrow":15502,"lett":15503,"Ġfolk":15504,"Ġchase":15505,"``":15506,"ĠBrus":15507,"Ġteens":15508,"cue":15509,"Ġterrain":15510,"hyd":15511,"ilight":15512,"ORY":15513,"Support":15514,"ews":15515,"lli":15516,"raints":15517,"ĠCand":15518,"Ġabused":15519,"achment":15520,"larg":15521,"Bas":15522,"ĠCancer":15523,"Ġ1978":15524,"Ġsupporter":15525,"access":15526,"ĠTermin":15527,"ĠTampa":15528,"ĠANY":15529,"Ġnewest":15530,"ĠCriminal":15531,"edu":15532,"Ġ1930":15533,"Ġadmits":15534,"Ġende":15535,"Ġfailures":15536,"urate":15537,"fulness":15538,"cycl":15539,"ĠSubject":15540,"Ġinfinite":15541,"three":15542,"WA":15543,"pit":15544,"ĠInstall":15545,"Rad":15546,"iliation":15547,"GM":15548,"Ġcontinent":15549,"Ġaccommodate":15550,"ĠClay":15551,"Ġpup":15552,"ĠFunction":15553,"Ġhammer":15554,"ĠAlberta":15555,"Ġrevised":15556,"Ġminorities":15557,"Ġmeasurement":15558,"Connell":15559,"Ġdisable":15560,"ĠMix":15561,"Incre":15562,"Ġfork":15563,"ĠRosen":15564,"Ġimplies":15565,"umblr":15566,"ANG":15567,"Ġproteins":15568,"Ġaggression":15569,"Ġfacilitate":15570,"SN":15571,"Ġillegally":15572,"uer":15573,"Ġacadem":15574,"Ġpuzz":15575,"ĠShift":15576,"pay":15577,"ollo":15578,"Ġaudiences":15579,"Build":15580,"Ġnoble":15581,"Ġsyntax":15582,"âĺħ":15583,"Ġbeam":15584,"ĠBed":15585,"ĠAld":15586,"Ġorigins":15587,"video":15588,"Ġ1977":15589,"ĠAssault":15590,"Ġgarage":15591,"Team":15592,"Ġverdict":15593,"Ġdwar":15594,"ĠVirtual":15595,"event":15596,"Keep":15597,"Ġsentiment":15598,"Ġwildlife":15599,"shirt":15600,"Ġburg":15601,"Ġrecommendation":15602,"represent":15603,"Ġgallery":15604,"owners":15605,"Ġscholar":15606,"Ġconvenience":15607,"ĠSwift":15608,"Ġconvinc":15609,"Cap":15610,"Ġwarfare":15611,"ĠVisual":15612,"Ġconstitute":15613,"Ġabort":15614,"ĠWeather":15615,"ĠLooking":15616,"ĠHem":15617,"Ġmartial":15618,"Ġincoming":15619,"etition":15620,"Ġtolerance":15621,"ĠCreated":15622,"Ġflows":15623,"ĠElder":15624,"Ġsouls":15625,"Ġfoul":15626,"ĠPain":15627,"ĠCAN":15628,"Ġ220":15629,"bc":15630,"hend":15631,"Ġgenius":15632,"Real":15633,"ĠWr":15634,"ometer":15635,"pad":15636,"Ġlimiting":15637,"ĠSi":15638,"ĠLore":15639,"ĠAdventures":15640,"Ġvaried":15641,"Disc":15642,"fin":15643,"ĠPersonal":15644,"Chris":15645,"Ġinvented":15646,"Ġdive":15647,"ĠRise":15648,"Ġoz":15649,"ĠComics":15650,"Ġexpose":15651,"ĠReb":15652,"letters":15653,"site":15654,"imated":15655,"Ġhacking":15656,"Ġeducated":15657,"ĠNobody":15658,"Ġdepri":15659,"Ġincentive":15660,"ãĤ·":15661,"Ġoversight":15662,"Ġtribes":15663,"ĠBelgium":15664,"Ġlicensing":15665,"ourt":15666,"Product":15667,"ahl":15668,"ĠGem":15669,"Ġspecialist":15670,"Ġcra":15671,"anners":15672,"ĠCorbyn":15673,"Ġ1973":15674,"READ":15675,"Ġsummar":15676,"Ġoverlook":15677,"ĠApplication":15678,"Ġinappropriate":15679,"Ġdownloaded":15680,"Que":15681,"ĠBears":15682,"Ġthumb":15683,"ĠCharacter":15684,"ĠReincarnated":15685,"ĠSid":15686,"Ġdemonstrates":15687,"sky":15688,"ĠBloomberg":15689,"ĠArray":15690,"ĠResults":15691,"ĠFourth":15692,"ĠEDT":15693,"ĠOscar":15694,"cend":15695,"Ġ106":15696,"ĠNULL":15697,"ĠHERE":15698,"match":15699,"ĠBrun":15700,"Ġglucose":15701,"ieg":15702,"egu":15703,"Ġcertified":15704,"Ġrelie":15705,"Ġhumanitarian":15706,"Ġprayers":15707,"King":15708,"Ġnan":15709,"hou":15710,"108":15711,"ulu":15712,"Ġrenewable":15713,"Ġdistinguish":15714,"Ġdense":15715,"ĠVent":15716,"ĠPackage":15717,"ĠBoss":15718,"Ġeditors":15719,"Ġmigr":15720,"Tra":15721,"ĠPeters":15722,"ĠArctic":15723,"2004":15724,"ĠCape":15725,"Ġlocally":15726,"Ġlasting":15727,"Ġhandy":15728,".).":15729,"Pan":15730,"ĠRES":15731,"Index":15732,"Ġtensions":15733,"Ġformerly":15734,"Ġideological":15735,"Ġsensors":15736,"Ġdealers":15737,"Ġdefines":15738,"Sk":15739,"Ġproceeds":15740,"Ġproxy":15741,"azines":15742,"ĠBash":15743,"ĠPad":15744,"ĠCraft":15745,"ealous":15746,"Ġsheets":15747,"ometry":15748,"June":15749,"clock":15750,"TT":15751,"ĠTheatre":15752,"ĠBuzz":15753,"Ġchapters":15754,"Ġmillenn":15755,"Ġdough":15756,"ĠCongressional":15757,"Ġimagined":15758,"avior":15759,"Ġclinic":15760,"Ġ1945":15761,"Ġholder":15762,"root":15763,"olester":15764,"Ġrestart":15765,"BN":15766,"ĠHamas":15767,"ĠJob":15768,"Ġorb":15769,"Ġram":15770,"Ġdisclose":15771,"Ġtranslate":15772,"Ġimmigrant":15773,"Ġannoying":15774,"Ġtreaty":15775,"anium":15776,"ĠTea":15777,"ĠLegion":15778,"Ġcrowds":15779,"ĠBec":15780,"ĠAer":15781,"ohyd":15782,"Bro":15783,"Looking":15784,"Ġlbs":15785,"Ġaggress":15786,"Ġseam":15787,"Ġintercept":15788,"ĠMI":15789,"mercial":15790,"activ":15791,"ĠCit":15792,"Ġdimension":15793,"Ġconsistency":15794,"Ġrushing":15795,"ĠDouglas":15796,"Ġtrim":15797,"Install":15798,"icker":15799,"Ġshy":15800,"106":15801,"Ġmentions":15802,"pelled":15803,"ĠTak":15804,"cost":15805,"Ġclassroom":15806,"Ġfortune":15807,"driven":15808,"Ġunle":15809,"ĠWheel":15810,"Ġinvestor":15811,"ĠMasters":15812,"kit":15813,"Ġassociations":15814,"ĠEvolution":15815,"oping":15816,"uscript":15817,"Ġprovincial":15818,"ĠWalter":15819,"avi":15820,"SO":15821,"Ġunlimited":15822,"English":15823,"ĠCards":15824,"ĠEbola":15825,"nered":15826,"Ġrevenge":15827,"Ġoutright":15828,"umper":15829,"Ġfitting":15830,"ĠSolid":15831,"Ġformally":15832,"Ġproblematic":15833,"Ġhazard":15834,"Ġencryption":15835,"Ġstraightforward":15836,"ĠAK":15837,"Ġpse":15838,"ĠOrb":15839,"ĠChamber":15840,"ĠMak":15841,"Contents":15842,"Ġloyalty":15843,"Ġlyrics":15844,"ĠSym":15845,"Ġwelcomed":15846,"Ġcooked":15847,"Ġmonop":15848,"Ġnurse":15849,"Ġmisleading":15850,"Ġeternal":15851,"Ġshifting":15852,"Ġ+=":15853,"Vis":15854,"Ġinstitutional":15855,"illary":15856,"Ġpant":15857,"VERT":15858,"ĠACC":15859,"ĠEnh":15860,"Ġincon":15861,"ĠREUTERS":15862,"Ġdonated":15863,"â̦â̦â̦â̦":15864,"Intern":15865,"Ġexhibit":15866,"Ġtire":15867,"ĠRic":15868,"ĠChampion":15869,"ĠMuhammad":15870,"NING":15871,"ĠSoccer":15872,"Ġmobility":15873,"Ġvarying":15874,"ĠMovie":15875,"Ġlord":15876,"oak":15877,"Field":15878,"Ġvector":15879,"usions":15880,"Ġscrap":15881,"Ġenabling":15882,"make":15883,"Tor":15884,".*":15885,"||":15886,"ĠWebsite":15887,"ĠNPC":15888,"Ġsocialist":15889,"ĠBilly":15890,"ĠAdditional":15891,"Ġcargo":15892,"Ġfarms":15893,"ĠSoon":15894,"ĠPrize":15895,"Ġmidnight":15896,"Ġ900":15897,"seen":15898,"ĠSpot":15899,"Ġsheep":15900,"Ġsponsored":15901,"ĠHi":15902,"ĠJump":15903,"Ġ1967":15904,"Microsoft":15905,"ĠAgent":15906,"Ġcharts":15907,"dir":15908,"Ġadjacent":15909,"Ġtricks":15910,"Ġmanga":15911,"Ġexagger":15912,"/>":15913,"football":15914,"ĠFCC":15915,"GC":15916,"ĠTier":15917,"andra":15918,"OUND":15919,"%),":15920,"Ġfruits":15921,"VC":15922,"ĠAA":15923,"Rober":15924,"Ġmidst":15925,"âĹ":15926,"anka":15927,"Ġlegislature":15928,"ĠNeil":15929,"Ġtourists":15930,"\"\"":15931,"ĠWarning":15932,"ĠNevertheless":15933,"ĠOfficial":15934,"ĠWhatever":15935,"Ġmold":15936,"Ġdrafted":15937,"Ġsubstances":15938,"Ġbreed":15939,"Ġtags":15940,"ĠTask":15941,"Ġverb":15942,"Ġmanufactured":15943,"comments":15944,"ĠPolish":15945,"Prov":15946,"Ġdetermines":15947,"Obama":15948,"kers":15949,"Ġutterly":15950,"Ġsect":15951,"sche":15952,"ĠGates":15953,"ĠChap":15954,"Ġaluminum":15955,"Ġzombie":15956,"ĠTouch":15957,"ĠUP":15958,"Ġsatisfy":15959,"Ġpredomin":15960,"ascript":15961,"Ġelaborate":15962,"Ġ1968":15963,"Ġmeasuring":15964,"ĠVari":15965,"anyahu":15966,"Ġsir":15967,"ulates":15968,"idges":15969,"ickets":15970,"ĠSpencer":15971,"TM":15972,"oubted":15973,"Ġprey":15974,"Ġinstalling":15975,"ĠCab":15976,"reed":15977,"reated":15978,"Supp":15979,"Ġwrist":15980,"ĠKerry":15981,"107":15982,"ĠKle":15983,"ĠRachel":15984,"Ġcotton":15985,"ĠARE":15986,"ĠEle":15987,"Control":15988,"Ġloads":15989,"ĠDod":15990,"anas":15991,"bone":15992,"Ġclassical":15993,"ĠRegional":15994,"ĠInteg":15995,"VM":15996,"Ġdesires":15997,"Ġautism":15998,"supported":15999,"ĠMessage":16000,"Ġcompact":16001,"writer":16002,"Ġ109":16003,"ĠHurricane":16004,"cision":16005,"Ġcycles":16006,"Ġdrill":16007,"Ġcolleague":16008,"Ġmaker":16009,"German":16010,"Ġmistaken":16011,"Sun":16012,"ĠGay":16013,"Ġwhatsoever":16014,"Ġsells":16015,"ĠAirl":16016,"liv":16017,"ĠOption":16018,"Ġsolved":16019,"Ġsectors":16020,"Ġhorizontal":16021,"Ġequation":16022,"ĠSkill":16023,"ĠBio":16024,"gement":16025,"ĠSnap":16026,"ĠLegal":16027,"Ġtrademark":16028,"Ġmakeup":16029,"Ġassembled":16030,"Ġsaves":16031,"ĠHalloween":16032,"ĠVermont":16033,"ĠFROM":16034,"Ġfarming":16035,"ĠPodcast":16036,"acceptable":16037,"ĠHigher":16038,"Ġasleep":16039,"ullivan":16040,"Ġreferen":16041,"ĠLev":16042,"Ġbullets":16043,"oko":16044,"HC":16045,"Ġstairs":16046,"Ġmaintains":16047,"ĠLower":16048,"ĠVi":16049,"Ġmarine":16050,"Ġacres":16051,"Ġcoordinator":16052,"ĠJoh":16053,"Ġcounterparts":16054,"ĠBrothers":16055,"Ġindict":16056,"bra":16057,"Ġchunk":16058,"Ġcents":16059,"Home":16060,"ĠMonth":16061,"Ġaccordingly":16062,"ifles":16063,"ĠGermans":16064,"ĠSyn":16065,"Hub":16066,"Ġeyeb":16067,"âĶĢâĶĢâĶĢâĶĢ":16068,"Ġranges":16069,"ĠHolland":16070,"ĠRobot":16071,"fc":16072,"Mike":16073,"Ġplasma":16074,"Ġswap":16075,"Ġathlete":16076,"ĠRams":16077,",'\"":16078,"Ġinfections":16079,"Ġcorrid":16080,"Ġvib":16081,"Ġpatches":16082,"Ġtraditionally":16083,"Ġrevelation":16084,"Ġsweep":16085,"Ġglance":16086,"Ġinex":16087,"2003":16088,"ĠRaw":16089,"working":16090,"osures":16091,"ĠDat":16092,"ĠLynch":16093,"Ġleverage":16094,"ĠReid":16095,"Ġcorrelation":16096,"iances":16097,"avascript":16098,"Ġrepository":16099,"retty":16100,"Ġ1972":16101,"240":16102,"Ġoun":16103,"pol":16104,"ĠReed":16105,"Ġtactical":16106,"isite":16107,"Apple":16108,"ĠQuinn":16109,"Ġraped":16110,"illo":16111,"Europe":16112,"Ġalgorithms":16113,"ĠRodrig":16114,"iu":16115,"Ġillum":16116,"Ġfame":16117,"Ġintroducing":16118,"Ġdelays":16119,"ĠRaiders":16120,"Ġwhistle":16121,"Ġnovels":16122,"ĠReally":16123,"Ġderiv":16124,"Ġpublications":16125,"ĠNeither":16126,"ĠCommerce":16127,"Ġaston":16128,"language":16129,"Notes":16130,"ĠRoth":16131,"ĠFear":16132,"Ġmate":16133,"Ġparade":16134,"ĠQB":16135,"Ġmaneu":16136,"ĠCincinnati":16137,"mitting":16138,"Ġwaist":16139,"ĠRew":16140,"Ġdiscont":16141,"а":16142,"Ġstaring":16143,"Ġalias":16144,"Ġsecurities":16145,"Ġtoilet":16146,"ĠJedi":16147,"Ġunlaw":16148,"vised":16149,"////////":16150,"](":16151,"ĠWeiss":16152,"Ġprest":16153,"ĠCompan":16154,"Ġmemo":16155,"ĠGrace":16156,"July":16157,"ĠElite":16158,"center":16159,"ĠStay":16160,"Ġgalaxy":16161,"Ġtooth":16162,"ĠSettings":16163,"Ġsubjected":16164,"ãĤ¦":16165,"Ġlineback":16166,"Ġretailers":16167,"ĠWant":16168,"Ġdangers":16169,"Air":16170,"Ġvoluntary":16171,"eway":16172,"Ġinterpreted":16173,"otine":16174,"ç":16175,"Ġpel":16176,"Service":16177,"ĠEventually":16178,"Ġcareers":16179,"Ġthreaten":16180,"Ġmemor":16181,"ĠBradley":16182,"ancies":16183,"sn":16184,"ĠUnknown":16185,"National":16186,"Ġshadows":16187,"ailand":16188,"ĠDash":16189,"Everyone":16190,"izzard":16191,"March":16192,"=(":16193,"Ġpulls":16194,"Ġstranger":16195,"Ġbackwards":16196,"ĠBernard":16197,"imensional":16198,"Ġchron":16199,"Ġtheoretical":16200,"ktop":16201,"Ġware":16202,"ĠInvestig":16203,"ĠIniti":16204,"ĠOperations":16205,"oven":16206,"ocide":16207,"*/":16208,"Ġflames":16209,"ĠCash":16210,"shit":16211,"Ġcab":16212,"ĠAnaly":16213,"ĠSeah":16214,"Ġdefining":16215,"Ġordering":16216,"Ġimmun":16217,"Ġpersistent":16218,"ACH":16219,"Russian":16220,"mans":16221,"Ġhind":16222,"Ġphotography":16223,"©":16224,"Ġhug":16225,"Ġ107":16226,"ĠHence":16227,"iots":16228,"udeau":16229,"Ġsubsidies":16230,"Ġroutinely":16231,"ĠDevice":16232,"itic":16233,"Ġdisgust":16234,"lander":16235,"Ġ1940":16236,"Ġassignment":16237,"ĠBesides":16238,"wick":16239,"ĠDust":16240,"usc":16241,"structed":16242,"111":16243,"develop":16244,"Ġfond":16245,"Ġintersection":16246,"Ġdignity":16247,"Ġcommissioner":16248,"Without":16249,"reach":16250,"Ġcartoon":16251,"Ġscales":16252,"ãĥŃ":16253,"FIG":16254,"Ġsurveys":16255,"ĠIndonesia":16256,"Ġartwork":16257,"Ġunch":16258,"Ġcycling":16259,"unct":16260,"auer":16261,"orate":16262,"ĠObviously":16263,"Ġcharacterized":16264,"feld":16265,"Ġaffirm":16266,"Ġinnings":16267,"Ġé":16268,"Ġaliens":16269,"Ġcloth":16270,"etooth":16271,"ĠCertain":16272,"§":16273,"Ġdigest":16274,"know":16275,"ĠXL":16276,"Ġpredictions":16277,"Ġdin":16278,"WAR":16279,"Ġaftermath":16280,"Example":16281,"ĠSuccess":16282,"ĠThr":16283,"IGN":16284,"Ġminer":16285,"Bus":16286,"Ġclarity":16287,"heimer":16288,"ĠOUT":16289,"ĠSend":16290,"ĠCircle":16291,"ĠDiet":16292,"Ġpronounced":16293,"Ġcreators":16294,"Ġearthquake":16295,"attery":16296,"geons":16297,"Ġod":16298,"Ġlaying":16299,"orp":16300,"Ult":16301,"project":16302,"Ġundermin":16303,"Ġsequel":16304,"Sam":16305,"ĠDarkness":16306,"Ġreception":16307,"bull":16308,"YS":16309,"ĠVir":16310,"Ġsequences":16311,"ĠCoin":16312,"Ġoutfit":16313,"ĠWait":16314,"119":16315,"Ġdelivers":16316,"......":16317,"Ġblown":16318,"ĠEsc":16319,"ĠMath":16320,"perm":16321,"ĠUl":16322,"Ġglim":16323,"Ġfacial":16324,"Ġgreenhouse":16325,"Ġtokens":16326,"/-":16327,"ĠAnnual":16328,"ĠONE":16329,"Ġteenage":16330,"ĠPhysical":16331,"ĠLang":16332,"ĠCelt":16333,"Ġsued":16334,"ividually":16335,"Ġpatience":16336,"chair":16337,"regular":16338,"Ġaug":16339,"inv":16340,"except":16341,"ĠLil":16342,"Ġnest":16343,"fd":16344,"sum":16345,"ĠChase":16346,"Russia":16347,"ĠJennifer":16348,"Ġoffseason":16349,"Overall":16350,"Fore":16351,"Ġriot":16352,"Aud":16353,"former":16354,"Ġdefenders":16355,"ĠCT":16356,"iotic":16357,"ribly":16358,"Ġautomated":16359,"Ġpenis":16360,"Ġinsist":16361,"Ġdiagram":16362,"ĠSQL":16363,"ĠGarc":16364,"Ġwitch":16365,"client":16366,"ierra":16367,"ambers":16368,"Ġrecount":16369,"far":16370,"Very":16371,"osterone":16372,"Ġappreciated":16373,"ĠPerfect":16374,"Section":16375,"Ġdoses":16376,"ocaust":16377,"Ġcostly":16378,"Ġgrams":16379,"ĠShi":16380,"Ġwrestling":16381,"Ġ1971":16382,"Ġtrophy":16383,"Ġnerve":16384,"ĠKaz":16385,"ĠExperience":16386,"Ġpledged":16387,"Ġplayback":16388,"Ġcreativity":16389,"bye":16390,"Ġattackers":16391,"Ġholders":16392,"ĠCoach":16393,"ĠPhD":16394,"Ġtransfers":16395,"Ġcolored":16396,"ĠHindu":16397,"Ġdrown":16398,"Ġlistened":16399,"ĠWA":16400,"iasm":16401,"PO":16402,"Ġappealing":16403,"Ġdisclosed":16404,"ĠChicken":16405,"agging":16406,"Ġpleaded":16407,"Ġnavigation":16408,"ĠReturns":16409,"Ġ[[":16410,"ROR":16411,"EA":16412,"Ġphotographer":16413,"ĠRider":16414,"ippers":16415,"Ġslice":16416,"Ġerect":16417,"Ġhed":16418,"issance":16419,"ĠVikings":16420,"urious":16421,"Ġappet":16422,"oubtedly":16423,"Child":16424,"Ġauthentic":16425,"oos":16426,"ĠMaking":16427,"Ġannouncing":16428,"Ġbod":16429,"Ġmeter":16430,"ĠNine":16431,"ĠRogue":16432,"Ġworkforce":16433,"Ġrenewed":16434,"Ġorganisations":16435,"acs":16436,"PLE":16437,"Short":16438,"Ġcompounds":16439,"ĠVisit":16440,"Ġenvelop":16441,"earth":16442,"Ġsupportive":16443,"ggle":16444,"ĠBrussels":16445,"ĠGuild":16446,"Create":16447,"REL":16448,"Ġaveraged":16449,"Ġ1969":16450,"riages":16451,"Ġlengthy":16452,"Ġforgot":16453,"Okay":16454,"ĠErd":16455,"Ġdealer":16456,"Ġrecession":16457,"DD":16458,"Ġdesperately":16459,"Ġhunger":16460,"Ġsticks":16461,"Ġmph":16462,"ĠFaith":16463,"Ġintentionally":16464,"Ġdemol":16465,"ueller":16466,"ĠSale":16467,"Ġdebris":16468,"spring":16469,"Ġleap":16470,">>>>":16471,"Ġcontainers":16472,"selling":16473,"ranean":16474,"attering":16475,"Ġcommented":16476,"ĠCM":16477,"onut":16478,"Ġwoods":16479,"especially":16480,"Ġorganize":16481,"ivic":16482,"ĠWoods":16483,"anga":16484,"squ":16485,"Ġmaj":16486,"amon":16487,"Ġaxis":16488,"Ġ1974":16489,"ĠDenmark":16490,"Ġwarrior":16491,"ĠPand":16492,"Ġoutlined":16493,"ĠBO":16494,"insula":16495,"zilla":16496,"ebook":16497,"Ġdare":16498,"Ġsearched":16499,"Ġnavigate":16500,"Sn":16501,"writing":16502,"Ġunited":16503,"Japan":16504,"ĠHebrew":16505,"Ġflame":16506,"Ġrelies":16507,"Ġcatching":16508,"ĠSho":16509,"Ġimprisonment":16510,"Ġpockets":16511,"Ġclosure":16512,"ĠFam":16513,"tim":16514,"adequ":16515,"Activity":16516,"Ġrecruiting":16517,"ĠWATCH":16518,"ĠArgentina":16519,"dest":16520,"Ġapologize":16521,"oro":16522,"Ġlacks":16523,"Ġtuned":16524,"ĠGriffin":16525,"Ġinfamous":16526,"Ġcelebrity":16527,"sson":16528,"Ġ----------------------------------------------------------------":16529,"ĠIsis":16530,"ĠDisplay":16531,"Ġcredibility":16532,"Ġeconomies":16533,"Ġheadline":16534,"ĠCowboys":16535,"Ġindef":16536,"Ġlately":16537,"Ġincentives":16538,"button":16539,"ĠMob":16540,"Aut":16541,"Ġresigned":16542,"ĠOm":16543,"camp":16544,"Ġprofiles":16545,"Ġschemes":16546,"olphins":16547,"ayed":16548,"Clinton":16549,"enh":16550,"ĠYahoo":16551,"Ġabst":16552,"Ġank":16553,"suits":16554,"Ġwished":16555,"ĠMarco":16556,"udden":16557,"Ġsphere":16558,"ĠBishop":16559,"Ġincorporated":16560,"ĠPlant":16561,"114":16562,"Ġhated":16563,"pic":16564,"Ġdonate":16565,"Ġlined":16566,"Ġbeans":16567,"Ġstealing":16568,"Ġcostume":16569,"Ġsheriff":16570,"Ġforty":16571,"Ġintact":16572,"Ġadapted":16573,"Ġtravelling":16574,"bart":16575,"Ġnicely":16576,"Ġdried":16577,"Ġscal":16578,"osity":16579,"NOTE":16580,"ĠBh":16581,"ĠBroncos":16582,"ĠIgn":16583,"Ġintimate":16584,"Ġchemistry":16585,"Ġoptimal":16586,"Deb":16587,"ĠGeneration":16588,"Ġ],":16589,"ichi":16590,"ĠWii":16591,"ĠYOUR":16592,"ventions":16593,"Write":16594,"Ġpopul":16595,"unning":16596,"ĠWor":16597,"Vol":16598,"Ġqueen":16599,"heads":16600,"KK":16601,"Ġanalyze":16602,"opic":16603,"earchers":16604,"Ġdot":16605,"legraph":16606,"astically":16607,"Ġupgrades":16608,"Ġcares":16609,"Ġextending":16610,"Ġfreeze":16611,"Ġinability":16612,"Ġorgans":16613,"Ġpretend":16614,"Ġoutlet":16615,"113":16616,"olan":16617,"ĠMall":16618,"uling":16619,"talk":16620,"Ġexpressing":16621,"ĠAlways":16622,"ĠBegin":16623,"files":16624,"Ġlicenses":16625,"%%":16626,"ĠMitt":16627,"Ġfilters":16628,"ĠMilwaukee":16629,"GN":16630,"Ġunfold":16631,"Mo":16632,"Ġnutrition":16633,"ppo":16634,"Bo":16635,"Ġfounding":16636,"Ġundermine":16637,"Ġeasiest":16638,"ĠCzech":16639,"ĠMack":16640,"Ġsexuality":16641,"ĠNixon":16642,"Win":16643,"ĠArn":16644,"ĠKin":16645,"ãĤ£":16646,"icer":16647,"Ġfortun":16648,"Ġsurfaces":16649,"aghd":16650,"Ġcarriers":16651,"ĠPART":16652,"ĠTib":16653,"Ġinterval":16654,"Ġfrustrating":16655,"ĠShip":16656,"ĠArmed":16657,"ffe":16658,"Ġboats":16659,"ĠAbraham":16660,"inis":16661,"Ġsuited":16662,"thread":16663,"iov":16664,"abul":16665,"ĠVenezuela":16666,"Ġtom":16667,"super":16668,"Ġcastle":16669,"although":16670,"ioxide":16671,"eches":16672,"Ġevolutionary":16673,"Ġnegotiate":16674,"Ġconfronted":16675,"Remember":16676,"Ġ170":16677,"Such":16678,"Ġ911":16679,"mult":16680,"ĠAbyss":16681,"urry":16682,"kees":16683,"spec":16684,"ĠBarbara":16685,"Ġbelonging":16686,"Ġvillain":16687,"istani":16688,"Ġaccountable":16689,"Ġportions":16690,"ĠDecl":16691,"Ur":16692,"ĠKate":16693,"gre":16694,"Ġmagazines":16695,"UCK":16696,"Ġregulate":16697,"omon":16698,"ĠAlmost":16699,"Ġoverview":16700,"Ġscram":16701,"Ġloot":16702,"ĠFitz":16703,"Ġcharacteristic":16704,"ĠSnake":16705,"say":16706,"ĠRico":16707,"Ġtrait":16708,"ĠJoined":16709,"aucus":16710,"Ġadaptation":16711,"ĠAirlines":16712,"Ġarchae":16713,"ĠIde":16714,"Ġbikes":16715,"Ġliterary":16716,"Ġinfluences":16717,"ĠUsed":16718,"Creat":16719,"Ġplea":16720,"ĠDefence":16721,"ĠAssass":16722,"Ġpond":16723,"ULT":16724,")\"":16725,"Ġevaluated":16726,"Ġobtaining":16727,"Ġdemographic":16728,"Ġvigil":16729,"aley":16730,"Ġspouse":16731,"ĠSeahawks":16732,"respons":16733,"ĠBelt":16734,"umatic":16735,"Ġrises":16736,"runner":16737,"ĠMichelle":16738,"Ġpotent":16739,"race":16740,"ĠPAC":16741,"Find":16742,"olesterol":16743,"ISS":16744,"ĠIntroduced":16745,"resses":16746,"ignment":16747,"Os":16748,"ĠTu":16749,"ĠDex":16750,"icides":16751,"Ġsparked":16752,"ĠLaura":16753,"ĠBryant":16754,"Ġsmiling":16755,"ĠNexus":16756,"Ġdefendants":16757,"ĠCatal":16758,"Ġdishes":16759,"shaped":16760,"Ġprolong":16761,"mt":16762,"($":16763,"ãĢĤ":16764,"Ġcalculations":16765,"ĠSame":16766,"Ġpiv":16767,"HH":16768,"Ġcancelled":16769,"Ġgrin":16770,"Ġterritories":16771,"istically":16772,"Come":16773,"ĠParent":16774,"Project":16775,"Ġneglig":16776,"ĠPrivacy":16777,"Ġammo":16778,"LECT":16779,"olutely":16780,"ĠEpic":16781,"Ġmisunder":16782,"wal":16783,"April":16784,"mos":16785,"pathy":16786,"ĠCarson":16787,"Ġalbums":16788,"ĠEasy":16789,"Ġpistol":16790,"<<":16791,"Ġ\\(":16792,"target":16793,"help":16794,"Ġinterpre":16795,"conscious":16796,"ĠHousing":16797,"ĠJoint":16798,"127":16799,"Ġbeers":16800,"science":16801,"ĠFirefox":16802,"effective":16803,"ĠCabin":16804,"ĠOkay":16805,"ĠApplic":16806,"Ġspacecraft":16807,"ĠSR":16808,"vet":16809,"ĠStrange":16810,"SB":16811,"Ġcorps":16812,"iberal":16813,"efficient":16814,"Ġprevalence":16815,"Ġeconomists":16816,"118":16817,"Thread":16818,"ordable":16819,"ODE":16820,"ĠCant":16821,"=-=-":16822,"ifiable":16823,"ĠAround":16824,"Ġpole":16825,"Ġwillingness":16826,"CLA":16827,"ĠKid":16828,"Ġcomplement":16829,"Ġscattered":16830,"Ġinmates":16831,"Ġbleeding":16832,"every":16833,"Ġqueue":16834,"ĠTrain":16835,"Ġhij":16836,"Ġmelee":16837,"pleted":16838,"Ġdigit":16839,"Ġgem":16840,"official":16841,"Ġlifting":16842,"е":16843,"Requ":16844,"itutes":16845,"Ġpackaging":16846,"ĠWorkers":16847,"hran":16848,"ĠLebanon":16849,"olesc":16850,"Ġpunished":16851,"ĠJuan":16852,"Ġjam":16853,"ĠDocument":16854,"Ġmapping":16855,"icates":16856,"Ġinevitably":16857,"Ġvanilla":16858,"ĠTon":16859,"Ġwatches":16860,"Ġleagues":16861,"Ġinitiated":16862,"degree":16863,"portion":16864,"Ġrecalls":16865,"Ġruin":16866,"Ġmelt":16867,"IAN":16868,"Ġhem":16869,"Exp":16870,"Ġbaking":16871,"ĠColomb":16872,"atible":16873,"Ġradius":16874,"plug":16875,"ĠIF":16876,"etically":16877,"Ġfict":16878,"HER":16879,"ĠTap":16880,"atinum":16881,"Ġink":16882,"Ġcoh":16883,"ĠWizard":16884,"both":16885,"tex":16886,"Ġspends":16887,"ĠCurrently":16888,"ĠPit":16889,"Ġneurons":16890,"ignt":16891,"Ġrall":16892,"Ġbuses":16893,"building":16894,"Ġadjustments":16895,"Ġcried":16896,"iblical":16897,"atted":16898,"ĠZion":16899,"ĠMatter":16900,"Ġmeditation":16901,"ĠDennis":16902,"Ġours":16903,"ĠTab":16904,"Ġrankings":16905,"ortal":16906,"Ġadvers":16907,"Ġsurrender":16908,"ĠGob":16909,"cium":16910,"omas":16911,"imeter":16912,"Ġmultiplayer":16913,"Ġheroin":16914,"Ġoptimistic":16915,"Ġindicator":16916,"ĠBrig":16917,"Ġgrocery":16918,"Ġapplicant":16919,"ĠRocket":16920,"vid":16921,"Exception":16922,"pent":16923,"Ġorganizing":16924,"Ġencounters":16925,"ĠTOD":16926,"Ġjewel":16927,"Save":16928,"ĠChristie":16929,"Ġheating":16930,"Ġlazy":16931,"ĠCP":16932,"Ġcousin":16933,"Config":16934,"Ġregener":16935,"Ġnearest":16936,"Ġachieving":16937,"ENS":16938,"throw":16939,"ĠRichmond":16940,"antle":16941,"2002":16942,"Ġanten":16943,"bird":16944,"133":16945,"Ġnarc":16946,"raint":16947,"unny":16948,"ĠHispanic":16949,"ournaments":16950,"Ġprophe":16951,"ĠThailand":16952,"ĠTi":16953,"Ġinjection":16954,"Ġinherit":16955,"ravis":16956,"Ġmedi":16957,"Ġwhoever":16958,"ĠDEBUG":16959,"GP":16960,"ĠHud":16961,"Card":16962,"prom":16963,"Ġpor":16964,"Ġoverhead":16965,"Law":16966,"Ġviolate":16967,"Ġheated":16968,"Ġdescriptions":16969,"Ġachievements":16970,"ĠBeer":16971,"ĠQuant":16972,"Was":16973,"Ġeighth":16974,"ĠIv":16975,"Ġspecialized":16976,"UPDATE":16977,"ĠDelta":16978,"Pop":16979,"Jul":16980,"ĠAsk":16981,"ophy":16982,"Ġnewsletters":16983,"ĠTool":16984,"Ġgard":16985,"ĠConfeder":16986,"ĠGMT":16987,"ĠAbbott":16988,"Ġimmunity":16989,"ĠVM":16990,"Islam":16991,"Ġimplicit":16992,"wd":16993,"Ġ1944":16994,"ravity":16995,"ometric":16996,"Ġsurviving":16997,"urai":16998,"ĠPrison":16999,"Ġrust":17000,"ĠSketch":17001,"Ġbees":17002,"ĠTheory":17003,"Ġmerit":17004,"Tex":17005,"chat":17006,"Ġmim":17007,"Ġpaste":17008,"ĠKoch":17009,"Ġignorance":17010,"ĠShoot":17011,"Ġbasement":17012,"United":17013,"ĠAdvis":17014,"height":17015,"Ġfoster":17016,"Ġdetain":17017,"information":17018,"Ġneural":17019,"';":17020,"Ġproves":17021,"allery":17022,"Ġinvitation":17023,"umbers":17024,"Ġcattle":17025,"Ġbicycle":17026,"zi":17027,"Ġconsultant":17028,"Ġapology":17029,"ĠTiger":17030,"Ġ123":17031,"999":17032,"Ġindividually":17033,"rt":17034,"igion":17035,"ĠBrazilian":17036,"Ġdisturb":17037,"Ġentrepreneurs":17038,"Ġforests":17039,"cerpt":17040,"plates":17041,"pher":17042,"clipse":17043,"Ġtwitter":17044,"Ġacids":17045,"ographical":17046,"hum":17047,"ĠBald":17048,"ifully":17049,"Ġcompiler":17050,"ĠDA":17051,"Ġdonor":17052,"asi":17053,"Ġtribal":17054,"lash":17055,"ĠConfig":17056,"Ġapplicants":17057,"Ġsalaries":17058,"135":17059,"Putin":17060,"ĠFocus":17061,"irs":17062,"Ġmisconduct":17063,"ĠHaz":17064,"Ġeaten":17065,"Mobile":17066,"Muslim":17067,"ĠMarcus":17068,"viol":17069,"Ġfavorable":17070,"Ġstub":17071,"adin":17072,"ĠHob":17073,"Ġfaithful":17074,"Ġelectronics":17075,"Ġvacuum":17076,"wait":17077,"backed":17078,"economic":17079,"dist":17080,"Ġtenure":17081,"Ġsincere":17082,"ĠTogether":17083,"ĠWave":17084,"Ġprogression":17085,"Ġdenying":17086,"Ġdistress":17087,"braska":17088,"third":17089,"Ġmixing":17090,"Ġcolonial":17091,"Ġprivately":17092,"Ġunrest":17093,"aternity":17094,"Ġpremises":17095,"anti":17096,"gregation":17097,"Ġlicence":17098,"ĠHind":17099,"ĠSamuel":17100,"Ġconvincing":17101,"ĠAce":17102,"ĠRust":17103,"ĠNetanyahu":17104,"Ġhandles":17105,"ĠPatch":17106,"oriented":17107,"aho":17108,"ĠGonz":17109,"Ġhackers":17110,"claimer":17111,"Ġcustoms":17112,"ĠGran":17113,"fighters":17114,"Ġluc":17115,"Ġmanuscript":17116,"arenthood":17117,"Ġdevil":17118,"Ġwarriors":17119,"Ġoffenders":17120,"William":17121,"Ġholidays":17122,"Ġnightmare":17123,"Ġlever":17124,"ifferent":17125,"Stat":17126,"Ġexhibition":17127,"puted":17128,"ĠPure":17129,"Ġalpha":17130,"Ġenthusiasm":17131,"ĠRepresentatives":17132,"EAR":17133,"ĠTyp":17134,"Ġwheat":17135,"ĠAlf":17136,"Ġcorrection":17137,"Ġevangel":17138,"ATT":17139,"Miss":17140,"Ġsoup":17141,"Ġimplied":17142,"param":17143,"Ġsexy":17144,"ĠLux":17145,"Ġrepublic":17146,"patch":17147,"ablish":17148,"Ġicons":17149,"Ġfathers":17150,"ĠGET":17151,"ĠCarib":17152,"Ġregulated":17153,"ĠCohen":17154,"ĠBobby":17155,"Ġner":17156,"Ġbent":17157,"ventory":17158,"ĠAlong":17159,"ĠEST":17160,"ĠWallace":17161,"Ġmurders":17162,"rise":17163,"kell":17164,"ĠCommonwealth":17165,"Ġnasty":17166,"eta":17167,"ĠMIT":17168,"Ġadministered":17169,"Ġgenuinely":17170,"Editor":17171,"nick":17172,"Ġhydro":17173,"********************************":17174,"ĠBle":17175,"Ġfines":17176,"Ġgorge":17177,"ausible":17178,"rh":17179,"Ġapple":17180,"mentioned":17181,"Ġrope":17182,"otyp":17183,"HR":17184,"Ġdisappointing":17185,"Ġcage":17186,"nik":17187,"Ġdoubts":17188,"ĠFREE":17189,"prints":17190,"ĠMUST":17191,"Ġvendors":17192,"ĠInqu":17193,"Ġliberals":17194,"Ġcontractor":17195,"Ġupside":17196,"children":17197,"Ġtricky":17198,"Ġregulators":17199,"charged":17200,"liter":17201,"Ġ***":17202,"Ġrebell":17203,"lang":17204,"Ġlocals":17205,"Ġphysicians":17206,"Ġhey":17207,"arse":17208,"tm":17209,"ĠLex":17210,"Ġbehavioral":17211,"successful":17212,"FX":17213,"Ġbrick":17214,"ovic":17215,"Ġconform":17216,"Ġreviewing":17217,"Ġinsights":17218,"Ġbiology":17219,"ĠRemove":17220,"ĠExtra":17221,"Ġcommitting":17222,"induced":17223,"ignty":17224,"igm":17225,"Ġatomic":17226,"Common":17227,"ĠEM":17228,"ĠPere":17229,"ĠItems":17230,"eh":17231,"Ġpreserved":17232,"ĠHood":17233,"Ġprisoner":17234,"Ġbankruptcy":17235,"Ġgren":17236,"ushes":17237,"Ġexploitation":17238,"Ġsignatures":17239,"Ġfinan":17240,"],\"":17241,"ĠMR":17242,"Ġmeg":17243,"remlin":17244,"Ġmusicians":17245,"Ġselecting":17246,"Ġexamining":17247,"INK":17248,"lated":17249,"Hi":17250,"Ġartic":17251,"Ġpets":17252,"Ġimpair":17253,"ĠMAN":17254,"Ġtablets":17255,"include":17256,"Range":17257,"Ġcaut":17258,"Ġlogs":17259,"Ġmounting":17260,"Ġunaware":17261,"Ġdynamics":17262,"ĠPalestine":17263,"ĠQuarter":17264,"ĠPurple":17265,"Ġma":17266,"ĠImport":17267,"Ġcollections":17268,"ciation":17269,"Ġsuccessor":17270,"Ġclone":17271,"Ġaiming":17272,"Ġpossessed":17273,"Ġsticking":17274,"Ġshaking":17275,"Ġlocate":17276,"ĠHockey":17277,"Turn":17278,"170":17279,"Ġfifteen":17280,"ĠHarrison":17281,"Ġcontinuously":17282,"ĠTC":17283,"ĠValent":17284,"ĠRescue":17285,"Ġbypass":17286,"amount":17287,"Ġmast":17288,"Ġprotects":17289,"Ġartistic":17290,"Ġsometime":17291,"Ġshoe":17292,"Ġshouted":17293,"ificant":17294,"etitive":17295,"ĠRegister":17296,"ĠJin":17297,"Ġconcentrated":17298,"lington":17299,"onies":17300,"Ġgenerator":17301,"yrim":17302,"ĠArmen":17303,"Ġclearing":17304,"ido":17305,"ĠTW":17306,"alph":17307,"Ġladies":17308,"Hard":17309,"Ġdialog":17310,"Ġinputs":17311,"æľ":17312,"Ġposes":17313,"Ġslots":17314,"ĠPremium":17315,"Ġleaks":17316,"Ġbosses":17317,"Ġ113":17318,"course":17319,"Acc":17320,"ĠNewton":17321,"ĠAustria":17322,"ĠMage":17323,"Ġteaches":17324,"abad":17325,"Ġwears":17326,"Ġcyl":17327,"Ġcurse":17328,"ĠSales":17329,"ĠWings":17330,"Ġpsy":17331,"Ġgaps":17332,"ĠIceland":17333,"ĠPinterest":17334,"Ġlandlord":17335,"Ġdefinitions":17336,"ĠKer":17337,"Ġsufficiently":17338,"ĠPence":17339,"ĠArchitect":17340,"Ġsurpass":17341,"Ġ114":17342,"Ġsuperhero":17343,"ĠDisease":17344,"Ġpriests":17345,"ĠCulture":17346,"Ġdefinitive":17347,"Ġsecretly":17348,"ĠDance":17349,"install":17350,"chief":17351,"ĠJessica":17352,"Would":17353,"Updated":17354,"Ġlocker":17355,"ĠKay":17356,"Ġmemorial":17357,"è¦":17358,"fat":17359,"Ġdisgu":17360,"Ġflavors":17361,"ĠBaseball":17362,"ĠResistance":17363,"Ġkicks":17364,"Ġenv":17365,"Ġteenagers":17366,"Dark":17367,"ĠCAR":17368,"Ġhalt":17369,"ĠLG":17370,"ĠGabriel":17371,"Ġfever":17372,"Ġsatur":17373,"Ġmall":17374,"Ġaffiliate":17375,"ĠSleep":17376,"ĠSpecific":17377,"ĠVel":17378,"Ġjar":17379,"ĠSacred":17380,"ĠEdwards":17381,"ĠACL":17382,"Ġretained":17383,"ĠGiant":17384,"Ġlimitation":17385,"inces":17386,"Ġrefusal":17387,"ĠTale":17388,"ĠButler":17389,"Ġaccidents":17390,"ĠCSS":17391,"Ġimported":17392,"ĠCopy":17393,"α":17394,"ERT":17395,"zel":17396,"Ġdivisions":17397,"hots":17398,"ĠAlb":17399,"ĠDS":17400,"Loader":17401,"Washington":17402,"atisf":17403,"ĠCreative":17404,"\\.":17405,"ĠAutom":17406,"redict":17407,"Ġreceptor":17408,"ĠCarlos":17409,"Method":17410,"oka":17411,"Ġmalicious":17412,"Ġstepping":17413,",[":17414,"ĠDad":17415,"Ġattraction":17416,"ĠEffects":17417,"ĠPirate":17418,"ĠCer":17419,"ĠIndustry":17420,"ĠRud":17421,"Ġcharter":17422,"Ġdining":17423,"Ġinsists":17424,"Ġconfigure":17425,"Ġ(#":17426,"ĠSimple":17427,"ĠScroll":17428,"UTC":17429,"175":17430,"ĠKon":17431,"Ġmarketplace":17432,"ĠãĤ":17433,"Ġrefres":17434,"Ġgates":17435,"erred":17436,"ĠPod":17437,"Ġbehave":17438,"Frank":17439,"node":17440,"Ġendorsed":17441,"hett":17442,"asive":17443,"ĠHomeland":17444,"Ġrides":17445,"ĠLeave":17446,"erness":17447,"Ġflooding":17448,"AFP":17449,"Ġrisen":17450,"Ġcontinually":17451,"Ġunanim":17452,"ĠContract":17453,"ĠPas":17454,"Ġguided":17455,"ĠChile":17456,"bd":17457,"Ġsucc":17458,"ptic":17459,"Ġcommittees":17460,"ĠLuther":17461,"ĠAnyone":17462,"Ġsab":17463,"124":17464,"Ġpixel":17465,"ĠBak":17466,"ĠTag":17467,"ĠBennett":17468,"Enter":17469,"small":17470,"ĠPresidential":17471,"Ġpul":17472,"Ġcontrace":17473,"archive":17474,"Ġcoastal":17475,"ĠKids":17476,"192":17477,"â̲":17478,"icky":17479,"INGTON":17480,"Ġwolf":17481,"ĠStalin":17482,"Tur":17483,"idget":17484,"amas":17485,"ĠUnless":17486,"Ġsponsor":17487,"Ġmorph":17488,"ĠChoose":17489,"Ġrunner":17490,"Ġunbel":17491,"Ġmud":17492,"ĠMana":17493,"Ġdubbed":17494,"Ġgodd":17495,"urers":17496,"window":17497,"Ġrelied":17498,"Ġcelebrating":17499,"osc":17500,"Ġ135":17501,"Ġlobbying":17502,"Ġincomplete":17503,"Ġrestriction":17504,"Ġincap":17505,"itus":17506,"Ġexpectation":17507,"ĠApollo":17508,"Ġintens":17509,"Ġsync":17510,"GH":17511,"Ġmanipulation":17512,"BY":17513,"Ġspear":17514,"Ġbreasts":17515,"Ġvolcan":17516,"ilia":17517,"Material":17518,"Ġformats":17519,"ĠBast":17520,"Ġparliamentary":17521,"Ġsnake":17522,"Ġservants":17523,"ĠTrudeau":17524,"ĠGrim":17525,"ĠArabic":17526,"ĠSCP":17527,"ĠBoys":17528,"station":17529,"Ġprospective":17530,"orde":17531,"initialized":17532,"Ġbored":17533,"ABLE":17534,"Ġaccessed":17535,"Ġtaxi":17536,"ĠShell":17537,"aiden":17538,"ursed":17539,"inates":17540,"ĠInsurance":17541,"ĠPete":17542,"September":17543,"650":17544,"Ġadventures":17545,"ĠCover":17546,"Ġtribute":17547,"Ġsketch":17548,"Ġempower":17549,"ĠØ":17550,"ĠGlenn":17551,"ĠDaw":17552,"=\\\"":17553,"ĠPolitics":17554,"Ġguides":17555,"Ġdioxide":17556,"ĠGore":17557,"ĠBright":17558,"ĠSierra":17559,"Ġvalued":17560,"cond":17561,"Ġpointer":17562,"Select":17563,"Ġrisky":17564,"Ġabsorb":17565,"images":17566,"Ġrefuses":17567,"Ġbonuses":17568,"___":17569,"Ġhilar":17570,"ĠFeatures":17571,"220":17572,"ĠCollector":17573,"Foot":17574,"Ġ1964":17575,"culus":17576,"Ġdawn":17577,"Ġworkout":17578,"ĠLO":17579,"Ġphilosophical":17580,"ĠSandy":17581,"ĠYouth":17582,"Ġliable":17583,"Af":17584,"blue":17585,"Ġoverturn":17586,"lessness":17587,"ĠTribune":17588,"ĠIng":17589,"Ġfactories":17590,"Ġcatches":17591,"Ġprone":17592,"Ġmatrix":17593,"Ġlogin":17594,"Ġinacc":17595,"Ġexert":17596,"sys":17597,"Ġneedle":17598,"ĠQur":17599,"Ġnotified":17600,"oulder":17601,"tx":17602,"Ġreminds":17603,"Ġpublishers":17604,"Ġnort":17605,"Ġgit":17606,"Ġflies":17607,"ĠEmily":17608,"Ġflowing":17609,"ĠAlien":17610,"ĠStrateg":17611,"Ġhardest":17612,"Ġmodification":17613,"API":17614,"ĠMY":17615,"Ġcrashes":17616,"stairs":17617,"number":17618,"Ġurging":17619,"channel":17620,"ĠFalcon":17621,"Ġinhabitants":17622,"Ġterrifying":17623,"Ġutilize":17624,"Ġbanner":17625,"Ġcigarettes":17626,"Ġsenses":17627,"ĠHolmes":17628,"Ġpractition":17629,"ĠPhillips":17630,"otto":17631,"Ġcompile":17632,"Model":17633,"ĠKo":17634,"Ġ[]":17635,"Americans":17636,"ĠTerms":17637,"Ġmedications":17638,"ĠAna":17639,"Ġfundamentally":17640,"ĠNotice":17641,"Ġweaker":17642,"Ġ0000":17643,"Ġgarlic":17644,"Ġoutbreak":17645,"Ġeconomist":17646,"ĠBirth":17647,"Ġobstacles":17648,"arcer":17649,"ĠOrthodox":17650,"Ġplacebo":17651,"ĠCrew":17652,"aspberry":17653,"ĠAngels":17654,"Ġdischarge":17655,"Ġdestructive":17656,"117":17657,"ĠRising":17658,"Ġdairy":17659,"late":17660,"Ġcollision":17661,"ĠTigers":17662,"eanor":17663,"ocumented":17664,"ĠInvalid":17665,"Ġdont":17666,"ĠLiter":17667,"ĠVa":17668,"Ġhydrogen":17669,"Ġvariants":17670,"ĠBrowns":17671,"Ġ1965":17672,"Ġindigenous":17673,"Ġtrades":17674,"Ġremainder":17675,"Ġswept":17676,"ĠImpact":17677,"Ġredist":17678,"Ġunint":17679,"graduate":17680,"ãĥķ":17681,"ĠWILL":17682,"ãģ®ç":17683,"ĠCritical":17684,"Ġfisher":17685,"Ġvicious":17686,"Ġreversed":17687,"Year":17688,"ĠSox":17689,"Ġshootings":17690,"Ġfilming":17691,"Ġtouchdowns":17692,"aires":17693,"mel":17694,"Ġgrandfather":17695,"Ġaffection":17696,"ingle":17697,"Ġoverly":17698,"Additional":17699,"Ġsupreme":17700,"ĠGrad":17701,"Ġsporting":17702,"Ġmercy":17703,"ĠBrooks":17704,"ounty":17705,"Ġperforms":17706,"Ġtightly":17707,"Ġdemons":17708,"Ġkillings":17709,"Ġfaction":17710,"ĠNova":17711,"auts":17712,"Ġundoubtedly":17713,"arin":17714,"Ġunderway":17715,"rak":17716,"Ġliv":17717,"ĠRegion":17718,"Ġbriefing":17719,"sers":17720,"cloud":17721,"ĠMik":17722,"usp":17723,"Ġprediction":17724,"azor":17725,"Ġportable":17726,"ĠGand":17727,"Ġpresenting":17728,"Ġ1080":17729,"»":17730,"ushi":17731,"ĠSpark":17732,"thereum":17733,"Ġjustification":17734,"ĠNy":17735,"Ġcontractors":17736,"mingham":17737,"ĠStyle":17738,"åħ":17739,"ĠChronicles":17740,"ĠPicture":17741,"Ġproving":17742,"Ġwives":17743,"sett":17744,"Ġmolecules":17745,"ĠFairy":17746,"Ġconsisting":17747,"Ġpier":17748,"alone":17749,"inition":17750,"Ġnucle":17751,"json":17752,"Ġgotta":17753,"Ġmobil":17754,"Ġverbal":17755,"arium":17756,"Ġmonument":17757,"ucked":17758,"Ġ256":17759,"Tech":17760,"minecraft":17761,"ĠTrack":17762,"Ġtile":17763,"Ġcompatibility":17764,"asis":17765,"Ġsadd":17766,"Ġinstructed":17767,"ĠMueller":17768,"Ġlethal":17769,"Ġhormone":17770,"Ġorche":17771,"else":17772,"Ġskelet":17773,"Ġentertaining":17774,"Ġminimize":17775,"again":17776,"Ġundergo":17777,"Ġconstraints":17778,"Ġcigarette":17779,"ĠIslamist":17780,"Ġtravels":17781,"ĠPanthers":17782,"lings":17783,"Care":17784,"Ġlawsuits":17785,"uras":17786,"Ġcryst":17787,"Ġlowered":17788,"Ġaerial":17789,"Ġcombinations":17790,"Ġhaun":17791,"Ġcha":17792,"Ġvine":17793,"Ġquantities":17794,"Ġlinking":17795,"bank":17796,"Ġsoy":17797,"Bill":17798,"ĠAngela":17799,"Ġrecipient":17800,"ĠProtest":17801,"Ġsocket":17802,"Ġsolidarity":17803,"ĠâĨ":17804,"mill":17805,"Ġvaries":17806,"ĠPakistani":17807,"Dragon":17808,"Ġune":17809,"Ġhorizon":17810,"³³³³³³³³":17811,"Ġprovinces":17812,"Ġfrankly":17813,"Ġenacted":17814,"notes":17815,"['":17816,"Ġ192":17817,"ocracy":17818,"Ġendorsement":17819,"Ġovertime":17820,"True":17821,"Lab":17822,"licted":17823,"ĠDNC":17824,"Ġbeats":17825,"ĠJamie":17826,"152":17827,"ĠINT":17828,"Contact":17829,"Ġaccounted":17830,"hash":17831,"ĠPackers":17832,"pires":17833,"Ġlesbian":17834,"Ġamendments":17835,"Ġhopeful":17836,"ĠFinland":17837,"Ġspotlight":17838,"Ġconfigured":17839,"Ġtroubled":17840,"Ġgaze":17841,"ĠCalgary":17842,"Ġreliability":17843,"Ġinsurg":17844,"swer":17845,"buy":17846,"ĠSkin":17847,"Ġpixels":17848,"Ġhandgun":17849,"Ġparas":17850,"Ġcategor":17851,"ĠEL":17852,"ĠRex":17853,"Indeed":17854,"Ġkinda":17855,"Ġconjunction":17856,"ĠBryan":17857,"ĠManufact":17858,"yang":17859,"Plus":17860,"SQL":17861,"ishment":17862,"Ġdominate":17863,"Ġnail":17864,"Ġoath":17865,"Ġerupt":17866,"ĠFine":17867,"itbart":17868,"ĠChip":17869,"ĠAbd":17870,"ĠNam":17871,"Ġbuyer":17872,"Ġdissent":17873,"Leaks":17874,"Contin":17875,"Ġrider":17876,"ĠSomeone":17877,"Ġillusion":17878,"cin":17879,"ĠBoeing":17880,"Ġinadequ":17881,"ovation":17882,"iants":17883,"Ġrebuild":17884,"450":17885,"ĠDestiny":17886,"SW":17887,"ĠTill":17888,"Hit":17889,"iaz":17890,"ĠBangl":17891,"achers":17892,"ĠReform":17893,"Ġsegments":17894,"Ġsystematic":17895,"dc":17896,"ĠConservatives":17897,"Ġportal":17898,"hor":17899,"ĠDragonbound":17900,"Ġdragged":17901,"omo":17902,"Ġthee":17903,"advert":17904,"ĠReports":17905,"ĠEt":17906,"Ġbarrels":17907,"August":17908,"Ġcomparisons":17909,"Ġhex":17910,"Ġanthrop":17911,"\"[":17912,"borough":17913,"abi":17914,"Ġpictured":17915,"playing":17916,"ĠAddress":17917,"ĠMirror":17918,"Smith":17919,"Ġtires":17920,"ĠNPR":17921,"AAAA":17922,"Ġclassification":17923,"ĠThan":17924,"ĠHarm":17925,"ĠRA":17926,"Ġrejection":17927,"mination":17928,"Ġranged":17929,"ĠFalls":17930,"DI":17931,"Host":17932,"ãĤ´":17933,"ĠExample":17934,"listed":17935,"thirds":17936,"Ġsafegu":17937,"brand":17938,"Ġprobable":17939,"Canada":17940,"ITION":17941,"ĠQaeda":17942,"Ġchick":17943,"Ġimports":17944,"hit":17945,"loc":17946,"WW":17947,"Ġblew":17948,"Ġanytime":17949,"Ġwholes":17950,"iked":17951,"Ġcalculation":17952,"create":17953,"ĠOri":17954,"Ġupgraded":17955,"Ġappar":17956,"utory":17957,"ĠMol":17958,"Brit":17959,"ĠJong":17960,"INAL":17961,"ĠStarting":17962,"Ġdice":17963,"urtle":17964,"Ġrelying":17965,"closure":17966,"Ġprofitable":17967,"Ġslaughter":17968,"ĠManual":17969,"caster":17970,"Ġ\"$":17971,"Ġfeather":17972,"ĠSimply":17973,"ieves":17974,"Ġdeterior":17975,"ĠPCI":17976,"Ġstamp":17977,"Ġflaws":17978,"Ġshade":17979,"hammer":17980,"Ġpassport":17981,"Ġconting":17982,"amel":17983,"Ġobservers":17984,"Ġneglect":17985,"ĠRB":17986,"ĠBrotherhood":17987,"Ġskeptical":17988,"family":17989,"usk":17990,"Ġemotionally":17991,"âĻ":17992,"ĠBeta":17993,"asonable":17994,"idity":17995,"ĠMul":17996,"Ġkicking":17997,"ĠCarm":17998,"ollah":17999,"VERTIS":18000,"ĠAthen":18001,"Ġladder":18002,"ĠBullet":18003,"å£":18004,"0001":18005,"ĠWildlife":18006,"ĠMask":18007,"ĠNan":18008,"Rev":18009,"Ġunacceptable":18010,"legal":18011,"Ġcrowded":18012,"agi":18013,"ĠCox":18014,"je":18015,"Ġmorality":18016,"Ġfuels":18017,"Ġcables":18018,"Ġmankind":18019,"ĠCaribbean":18020,"Ġanchor":18021,"Ġbyte":18022,"ĠOften":18023,"ĠOz":18024,"Ġcrafted":18025,"Ġhistorian":18026,"ĠWu":18027,"Ġtowers":18028,"ĠCitizens":18029,"Ġhelm":18030,"Ġcredentials":18031,"Ġsingular":18032,"ĠJesse":18033,"Ġtackles":18034,"Ġcontempt":18035,"Ġafore":18036,"ĠShadows":18037,"Ġnil":18038,"Ġurgent":18039,"apple":18040,"blood":18041,"Ġvon":18042,"Ġoffline":18043,"Ġbreathe":18044,"Ġjumps":18045,"Ġirrelevant":18046,"oxic":18047,"omal":18048,"important":18049,"Jim":18050,"Ġgloves":18051,"arming":18052,"depth":18053,"Ġtalents":18054,"ookie":18055,"ĠSB":18056,"Ġpalm":18057,"uffs":18058,"esta":18059,"IGH":18060,"Ġcanon":18061,"ĠVerizon":18062,"ĠPle":18063,"Ġcoupled":18064,"velt":18065,"Ġfundraising":18066,"ĠGetting":18067,"ĠDLC":18068,"Ġmathematical":18069,"ĠHS":18070,"ĠCardinals":18071,"telling":18072,"Ġsponsors":18073,"ĠÏ":18074,"ĠBulls":18075,"option":18076,"Ġpropose":18077,"Ġmemorable":18078,"Ġembraced":18079,"Ġdeclining":18080,"Health":18081,"eda":18082,"Ġ};":18083,"Ġspam":18084,"mile":18085,"Ġpitcher":18086,"ĠEight":18087,"Ġcaring":18088,"utic":18089,"role":18090,"Ġairline":18091,"ernandez":18092,"ĠAthlet":18093,"Ġcertification":18094,"uxe":18095,"riger":18096,"Ġempir":18097,"Ġsensation":18098,"Ġdism":18099,"Ġbolt":18100,"Ġevolve":18101,"House":18102,"Ġconsultation":18103,"ĠDuty":18104,"Ġtouches":18105,"ĠNathan":18106,"Ġfaint":18107,"had":18108,"\"(":18109,"ĠConsumer":18110,"ĠExtreme":18111,"Ġ127":18112,"ĠHerm":18113,"ĠSacrament":18114,"izoph":18115,"Ġanxious":18116,"ulously":18117,"Ġsocially":18118,"ĠUTC":18119,"Ġsolving":18120,"ĠLetter":18121,"History":18122,"educ":18123,"Price":18124,"));":18125,"Ġreload":18126,"amic":18127,"Ġpork":18128,"Ġdiscourse":18129,"Ġtournaments":18130,"airo":18131,"ĠKur":18132,"ĠCosta":18133,"Ġviolating":18134,"Ġinterfere":18135,"Ġrecreational":18136,"uffle":18137,"Ġspeeches":18138,"Ġneeding":18139,"Ġremembers":18140,"Ġcredited":18141,"nia":18142,"focused":18143,"amera":18144,"Ġbru":18145,"umbs":18146,"ĠCuban":18147,"Ġpreceding":18148,"Ġnonsense":18149,"acial":18150,"Ġsmartphones":18151,"ĠStories":18152,"Sports":18153,"ĠEmergency":18154,"ouncing":18155,"efined":18156,"Ġber":18157,"Ġconsulting":18158,"Ġmasters":18159,"heastern":18160,".\"[":18161,"ĠRunning":18162,"Ġsuscept":18163,"ĠFeng":18164,"America":18165,"prises":18166,"stitial":18167,"ĠWeekly":18168,"ĠGreater":18169,"modules":18170,"ifter":18171,"Graphics":18172,"uler":18173,"Ġwholly":18174,"Ġsuppress":18175,"Ġconcealed":18176,"Ġhappily":18177,"Ġaccepts":18178,"ĠEnjoy":18179,"Ġrivers":18180,"ĠExcept":18181,"225":18182,"ĠNHS":18183,"ĠMcConnell":18184,"Ġpussy":18185,"ferred":18186,"utable":18187,"Ġattain":18188,"Ġ>=":18189,"Ġdeposits":18190,"rophic":18191,"Ġnotorious":18192,"ĠShaw":18193,"ilitation":18194,"Ġepidemic":18195,"allic":18196,"Ġsmallest":18197,"ovich":18198,"Ġaccessories":18199,"perties":18200,"Ġsurplus":18201,"ĠMech":18202,"Ġambig":18203,"ĠImmigration":18204,"Ġchim":18205,"eval":18206,"Ġpracticing":18207,"ĠMystery":18208,"Ġdomains":18209,"ĠSilicon":18210,"apps":18211,"Ġkilometers":18212,"ea":18213,"ĠSmash":18214,"Ġwarranty":18215,"Ġnost":18216,"sil":18217,"rev":18218,"Jon":18219,"ĠDublin":18220,"Ġtastes":18221,"Ġbout":18222,"great":18223,"error":18224,"Ġswitches":18225,"ĠBapt":18226,"DO":18227,"oki":18228,"Ġsourced":18229,"produ":18230,"Ġattachment":18231,"ĠIssue":18232,"ĠQuestion":18233,"Join":18234,"Ġfitted":18235,"Ġunlawful":18236,"^^":18237,"erek":18238,"Ġauthentication":18239,"Ġstole":18240,"Ġaccountability":18241,"label":18242,"Search":18243,"Ġalbeit":18244,"atican":18245,"funded":18246,"ĠAdding":18247,"ĠIQ":18248,"Ġsubmar":18249,"lit":18250,"aque":18251,"ĠLearning":18252,"Ġinteger":18253,"Master":18254,"ĠChrom":18255,"Ġpremier":18256,"Op":18257,"ĠLiu":18258,"Ġblessed":18259,"ĠGlobe":18260,"ĠResponse":18261,"Ġlegitim":18262,"ĠMerkel":18263,"Ġdisposal":18264,"´":18265,"Ġgauge":18266,"peat":18267,"Ġinduced":18268,"Ġquestionable":18269,"arthy":18270,"ĠVit":18271,"ĠFeed":18272,"Until":18273,"Ut":18274,"worthy":18275,"RY":18276,"ĠHerald":18277,"ĠHammer":18278,"Ġmedal":18279,"ĠRivers":18280,"ĠHack":18281,"Ġclarify":18282,"Ġtracked":18283,"Ġautonomous":18284,"Ġtenant":18285,"ĠQatar":18286,"erie":18287,"Ġgrim":18288,"ĠMonitor":18289,"Ġresistant":18290,"ĠSpec":18291,"ĠWells":18292,"NAS":18293,"148":18294,"Ġminers":18295,"iotics":18296,"Ġmisses":18297,"116":18298,"gian":18299,"git":18300,"ĠEyes":18301,"pres":18302,"Ġgraduated":18303,"Ġangel":18304,"Ġsynchron":18305,"Ġefficiently":18306,"Ġtransmitted":18307,"Harry":18308,"Ġglobally":18309,"ENCE":18310,"ĠMontana":18311,"raged":18312,"ĠPrevention":18313,"Ġpiss":18314,"ĠLl":18315,"Ġshelf":18316,"ĠBJP":18317,"ĠTestament":18318,"ĠLate":18319,"iker":18320,"ĠHapp":18321,"ĠJulian":18322,"hall":18323,"Ġspont":18324,"Ġshutdown":18325,"Ġinconsistent":18326,"Ġsubscribers":18327,"Ġskeleton":18328,"ĠNebraska":18329,"Ġinspire":18330,"ĠVoid":18331,"Feed":18332,"Ġangles":18333,"ĠSprings":18334,"Ġbenchmark":18335,"Ġvaccines":18336,"izophren":18337,"sexual":18338,"uffed":18339,"Ġshine":18340,"ĠKath":18341,"Ġgesture":18342,"inea":18343,"Ġrip":18344,"Ġoppression":18345,"Ġconscience":18346,"bt":18347,"ĠLum":18348,"Ġincidence":18349,"ĠFa":18350,"wr":18351,"Ġmineral":18352,"ĠSpurs":18353,"alky":18354,"Ġthunder":18355,"Ġopio":18356,"Being":18357,"ĠPalm":18358,"Ġwasted":18359,"Ġlb":18360,"iaries":18361,"ĠInitiative":18362,"Ġcurric":18363,"Ġmarker":18364,"ĠMcL":18365,"Ġextensions":18366,"ĠPv":18367,"ĠArms":18368,"Ġofferings":18369,"Ġdefenses":18370,"Ġvendor":18371,"Ġcontradict":18372,"ĠColin":18373,"Ġreddit":18374,"Ġperipher":18375,"122":18376,"Ġsins":18377,"Edit":18378,"ICT":18379,"Soft":18380,"ĠShah":18381,"Ġadministrator":18382,"ĠTrip":18383,"Ġpornography":18384,"Ġtuition":18385,"inence":18386,"ĠProgress":18387,"Ġcatalog":18388,"Ġsuite":18389,"Ġhike":18390,"Ġreproductive":18391,"engine":18392,"Ġdrought":18393,"ĠNoah":18394,"Ġ230":18395,"Ġdude":18396,"Ġrelaxed":18397,"Ġpartition":18398,"Ġparticipant":18399,"Ġtelesc":18400,"Ġfeas":18401,"ĠFF":18402,"owner":18403,"Ġsweeping":18404,"Ġlenses":18405,"Ġmatchup":18406,"ĠRepl":18407,"ournals":18408,"Ġcredible":18409,"Ġgrandmother":18410,"Ġthermal":18411,"Ġsubscribing":18412,"Ġidentities":18413,"colm":18414,"UCT":18415,"Ġreluctant":18416,"users":18417,"ĠCort":18418,"Ġassisted":18419,"OSS":18420,"ATIONS":18421,"ISH":18422,"Ġpharmaceutical":18423,"icable":18424,"adian":18425,"ĠSonic":18426,"ĠFury":18427,"ĠMong":18428,"AH":18429,"ĠPsychology":18430,"Ġphosph":18431,"Ġtreats":18432,"ŃĶ":18433,"Ġsteadily":18434,"ĠHello":18435,"Ġrelates":18436,"Ġclue":18437,"Expl":18438,"auth":18439,"Ġrevision":18440,"Ġeld":18441,"osion":18442,"Ġbron":18443,"144":18444,"rikes":18445,"Ġmines":18446,"Ġblanket":18447,"ĠFail":18448,"eled":18449,"ĠImagine":18450,"ĠPlanned":18451,"aic":18452,"Request":18453,"Mad":18454,"ĠHorse":18455,"ĠEagle":18456,"Ġcapac":18457,"157":18458,"Ġling":18459,"ĠNice":18460,"ĠParenthood":18461,"minster":18462,"ogs":18463,"ensitive":18464,"Nothing":18465,"Ġcarn":18466,"Fin":18467,"ĠPE":18468,"Ġrifles":18469,"ĠLP":18470,"Sand":18471,"ĠguiActive":18472,"Ġtourist":18473,"CNN":18474,"Ġunveiled":18475,"Ġpredecessor":18476,"}{":18477,"uber":18478,"Ġoffshore":18479,"Ġoptical":18480,"ĠRot":18481,"ĠPearl":18482,"eton":18483,"Ġstared":18484,"Ġfarther":18485,"atility":18486,"contin":18487,"ĠGy":18488,"ĠFoster":18489,"ĠCoc":18490,"rients":18491,"Ġdesigning":18492,"ĠEconomy":18493,"ONG":18494,"Women":18495,"ĠNancy":18496,"erver":18497,"Ġmascul":18498,"Ġcasualties":18499,"Ġ225":18500,"ĠSullivan":18501,"ĠChoice":18502,"Ġaster":18503,"ws":18504,"Ġhotels":18505,"Ġconsiderations":18506,"Ġcouch":18507,"ĠStrip":18508,"ĠGn":18509,"Ġmanipulate":18510,"lied":18511,"Ġsynthetic":18512,"Ġassaulted":18513,"Ġoffenses":18514,"ĠDrake":18515,"Ġimpe":18516,"October":18517,"ĠHeritage":18518,"hl":18519,"ĠBlair":18520,"Unlike":18521,"Ġgrief":18522,"Ġ450":18523,"Ġopted":18524,"Ġresignation":18525,"ilo":18526,"Ġverse":18527,"ĠTomb":18528,"Ġupt":18529,"Ġaired":18530,"ĠHook":18531,"ĠMLB":18532,"Ġassumes":18533,"outed":18534,"ĠVers":18535,"Ġinferior":18536,"Ġbundle":18537,"ĠDNS":18538,"ographer":18539,"Ġmultip":18540,"ĠSouls":18541,"Ġillustrated":18542,"Ġtactic":18543,"Ġdressing":18544,"Ġduo":18545,"Conf":18546,"Ġrelent":18547,"Ġcant":18548,"Ġscarce":18549,"Ġcandy":18550,"ĠCF":18551,"Ġaffiliated":18552,"Ġsprint":18553,"ylan":18554,"ĠGarcia":18555,"Ġjunk":18556,"Print":18557,"exec":18558,"Crit":18559,"Ġportrait":18560,"iries":18561,"ĠOFF":18562,"Ġdisputes":18563,"WR":18564,"Love":18565,"ãģĦ":18566,"ĠReyn":18567,"Ġhipp":18568,"opath":18569,"Ġfloors":18570,"ĠFeel":18571,"Ġworries":18572,"Ġsettlements":18573,"ĠPos":18574,"Ġmosque":18575,"Ġfinals":18576,"Ġcrushed":18577,"ĠProbably":18578,"ĠBot":18579,"ĠMans":18580,"ĠPeriod":18581,"Ġsovereignty":18582,"Ġseller":18583,"Ġapost":18584,"Ġamateur":18585,"Ġdorm":18586,"Ġconsuming":18587,"Ġarmour":18588,"ĠRoose":18589,"Ġintensive":18590,"Ġeliminating":18591,"ĠSunni":18592,"ĠAleppo":18593,"jin":18594,"Ġadvise":18595,"pal":18596,"ĠHalo":18597,"Ġdescent":18598,"Ġsimpler":18599,"Ġbooth":18600,"STR":18601,"Later":18602,"ĠCave":18603,"===":18604,"Ġmol":18605,"Ġfist":18606,"Ġshotgun":18607,"supp":18608,"Ġrobbery":18609,"Effect":18610,"Ġobscure":18611,"ĠProfessional":18612,"Ġembassy":18613,"Ġmilitant":18614,"Ġincarcer":18615,"Ġgenerates":18616,"Ġlaunches":18617,"Ġadministrators":18618,"Ġshaft":18619,"Ġcircular":18620,"Ġfreshman":18621,"ĠWes":18622,"ĠJoel":18623,"ĠDrew":18624,"ĠDuncan":18625,"ĠApparently":18626,"sight":18627,"ĠInternal":18628,"ĠIndividual":18629,"ĠFE":18630,"Ġbore":18631,"ĠMt":18632,"Ġbroadly":18633,"ĠOptions":18634,"ountain":18635,"ipes":18636,"ĠVideos":18637,"204":18638,"Ġhills":18639,"Ġsimulation":18640,"Ġdisappointment":18641,"itan":18642,"ĠLaboratory":18643,"Ġupward":18644,"Ġboundary":18645,"Ġdarker":18646,"hart":18647,"Ġdominance":18648,"Cong":18649,"ĠOracle":18650,"ĠLords":18651,"Ġscholarship":18652,"ĠVincent":18653,"ede":18654,"ĠRah":18655,"Ġencourages":18656,"rov":18657,"Ġquo":18658,"Ġpremise":18659,"ĠCrisis":18660,"ĠHolocaust":18661,"Ġrhythm":18662,"Ġmetric":18663,"club":18664,"Ġtransported":18665,"Ġnod":18666,"ĠPist":18667,"Ġancestors":18668,"ĠFreder":18669,"thumbnails":18670,"ĠCE":18671,"OND":18672,"Phil":18673,"venge":18674,"ĠProducts":18675,"castle":18676,"Ġqualifying":18677,"ĠKaren":18678,"VERTISEMENT":18679,"Ġmighty":18680,"Ġexplanations":18681,"Ġfixing":18682,"Di":18683,"Ġdeclaring":18684,"Ġanonymity":18685,"Ġjuven":18686,"ĠNord":18687,"ĠDoom":18688,"ĠActually":18689,"Ok":18690,"phis":18691,"ĠDesert":18692,"Ġ116":18693,"IK":18694,"ĠFM":18695,"Ġincomes":18696,"VEL":18697,"okers":18698,"Ġpecul":18699,"Ġlightweight":18700,"gue":18701,"Ġaccent":18702,"Ġincrement":18703,"ĠChan":18704,"Ġcomplaining":18705,"ĠBaghd":18706,"Ġmidfielder":18707,"Ġoverhaul":18708,"Process":18709,"ĠHollow":18710,"ĠTitans":18711,"Small":18712,"manuel":18713,"ĠUnity":18714,"ĠEvents":18715,"Sty":18716,"Ġdisproportion":18717,"nesty":18718,"enes":18719,"ĠCod":18720,"Ġdemonstrations":18721,"ĠCrimson":18722,"ĠOH":18723,"Ġenrolled":18724,"Ġcel":18725,"ĠBrett":18726,"Ġaide":18727,"Ġheels":18728,"Ġbroadband":18729,"Ġmarking":18730,"Ġwizard":18731,"ĠNJ":18732,"ĠChiefs":18733,"Ġingredient":18734,"Ġdug":18735,"ĠShut":18736,"urchase":18737,"endor":18738,"Ġfarmer":18739,"ĠGoldman":18740,"129":18741,"155":18742,"Order":18743,"Ġlion":18744,"iably":18745,"Ġstain":18746,"array":18747,"ilitary":18748,"ĠFAQ":18749,"Ġexploded":18750,"ĠMcCarthy":18751,"ĠTweet":18752,"ĠGreens":18753,"eking":18754,"ln":18755,"ensen":18756,"Ġmotorcycle":18757,"Ġparticle":18758,"Ġcholesterol":18759,"Bron":18760,"Ġstair":18761,"Ġoxid":18762,"Ġdesirable":18763,"ibles":18764,"Ġtheor":18765,"forcing":18766,"Ġpromotional":18767,"ovo":18768,"boot":18769,"ĠBonus":18770,"rawling":18771,"Ġshortage":18772,"ĠPsy":18773,"Ġrecruited":18774,"Ġinfants":18775,"Ġtestosterone":18776,"Ġdeduct":18777,"Ġdistinctive":18778,"Ġfirmware":18779,"built":18780,"145":18781,"Ġexplored":18782,"Ġfactions":18783,"Ġvide":18784,"Ġtattoo":18785,"Ġfinancially":18786,"Ġfatigue":18787,"Ġproceeding":18788,"constitutional":18789,"Ġmiser":18790,"Ġchairs":18791,"gging":18792,"ipple":18793,"Ġdent":18794,"Ġdisreg":18795,"çĶ":18796,"stant":18797,"llo":18798,"bps":18799,"akening":18800,"Ġabnormal":18801,"ĠERA":18802,"士":18803,"ĠHBO":18804,"ĠMAR":18805,"Ġconcess":18806,"Ġservant":18807,"Ġaspir":18808,"lav":18809,"ĠPanel":18810,"amo":18811,"Ġprecip":18812,"Ġrecordings":18813,"Ġproceeded":18814,"Ġcolony":18815,"ĠTang":18816,"ablo":18817,"Ġstripped":18818,"Left":18819,"too":18820,"Ġpotatoes":18821,"Ġfinest":18822,"%).":18823,"Ġcrap":18824,"ĠZach":18825,"abases":18826,"ĠGoth":18827,"Ġbillionaire":18828,"wolf":18829,"Ġsanction":18830,"SK":18831,"Ġlogged":18832,"Po":18833,"eyed":18834,"unal":18835,"Ġcricket":18836,"Ġarmies":18837,"Ġuncovered":18838,"Cloud":18839,"ón":18840,"Ġrebounds":18841,"Ġmes":18842,"Oper":18843,"Pac":18844,"Ġnationally":18845,"Ġinserted":18846,"pict":18847,"Ġgovernance":18848,"и":18849,"Ġprivileges":18850,"GET":18851,"Ġfavorites":18852,"imity":18853,"Ġlover":18854,"them":18855,"empl":18856,"Ġgorgeous":18857,"Ann":18858,"Ġslipped":18859,"Ġveto":18860,"Bob":18861,"Ġslim":18862,"ucc":18863,"ĠFame":18864,"uddenly":18865,"Ġdenies":18866,"ĠMaur":18867,"Ġdistances":18868,"Ġwanna":18869,"tar":18870,"ĠSER":18871,"ĠâĪ":18872,"Ġlemon":18873,"athetic":18874,"Ġliteral":18875,"Ġdistinguished":18876,"Ġanswering":18877,"GI":18878,"Ġreligions":18879,"ĠPhilos":18880,"ĠLay":18881,"Ġcompos":18882,"irements":18883,"ĠKos":18884,"inez":18885,"rolling":18886,"Ġyoungest":18887,"andise":18888,"ĠBorn":18889,"Ġaltar":18890,"amina":18891,"ĠBoot":18892,"voc":18893,"Ġdigging":18894,"Ġpressures":18895,"Ġlen":18896,"264":18897,"Ġassassination":18898,"ĠBirmingham":18899,"ĠMyth":18900,"Ġsovereign":18901,"ĠArtist":18902,"ĠPhotograph":18903,"Ġdepicted":18904,"Ġdispens":18905,"orthy":18906,"Ġambul":18907,"integ":18908,"ĠCele":18909,"ĠTibet":18910,"Ġhierarchy":18911,"Ġcu":18912,"Ġpreseason":18913,"ĠPeterson":18914,"Ġcolours":18915,"Ġworrying":18916,"Ġbackers":18917,"ĠPalmer":18918,"Ġμ":18919,"Ġcontributor":18920,"Ġhearings":18921,"Ġurine":18922,"ĠÙ":18923,"ourgeois":18924,"Similar":18925,"ĠZimmer":18926,"something":18927,"ĠUSC":18928,"Ġstrengths":18929,"ĠFI":18930,"Ġlogging":18931,"Asked":18932,"ĠThai":18933,"inqu":18934,"ĠWalt":18935,"Ġcrews":18936,"itism":18937,"301":18938,"Ġsharply":18939,"umed":18940,"Ġredirect":18941,"rators":18942,"Inf":18943,"ĠWeapons":18944,"Ġteasp":18945,"1999":18946,"Live":18947,"ĠEspecially":18948,"ĠSter":18949,"ĠVeterans":18950,"Ġintro":18951,"otherapy":18952,"Ġmalware":18953,"Ġbreeding":18954,"Ġmolecular":18955,"ĠRoute":18956,"ĠComment":18957,"ochem":18958,"Ġain":18959,"Season":18960,"Ġlinebacker":18961,"Ä«":18962,"ĠEconomics":18963,"esar":18964,"ĠLives":18965,"ĠEmma":18966,"Ġkin":18967,"ĠTerrit":18968,"Ġplanted":18969,"oton":18970,"ĠButter":18971,"ĠSpons":18972,"PER":18973,"Ġdungeon":18974,"Ġsymbolic":18975,"Ġfilmed":18976,"Ġdiets":18977,"Ġconcludes":18978,"Ġcertainty":18979,"ĠFormat":18980,"Ġstrangers":18981,"format":18982,"ĠPhase":18983,"Ġcopied":18984,"Ġmetres":18985,"lda":18986,"ĠUsers":18987,"Ġdeliberate":18988,"Ġwashed":18989,"ĠLance":18990,"imation":18991,"Ġimproper":18992,"ĠGenesis":18993,"ickr":18994,"ĠKush":18995,"Ġrealise":18996,"Ġembarrassing":18997,"alking":18998,"bucks":18999,"Ġverified":19000,"Ġoutline":19001,"years":19002,"ĠIncome":19003,"202":19004,"Ġzombies":19005,"Final":19006,"ĠMillenn":19007,"Ġmodifications":19008,"ĠVision":19009,"ĠMoses":19010,"verb":19011,"iterranean":19012,"ĠJet":19013,"Ġnaval":19014,"ĠAgg":19015,"Ġurl":19016,"Ġvictories":19017,"Ġnonetheless":19018,"Ġinjust":19019,"ĠFact":19020,"çļ":19021,"Ġinsufficient":19022,"review":19023,"facebook":19024,"Ġnegotiating":19025,"Ġguarantees":19026,"imen":19027,"utenberg":19028,"Ġgambling":19029,"Ġcongr":19030,"Loading":19031,"Ġnevertheless":19032,"Ġpresidents":19033,"ĠIndustrial":19034,"Ġ118":19035,"Ġpoured":19036,"ĠTory":19037,"Ġ175":19038,"Ġ:=":19039,"Scott":19040,"angered":19041,"Tok":19042,"Ġorganizers":19043,"Mat":19044,"ĠGrowth":19045,"Ġadul":19046,"Ġensures":19047,"Ġ117":19048,"é¾įå":19049,"Ġmassacre":19050,"Ġgrades":19051,"before":19052,"ADVERTISEMENT":19053,"ĠSlow":19054,"ĠMMA":19055,"âĢĶ\"":19056,"ĠVatican":19057,"Qaeda":19058,"Ġowe":19059,"6666":19060,"ĠSorry":19061,"ĠGrass":19062,"Ġbackgrounds":19063,"Ġexhausted":19064,"Ġclan":19065,"Ġcompromised":19066,"ĠElf":19067,"ĠIsaac":19068,"enson":19069,"Invest":19070,"IFA":19071,"Ġinterrupted":19072,"ãĥīãĥ©":19073,"Ġtwisted":19074,"ĠDragons":19075,"Mode":19076,"ĠKremlin":19077,"Ġfertil":19078,"heres":19079,"phan":19080,"ĠNode":19081,"fed":19082,"ĠOrc":19083,"Ġunwilling":19084,"Cent":19085,"Ġpriorit":19086,"Ġgraduates":19087,"Ġsubjective":19088,"Ġissuing":19089,"ĠLt":19090,"Ġviewer":19091,"Ġwoke":19092,"Thus":19093,"brook":19094,"Ġdepressed":19095,"Ġbracket":19096,"ĠGor":19097,"ĠFighting":19098,"Ġstriker":19099,"Report":19100,"ĠPortugal":19101,"Ġneo":19102,"wed":19103,"199":19104,"Ġfleeing":19105,"shadow":19106,"identified":19107,"USE":19108,"Steam":19109,"Ġstretched":19110,"Ġrevelations":19111,"arted":19112,"ĠDw":19113,"Ġalignment":19114,"eston":19115,"ĠJared":19116,"Sep":19117,"Ġblogs":19118,"update":19119,"gom":19120,"risk":19121,"Ġclash":19122,"ĠHour":19123,"Ġruntime":19124,"Ġunwanted":19125,"Ġscam":19126,"Ġrack":19127,"Ġenlight":19128,"onest":19129,"ĠFerr":19130,"Ġconvictions":19131,"Ġpiano":19132,"Ġcirculation":19133,"ĠWelcome":19134,"Ġbacklash":19135,"ĠWade":19136,"Ġreceivers":19137,"otive":19138,"Jeff":19139,"Ġnetworking":19140,"ĠPrep":19141,"ĠExplorer":19142,"Ġlecture":19143,"Ġuploaded":19144,"ĠMeat":19145,"BLE":19146,"ĠNazis":19147,"ĠSynd":19148,"stud":19149,"roots":19150,"rians":19151,"Ġportrayed":19152,"Ġ??":19153,"ĠBuddha":19154,"sun":19155,"Robert":19156,"ĠComplex":19157,"Ġoversee":19158,"Ġstealth":19159,"Title":19160,"ĠJobs":19161,"ĠKum":19162,"Ġappreciation":19163,"ĠMOD":19164,"Ġbasics":19165,"Ġclips":19166,"Ġnursing":19167,"Ġproposition":19168,"Ġrealised":19169,"ĠNYC":19170,"Ġallocated":19171,"rium":19172,"aran":19173,"ĠProduction":19174,"ĠVote":19175,"Ġsmugg":19176,"Ġhunter":19177,"azer":19178,"ĠChanges":19179,"Ġfluct":19180,"yon":19181,"Array":19182,"Ġkits":19183,"Water":19184,"Ġuncommon":19185,"Ġresting":19186,"ells":19187,"would":19188,"Ġpursued":19189,"Ġassertion":19190,"ometown":19191,"ĠMosul":19192,"ĠPlatform":19193,"iolet":19194,"Ġshareholders":19195,"Ġtrails":19196,"Pay":19197,"ĠEnforcement":19198,"types":19199,"ĠAnonymous":19200,"Ġsatisfying":19201,"ilogy":19202,"Ġ('":19203,"wave":19204,"city":19205,"Steve":19206,"Ġconfrontation":19207,"ĠEld":19208,"Capt":19209,"ahan":19210,"htm":19211,"ĠCtrl":19212,"ONS":19213,"230":19214,"ifa":19215,"holding":19216,"Ġdelicate":19217,"Ġjaw":19218,"ĠGoing":19219,"orum":19220,"Sal":19221,"Ġdull":19222,"ĠBeth":19223,"Ġprisons":19224,"Ġego":19225,"ĠElsa":19226,"avorite":19227,"ĠGang":19228,"ĠNuclear":19229,"Ġspider":19230,"atsu":19231,"Ġsampling":19232,"Ġabsorbed":19233,"ĠPharm":19234,"ieth":19235,"Ġbucket":19236,"ĠRecomm":19237,"OF":19238,"ĠFactory":19239,"ANCE":19240,"Ġbacter":19241,"Has":19242,"ĠObserv":19243,"121":19244,"Ġpremiere":19245,"Develop":19246,"Ġcurrencies":19247,"Cast":19248,"Ġaccompanying":19249,"ĠNashville":19250,"Ġfatty":19251,"ĠBrend":19252,"Ġlocks":19253,"Ġcentered":19254,"ĠUT":19255,"aughs":19256,"orie":19257,"ĠAffordable":19258,"vance":19259,"DL":19260,"emet":19261,"Ġthrone":19262,"ĠBluetooth":19263,"Ġnaming":19264,"ifts":19265,"ADE":19266,"Ġcorrected":19267,"Ġpromptly":19268,"ĠSTR":19269,"Ġgenome":19270,"Ġcope":19271,"Ġvalley":19272,"Ġrounded":19273,"ĠKend":19274,"alion":19275,"pers":19276,"Ġtourism":19277,"Ġstark":19278,"vl":19279,"Ġblowing":19280,"ĠSchedule":19281,"std":19282,"Ġunhappy":19283,"Ġlitigation":19284,"cedes":19285,"Ġandroid":19286,"Ġintegral":19287,"erers":19288,"uded":19289,"tax":19290,"Ġreiter":19291,"ĠMotors":19292,"ociated":19293,"Ġwonders":19294,"ĠApost":19295,"ucking":19296,"ĠRoosevelt":19297,"fram":19298,"Ġyields":19299,"Ġconstitutes":19300,"awk":19301,"Interest":19302,"Ġinterim":19303,"Ġbreakthrough":19304,"ĠCher":19305,"Ġprosec":19306,"ĠDj":19307,"ĠMT":19308,"Resp":19309,"ĠPT":19310,"Ġsperm":19311,"edit":19312,"BT":19313,"Linux":19314,"country":19315,"league":19316,"Ġdick":19317,"Ġoct":19318,"Ġinserting":19319,"Ġscra":19320,"ĠBrewing":19321,"Ġ1966":19322,"Ġrunners":19323,"Ġplun":19324,"idy":19325,"ĠDian":19326,"Ġdysfunction":19327,"Ġexclusion":19328,"Ġdisgr":19329,"Ġincorporate":19330,"Ġreconc":19331,"Ġnominated":19332,"ĠArcher":19333,"draw":19334,"achelor":19335,"Ġwritings":19336,"Ġshallow":19337,"Ġhast":19338,"ĠBMW":19339,"ĠRS":19340,"Ġthigh":19341,"Ġ1963":19342,"Ġlamb":19343,"Ġfavored":19344,"agle":19345,"Ġcooler":19346,"ĠHours":19347,"ĠGU":19348,"ĠOrigin":19349,"Ġglimpse":19350,"--------------------":19351,"Lim":19352,"Ġcheek":19353,"Ġjealous":19354,"-'":19355,"Ġharness":19356,"ĠPoison":19357,"Ġdisabilities":19358,"neapolis":19359,"Ġoutlook":19360,"Ġnotify":19361,"ĠIndianapolis":19362,"Ġabrupt":19363,"nsic":19364,"Ġencrypted":19365,"Ġforfe":19366,"reath":19367,"Ġrabb":19368,"Ġfoundations":19369,"Ġcompliment":19370,"ĠInterview":19371,"ĠSwe":19372,"Ġadolesc":19373,"Ġmonitors":19374,"ĠSacramento":19375,"Ġtimely":19376,"Ġcontempl":19377,"Ġpositioned":19378,"Ġposters":19379,"phies":19380,"iovascular":19381,"void":19382,"ĠFifth":19383,"Ġinvestigative":19384,"OUN":19385,"Ġintegrate":19386,"ĠINC":19387,"isha":19388,"iblings":19389,"ĠRequest":19390,"ĠRodriguez":19391,"Ġslides":19392,"ĠDX":19393,"Ġfeminism":19394,"Ġdatas":19395,"Ġbend":19396,"irus":19397,"ĠNigeria":19398,"Fox":19399,"Change":19400,"Ġairplane":19401,"ĠLaden":19402,"Ġpublicity":19403,"ixty":19404,"Ġcommitments":19405,"Ġaggregate":19406,"Ġdisplaying":19407,"ĠArrow":19408,"Ġ122":19409,"Ġrespects":19410,"android":19411,"six":19412,"ĠSha":19413,"Ġrestoration":19414,")\\":19415,"WS":19416,"oys":19417,"Ġillustrate":19418,"without":19419,"126":19420,"ĠâĶĤ":19421,"Ġpickup":19422,"nels":19423,"Ġ....":19424,"food":19425,"ĠFen":19426,")?":19427,"Ġphenomena":19428,"Ġcompanions":19429,"ĠWrite":19430,"Ġspill":19431,"Ġbridges":19432,"ĠUpdated":19433,"ĠFo":19434,"Ġinsects":19435,"ASHINGTON":19436,"Ġscare":19437,"iltr":19438,"ĠZhang":19439,"Ġseverity":19440,"Ġindul":19441,"149":19442,"ĠCoffee":19443,"Ġnorms":19444,"Ġpulse":19445,"ĠFT":19446,"Ġhorrific":19447,"ĠDestroy":19448,"ĠJSON":19449,"Ġolive":19450,"Ġdiscusses":19451,"Rest":19452,"Elect":19453,"ĠWinn":19454,"ĠSurviv":19455,"ĠHait":19456,"Sure":19457,"oped":19458,"Ġrooted":19459,"ĠSke":19460,"ĠBronze":19461,"Ġlol":19462,"Default":19463,"Ġcommodity":19464,"redited":19465,"Ġlibertarian":19466,"Ġforbidden":19467,"Ġgran":19468,"à¨":19469,"Ġlag":19470,"enz":19471,"drive":19472,"Ġmathematics":19473,"Ġwires":19474,"Ġcritically":19475,"Ġcarbohyd":19476,"ĠChancellor":19477,"ĠEddie":19478,"Ġbanning":19479,"ĠFri":19480,"Ġcomplications":19481,"etric":19482,"ĠBangladesh":19483,"Ġbandwidth":19484,"Stop":19485,"ĠOriginally":19486,"Ġhalfway":19487,"ynasty":19488,"shine":19489,"Ġtales":19490,"rities":19491,"avier":19492,"Ġspinning":19493,"ĠWHO":19494,"Ġneighbourhood":19495,"bach":19496,"Ġcommerce":19497,"ĠSle":19498,"BU":19499,"Ġentrepreneur":19500,"Ġpeculiar":19501,"ĠComments":19502,"fre":19503,"320":19504,"ICS":19505,"Ġimagery":19506,"ĠCanon":19507,"ĠElectronic":19508,"short":19509,"((":19510,"Dig":19511,"Ġcommem":19512,"uced":19513,"Ġinclined":19514,"ĠSummon":19515,"Ġcliff":19516,"ĠMediterranean":19517,"Ġpoetry":19518,"Ġprosperity":19519,"ĠRece":19520,"Ġpills":19521,"member":19522,"Ġfinale":19523,"unc":19524,"ĠGig":19525,"ä½":19526,"Ġlod":19527,"Ġbackward":19528,"-+":19529,"ĠForward":19530,"Ġthri":19531,"sure":19532,"Ġsoap":19533,"ĠFX":19534,"RES":19535,"ĠSexual":19536,"oulos":19537,"Ġfoolish":19538,"Ġrighteous":19539,"Ġcoff":19540,"terrorism":19541,"ustain":19542,"oter":19543,"Ġabuses":19544,"next":19545,"Ġabusive":19546,"Ġthereafter":19547,"Ġprohibition":19548,"ĠSUP":19549,"Ġdip":19550,"Ġripped":19551,"Ġinherited":19552,"Ġbats":19553,"stru":19554,"GT":19555,"Ġflawed":19556,"phabet":19557,"Ġfog":19558,"doors":19559,"Ġimaging":19560,"Ġdigits":19561,"ĠHungary":19562,"Ġarrog":19563,"Ġteachings":19564,"Ġprotocols":19565,"ĠBanks":19566,"à¸":19567,"pound":19568,"ĠCurt":19569,".\")":19570,"./":19571,"Ġexemption":19572,"endix":19573,"ĠMull":19574,"Ġimproves":19575,"ĠGamer":19576,"dimensional":19577,"Icon":19578,"ĠMargaret":19579,"Status":19580,"dates":19581,"Ġintends":19582,"Ġdepict":19583,"Ġparked":19584,"Joe":19585,"ĠMarines":19586,"chnology":19587,"!).":19588,"Ġjudged":19589,"Ġweights":19590,"Ray":19591,"Ġapartments":19592,"hester":19593,"Ġreinforce":19594,"Ġoffender":19595,"occup":19596,"Ġsore":19597,"ept":19598,"ĠPHP":19599,"ĠBrow":19600,"Ġauthorization":19601,"ĠRisk":19602,"ĠDelaware":19603,"ĠQU":19604,"Ġnotifications":19605,"Ġsunlight":19606,"Ġexclude":19607,"dat":19608,"Ġmesh":19609,"ĠSudan":19610,"Ġbelonged":19611,"Ġsubway":19612,"Ġnoon":19613,"ĠInterior":19614,"olics":19615,"ĠLakers":19616,"Ġcoding":19617,"Disclaimer":19618,"Calif":19619,"Old":19620,"Ġdisl":19621,"?????":19622,"Ġconfirms":19623,"Ġrecruitment":19624,"Ġhomicide":19625,"Consider":19626,"ĠJeffrey":19627,"fty":19628,"};":19629,"Ġobjection":19630,"doing":19631,"ĠLeo":19632,"Want":19633,"Ġglow":19634,"ĠClarke":19635,"ĠNorman":19636,"Ġverification":19637,"Ġpacket":19638,"ĠFormula":19639,"Ġplag":19640,"esville":19641,"Ġshouting":19642,"Ġov":19643,"ĠREC":19644,"ĠBub":19645,"Ġninth":19646,"Ġenerg":19647,"Ġvalidity":19648,"Ġups":19649,"jack":19650,"Ġneighboring":19651,"ĠNec":19652,"eworks":19653,"ĠHab":19654,"arez":19655,"Ġspine":19656,"Ġeventual":19657,"ĠLeaders":19658,"ĠCarn":19659,"Ġprobation":19660,"Ġromance":19661,"msg":19662,"ĠMechanical":19663,"ERY":19664,"Rock":19665,"Ġpartisan":19666,"Node":19667,"assets":19668,"minent":19669,"Ġforeigners":19670,"Ġtestify":19671,"ĠUsually":19672,"lords":19673,"ĠGren":19674,"ĠPowell":19675,"BIL":19676,"Ġsr":19677,"Ġaddict":19678,"Ġshells":19679,"Ġsigh":19680,"ĠYale":19681,"ternity":19682,"Ġ750":19683,"EU":19684,"ĠRifle":19685,"Ġpatron":19686,"ema":19687,"ĠBannon":19688,"anity":19689,"Ġtropical":19690,"ĠVII":19691,"cross":19692,"Everything":19693,"ĠISO":19694,"Ġhumble":19695,"assing":19696,"ĠFIG":19697,"Ġupdating":19698,"yson":19699,"Ġcalcium":19700,"Ġcompetent":19701,"Ġsteering":19702,"Prot":19703,"ĠSY":19704,"ĠFinals":19705,"ĠRug":19706,"159":19707,"137":19708,"ĠGolf":19709,"Ġ126":19710,"Ġaccommodation":19711,"ĠHughes":19712,"Ġaesthetic":19713,"artisan":19714,"ĠTwilight":19715,"Ġprince":19716,"ĠAgriculture":19717,"ĠDisco":19718,"Ġprecedent":19719,"Ġtyping":19720,"authorized":19721,"Option":19722,"ĠAub":19723,"lishes":19724,"acht":19725,"mag":19726,"Peter":19727,"ĠUFO":19728,"monton":19729,"ĠLith":19730,"Ġarom":19731,"Ġsecuring":19732,"Ġconfined":19733,"private":19734,"Ġswords":19735,"Ġmarkers":19736,"Ġmetabolic":19737,"select":19738,"ĠCurse":19739,"ĠOt":19740,"gressive":19741,"Ġincumb":19742,"ĠSaga":19743,"Ġpriced":19744,"Ġclearance":19745,"Content":19746,"Ġdrilling":19747,"Ġnotices":19748,"Ġbourgeois":19749,"Ġvest":19750,"Ġcookie":19751,"ĠGuardians":19752,"rys":19753,"inyl":19754,"Ġ124":19755,"Ġplausible":19756,"ongh":19757,"ĠOdin":19758,"Ġconception":19759,"ĠYuk":19760,"ĠBaghdad":19761,"ĠFlag":19762,"Austral":19763,"ĠIBM":19764,"Ġinternationally":19765,"ĠWikiLeaks":19766,"IED":19767,"Ġcyn":19768,"Ġchooses":19769,"ĠPill":19770,"Ġcombining":19771,"Ġradi":19772,"ĠMohammed":19773,"defense":19774,"atching":19775,"Subject":19776,"iciency":19777,"Frame":19778,"Ġ{\"":19779,"Ġchess":19780,"Ġtimer":19781,"190":19782,"Ġtin":19783,"Ġordinance":19784,"emetery":19785,"Ġaccusing":19786,"Ġnoticeable":19787,"Ġcentres":19788,"Ġlid":19789,"ĠMills":19790,"imgur":19791,"Ġzoom":19792,"ergic":19793,"Ġcompression":19794,"prim":19795,"find":19796,"Ġsurg":19797,"Ġpand":19798,"ĠKee":19799,"ĠChad":19800,"cellence":19801,"oyle":19802,"Ġsocialism":19803,"ĠTravis":19804,"ĠMHz":19805,"Ġguild":19806,"ALLY":19807,"ĠSubscribe":19808,"ĠRelated":19809,"Ġoccurrence":19810,"itching":19811,"Ġfictional":19812,"Ġcrush":19813,"ĠEA":19814,"cod":19815,"mix":19816,"ĠTriple":19817,"Ġretrieve":19818,"Ġstimulus":19819,"Ġpsychiat":19820,"ĠDoor":19821,"Ġhomosexuality":19822,"Ġelementary":19823,"Ġcellular":19824,"idian":19825,"ĠLaun":19826,"Ġintriguing":19827,"Ġfoam":19828,"ĠBass":19829,"idi":19830,"itsu":19831,"Ġassure":19832,"Ġcongrat":19833,"Ġbusinessman":19834,"ĠBoost":19835,"close":19836,"Ġlied":19837,"Ġsciences":19838,"ĠOmega":19839,"ĠGraphics":19840,"Ġ<=":19841,"spoken":19842,"Ġconnectivity":19843,"Saturday":19844,"ĠAvengers":19845,"Ġtoggle":19846,"Ġankle":19847,"Ġnationalist":19848,"model":19849,"ĠPool":19850,"ophobia":19851,"Var":19852,"ĠMons":19853,"atories":19854,"Ġaggressively":19855,"Clear":19856,"Forge":19857,"acters":19858,"Ġhedge":19859,"Ġpipes":19860,"Ġblunt":19861,"Ġsq":19862,"Ġremotely":19863,"Wed":19864,"asers":19865,"Ġrefriger":19866,"Ġtiles":19867,"Ġrescued":19868,"Ġcomprised":19869,"insky":19870,"Ġmanif":19871,"avanaugh":19872,"Ġprolifer":19873,"Ġaligned":19874,"xml":19875,"Ġtriv":19876,"Ġcoordination":19877,"ĠPER":19878,"ĠQuote":19879,"134":19880,"bf":19881,"ĠSaw":19882,"Ġtermination":19883,"Ġ190":19884,"Ġadditions":19885,"Ġtrio":19886,"Ġprojections":19887,"Ġpositively":19888,"Ġinclusive":19889,"Ġmembr":19890,"1990":19891,"older":19892,"Ġpracticed":19893,"inkle":19894,"Arch":19895,"Ġstarters":19896,"arius":19897,"Ġintermediate":19898,"ĠBenef":19899,"ĠKiller":19900,"Ġinterventions":19901,"ĠKil":19902,"ĠFlying":19903,"Inv":19904,"Ġpremature":19905,"Ġpsychiatric":19906,"Ġindie":19907,"Ġcollar":19908,"ĠRainbow":19909,"afi":19910,"Ġdisruption":19911,"ĠFOX":19912,"casting":19913,"Ġmisdem":19914,"cro":19915,"Ġwipe":19916,"ardon":19917,"Ġbast":19918,"ĠTommy":19919,"ĠRepresentative":19920,"Ġbelly":19921,"ĠPO":19922,"ĠBreitbart":19923,"132":19924,"Ġmessaging":19925,"Should":19926,"References":19927,"ĠGRE":19928,"istical":19929,"LP":19930,"ĠCav":19931,"ĠCrazy":19932,"Ġintuitive":19933,"keeping":19934,"ĠMoss":19935,"Ġdiscontin":19936,"ĠModule":19937,"Ġunrelated":19938,"ĠPractice":19939,"ĠTransport":19940,"Ġstatistically":19941,"orns":19942,"Ġsized":19943,"pu":19944,"Ġcaf":19945,"ĠWorlds":19946,"ĠRodgers":19947,"ĠLun":19948,"ĠComic":19949,"living":19950,"Ġcared":19951,"Ġclimbed":19952,"){":19953,"Ġconsisted":19954,"Ġmedieval":19955,"folk":19956,"Ġhacked":19957,"Ġdire":19958,"ĠHermione":19959,"Ġtended":19960,"ceans":19961,"Daniel":19962,"went":19963,"Ġlegislators":19964,"Ġredes":19965,"games":19966,"Ġgn":19967,"amiliar":19968,"Ġ++":19969,"ggy":19970,"threat":19971,"Ġmagnet":19972,"Ġperceive":19973,"Ġzip":19974,"Ġindictment":19975,"Ġcritique":19976,"gard":19977,"ĠSafe":19978,"ĠCream":19979,"Ġadvent":19980,"oba":19981,"Ġvowed":19982,"ousands":19983,"Ġski":19984,"Ġabortions":19985,"uart":19986,"Ġstunned":19987,"Ġadvancing":19988,"Ġlacked":19989,"Ġ\\\"":19990,"Ġschizophren":19991,"Ġelegant":19992,"Ġconferences":19993,"Ġcanceled":19994,"ĠHudson":19995,"ĠHopefully":19996,"Ġtrump":19997,"Ġfrequencies":19998,"Ġmeteor":19999,"ĠJunior":20000,"ĠFleet":20001,"ĠMalcolm":20002,"ĠTools":20003,"Ġ........":20004,"Ġhobby":20005,"ĠEuropeans":20006,"Ġ1500":20007,"ĠInto":20008,"Ġsway":20009,"ĠAppro":20010,"ĠCompl":20011,"Community":20012,"Ġtide":20013,"ĠSummit":20014,"ä»":20015,"Ġintervals":20016,"ĠEther":20017,"Ġhabitat":20018,"ĠStevens":20019,"lishing":20020,"ĠDomain":20021,"Ġtriggers":20022,"Ġchasing":20023,"Ġcharm":20024,"ĠFlower":20025,"itored":20026,"Ġblessing":20027,"Ġtextures":20028,"Five":20029,"Ġliquor":20030,"RP":20031,"FIN":20032,"Ġ1962":20033,"CAR":20034,"Unknown":20035,"Ġresil":20036,"ĠLily":20037,"Ġabundance":20038,"Ġpredictable":20039,"rar":20040,"Ġbullshit":20041,"leen":20042,"chet":20043,"Mor":20044,"Much":20045,"ä¹":20046,"Ġemphasized":20047,"Ġcrust":20048,"Ġprimitive":20049,"Ġenjoyable":20050,"ĠPictures":20051,"Ġteammate":20052,"pler":20053,"ĠTol":20054,"ĠKane":20055,"Ġsummoned":20056,"thy":20057,"rama":20058,"ĠHonda":20059,"Ġrealizing":20060,"Ġquicker":20061,"Ġconcentrate":20062,"clear":20063,"Ġ210":20064,"ĠErdogan":20065,"aris":20066,"Ġresponds":20067,"ĠBI":20068,"Ġeligibility":20069,"Ġpushes":20070,"ĠIdaho":20071,"Ġaggrav":20072,"Ġruins":20073,"urations":20074,"Ġbans":20075,"Ġanat":20076,"share":20077,"Ġgrind":20078,"hin":20079,"umen":20080,"Ġutilities":20081,"ĠYankees":20082,"Ġdatabases":20083,"ĠDD":20084,"Ġdisplaced":20085,"Ġdependencies":20086,"Ġstimulation":20087,"hun":20088,"houses":20089,"ĠPretty":20090,"ĠRavens":20091,"ĠTODAY":20092,"Ġassociates":20093,"Ġtherape":20094,"cled":20095,"Ġdeer":20096,"Ġrepairs":20097,"rentice":20098,"Ġreceptors":20099,"Ġremed":20100,"ĠCe":20101,"Ġmarriages":20102,"Ġballots":20103,"ĠSoldier":20104,"Ġhilarious":20105,"opl":20106,"138":20107,"Ġinherently":20108,"Ġignorant":20109,"Ġbounce":20110,"ĠEaster":20111,"RELATED":20112,"ĠCurrency":20113,"EV":20114,"ãĥŀ":20115,"ĠLead":20116,"Ġdeceased":20117,"Brien":20118,"ĠMusk":20119,"JS":20120,"Ġmerge":20121,"hearted":20122,"creat":20123,"mitt":20124,"mund":20125,"ĠâĢĭ":20126,"ĠBag":20127,"Ġprojection":20128,"Ġjava":20129,"ĠStandards":20130,"ĠLeonard":20131,"Ġcoconut":20132,"ĠPopulation":20133,"Ġtraject":20134,"Ġimply":20135,"Ġcuriosity":20136,"ĠDB":20137,"ĠFresh":20138,"ĠPor":20139,"Ġheavier":20140,"neys":20141,"gomery":20142,"Ġdeserved":20143,"Ġphrases":20144,"ĠGC":20145,"Ġyeast":20146,"desc":20147,"Death":20148,"Ġreboot":20149,"Ġmetadata":20150,"ICAL":20151,"Ġrepay":20152,"ĠIndependence":20153,"Ġsuburban":20154,"icals":20155,"Ġatop":20156,"Ġallocation":20157,"generation":20158,"ĠGram":20159,"Ġmoisture":20160,"Ġpine":20161,"ĠLiberals":20162,"Ġaides":20163,"Ġunderest":20164,"ĠBerry":20165,"Ġceremon":20166,"370":20167,"astrous":20168,"ĠPirates":20169,"Ġtense":20170,"ĠIndustries":20171,"ĠAppeals":20172,"ĠNear":20173,"Ġè£ıç":20174,"Ġlovers":20175,"ĠCAP":20176,"ĠCraw":20177,"Ġgiants":20178,"Ġefficacy":20179,"Element":20180,"ĠBehavior":20181,"ĠToyota":20182,"Ġintest":20183,"Priv":20184,"AI":20185,"Ġmaneuver":20186,"Ġperfection":20187,"Ġbang":20188,"paper":20189,"rill":20190,"George":20191,"border":20192,"inters":20193,"ĠSeth":20194,"Ġclues":20195,"ĠLevi":20196,"ĠRevenue":20197,"147":20198,"Ġvapor":20199,"Ġfortunate":20200,"Ġthreatens":20201,"Ġvet":20202,"Ġdependency":20203,"ersed":20204,"article":20205,"ĠBlizzard":20206,"Ġchlor":20207,"Ġminus":20208,"ĠBills":20209,"Ġcryptocurrency":20210,"Ġmetabolism":20211,"tering":20212,"Ġpestic":20213,"steps":20214,"ĠTreasure":20215,"racted":20216,"ĠConstant":20217,"Ġtemp":20218,"139":20219,"ĠDetective":20220,"urally":20221,"Ġrecovering":20222,"Ġcortex":20223,"Ġ144":20224,"closed":20225,"Ġprejudice":20226,"aunted":20227,"Ġstorms":20228,"ĠNOW":20229,"Ġmachinery":20230,"Address":20231,"Ġcompelled":20232,"270":20233,"Ġdespair":20234,"bane":20235,"Ġvegetable":20236,"Ġbeds":20237,"Learn":20238,"Ġcolorful":20239,"Ġspike":20240,"Ġmargins":20241,"Ġsympathy":20242,"Ġworkshop":20243,"ĠCBC":20244,"Sat":20245,"Ġburns":20246,"ĠGender":20247,"Ġ129":20248,"ĠCable":20249,"Ġdebts":20250,"ĠTheresa":20251,"Ġreflecting":20252,"Ġairst":20253,"Ġrim":20254,"ramid":20255,"Ġweaknesses":20256,"Writ":20257,"oggle":20258,"ti":20259,"ĠCharge":20260,"Ġweighed":20261,"Ġ(.":20262,"Ġlaughter":20263,"Ġrouter":20264,"ĠDemocracy":20265,"Dear":20266,"Ġhasht":20267,"Ġdy":20268,"Ġhints":20269,"running":20270,"Ġfinishes":20271,"arus":20272,"Mass":20273,"result":20274,"ascus":20275,"Ġvintage":20276,"Ġconqu":20277,"Ġwildly":20278,"acist":20279,"Ġlingu":20280,"Ġprotagonist":20281,"strom":20282,"teenth":20283,"ĠSolo":20284,"mac":20285,"filled":20286,"Ġrenown":20287,"itives":20288,"Ġmotive":20289,"ĠAntar":20290,"ĠMann":20291,"ĠAdjust":20292,"Ġrockets":20293,"Ġtroubling":20294,"ei":20295,"Ġorganisms":20296,"assis":20297,"Christian":20298,"Ġ145":20299,"ĠHass":20300,"Ġswall":20301,"Ġwax":20302,"ĠSurvival":20303,"VS":20304,"ĠMurd":20305,"vd":20306,"standard":20307,"Ġdragons":20308,"Ġacceleration":20309,"rational":20310,"final":20311,"Ġpaired":20312,"ĠEthereum":20313,"Ġinterfaces":20314,"Ġresent":20315,"Ġartifacts":20316,"Å«":20317,"arel":20318,"Ġcompetitor":20319,"ĠNicholas":20320,"ĠSurface":20321,"cpp":20322,"ĠTot":20323,"Ġeconomically":20324,"Ġorganised":20325,"Ġenforced":20326,"inho":20327,"Ġvarieties":20328,"Ġabdom":20329,"ĠBailey":20330,"idav":20331,"ĠSalv":20332,"paid":20333,"Ġaltitude":20334,"essert":20335,"ĠGutenberg":20336,"area":20337,"opoulos":20338,"Ġprofessors":20339,"iggs":20340,"ĠFate":20341,"hey":20342,"Ġ3000":20343,"Dist":20344,"Ġtwins":20345,"cill":20346,"ĠMaps":20347,"Ġtraps":20348,"Ġweed":20349,"ĠKiss":20350,"Ġyoga":20351,"Ġrecipients":20352,"ĠWestminster":20353,"Ġpools":20354,"ĠWalmart":20355,"188":20356,"ĠSchools":20357,"attack":20358,"ĠARM":20359,"paragraph":20360,"Warning":20361,"jl":20362,"Ġselfish":20363,"anchez":20364,"ĠHeights":20365,"Fre":20366,"ĠSoph":20367,"Ġ--------------------------------":20368,"tml":20369,"333":20370,"Ġraids":20371,"Ġsatellites":20372,"KEY":20373,"Ġlasts":20374,"ÑĤ":20375,"Ins":20376,"ĠDame":20377,"Ġunpredict":20378,"///":20379,"ghai":20380,"Ġartillery":20381,"Ġcruise":20382,"Ġgel":20383,"ĠCabinet":20384,"Ġblows":20385,"ĠEsp":20386,"Ġproximity":20387,"othe":20388,"ĠSkills":20389,"ĠUpper":20390,"obo":20391,"ĠNDP":20392,"Ġenjoys":20393,"Ġrepeating":20394,"ĠConstruction":20395,"ĠQuestions":20396,"Hillary":20397,"Ġuint":20398,"Ġprocessors":20399,"ĠGibson":20400,"ĠMultiple":20401,"qa":20402,"ĠBom":20403,"ĠMiles":20404,"ventional":20405,"Ġhurts":20406,"skin":20407,"ĠAIDS":20408,"Ġadvisers":20409,"ĠRoot":20410,"Ġmethodology":20411,"ĠDale":20412,"Ġdeton":20413,"ĠKnowledge":20414,"sequently":20415,"Ġ121":20416,"Ġconnects":20417,"Cy":20418,"ĠDanger":20419,"Ġcontributors":20420,"ĠBent":20421,"Ġbrass":20422,"ĠGuns":20423,"into":20424,"ĠFortune":20425,"Ġbroker":20426,"balance":20427,"Ġlengths":20428,"Ġvic":20429,"Ġaveraging":20430,"Ġappropriately":20431,"ĠCamera":20432,"Ġsandwich":20433,"ĠCDC":20434,"Ġcoordinate":20435,"Ġnavig":20436,"Ġgoodness":20437,"laim":20438,"Ġbrake":20439,"Ġextremist":20440,"ĠWake":20441,"ĠMend":20442,"ĠTiny":20443,"ĠCOL":20444,"ĠRF":20445,"ĠDual":20446,"ĠWine":20447,"Case":20448,"Ġrefined":20449,"Ġlamp":20450,"Lead":20451,"Ġbapt":20452,"ĠCarb":20453,"ĠSadd":20454,"ĠMinneapolis":20455,"PDF":20456,"Early":20457,"ĠHidden":20458,"Its":20459,"ĠTIME":20460,"Ġpap":20461,"Ġcommissioned":20462,"ĠFew":20463,"ĠColts":20464,"ĠBren":20465,"Ġbothered":20466,"Ġlikewise":20467,"Exper":20468,"ĠSchw":20469,"cry":20470,"nn":20471,"ĠMitch":20472,"imon":20473,"MG":20474,"bm":20475,"UMP":20476,"rays":20477,"Ġregistry":20478,"Ġ270":20479,"achine":20480,"rella":20481,"anting":20482,"00000":20483,"Ġruined":20484,"spot":20485,"Ġta":20486,"Ġmaximize":20487,"Ġinconven":20488,"Dead":20489,"Human":20490,"Enabled":20491,"ĠMarie":20492,"Ġchill":20493,"ĠParadise":20494,"Ġstarring":20495,"ĠLatino":20496,"ĠProtocol":20497,"ĠEVER":20498,"Ġsuppliers":20499,"message":20500,"ĠBrock":20501,"Ġserum":20502,"âĸĪâĸĪâĸĪâĸĪ":20503,"Ġencomp":20504,"Ġambition":20505,"uese":20506,"Ġarrows":20507,"Andrew":20508,"Ġantenna":20509,"Ġ1961":20510,"ĠBark":20511,"Ġbool":20512,"ãĤª":20513,"ĠStorage":20514,"Ġrailway":20515,"Ġtougher":20516,"ĠCad":20517,"Ġwashing":20518,"Py":20519,"']":20520,"embed":20521,"ĠMemphis":20522,"ackle":20523,"Ġfamously":20524,"ĠFortunately":20525,"ovies":20526,"Ġmindset":20527,"Ġsneak":20528,"ĠDh":20529,"RAW":20530,"ĠSimpson":20531,"Ġlivest":20532,"Ġlandmark":20533,"Ġcement":20534,"Low":20535,"Ġthrilled":20536,"ĠCourse":20537,"inel":20538,"Ġchuck":20539,"idate":20540,"global":20541,"Ġwhit":20542,"Ġ�":20543,"adays":20544,"ski":20545,"ĠSV":20546,"Ġviruses":20547,"306":20548,"ĠRespons":20549,"Ġtheaters":20550,"ĠBranch":20551,"ĠGeneva":20552,"ĠMK":20553,"Ġunbeliev":20554,"Ġcommunist":20555,"Original":20556,"ĠReceived":20557,"ĠTransfer":20558,"ĠArg":20559,"Input":20560,"ĠStrategy":20561,"Ġpalace":20562,"thening":20563,"Dri":20564,"Ġsentencing":20565,"umbnail":20566,"Ġpins":20567,"recy":20568,"Ġsiblings":20569,"Getting":20570,"ĠBU":20571,"ĠNorthwest":20572,"Ġprolonged":20573,"ĠSakura":20574,"Comb":20575,"ĠBour":20576,"Ġinadequate":20577,"ĠKash":20578,"Ġusername":20579,"ĠImprove":20580,"Ġbattling":20581,"ĠMAC":20582,"Ġcurriculum":20583,"Ġsoda":20584,"ĠCannon":20585,"Ġsensible":20586,"spons":20587,"December":20588,"Ġwicked":20589,"ĠPengu":20590,"Ġdictators":20591,"ĠHearts":20592,"ogyn":20593,"Ġsimilarities":20594,"ĠStats":20595,"Ġhollow":20596,"itations":20597,"\":[":20598,"Ġhover":20599,"ĠListen":20600,"sch":20601,"Sund":20602,"Ġcad":20603,"ĠParks":20604,"Ġlur":20605,"Ġhype":20606,"ĠLem":20607,"NAME":20608,"isure":20609,"Friday":20610,"Ġshoots":20611,"Ġcloses":20612,"Ġdb":20613,"ĠRidge":20614,"ĠDifferent":20615,"Ġreplies":20616,"ĠBroadway":20617,"opers":20618,"Ġintoler":20619,"ĠZeus":20620,"akespe":20621,"Ġproprietary":20622,"Ġrequesting":20623,"Ġcontrollers":20624,"ĠMIN":20625,"imedia":20626,"becca":20627,"Ġexpans":20628,"Ġoils":20629,"Bot":20630,"ĠChand":20631,"Ġprinter":20632,"Ġtopped":20633,"ĠPOL":20634,"ĠEarlier":20635,"Social":20636,"avin":20637,"Ġdecreases":20638,"ĠSeb":20639,"Ġspecifications":20640,"ĠBlast":20641,"ĠKurt":20642,"Ġfreel":20643,"Brown":20644,"Ġdilig":20645,"roe":20646,"ĠProblem":20647,"ĠQuad":20648,"Ġdecentral":20649,"ĠVector":20650,"anut":20651,"Ġplugins":20652,"ĠGregory":20653,"Ġfucked":20654,"elines":20655,"ĠAmbassador":20656,"take":20657,"Ġcleans":20658,"ongyang":20659,"Anonymous":20660,"stro":20661,"\"}":20662,"aline":20663,"ĠOdd":20664,"ĠEug":20665,"216":20666,"Ġboil":20667,"ĠPowers":20668,"Ġnurses":20669,"Obviously":20670,"ĠTechnical":20671,"Ġexceeded":20672,"ORS":20673,"Ġextremists":20674,"Ġtraces":20675,"expl":20676,"Ġcomr":20677,"ĠSach":20678,")/":20679,"Ġmasks":20680,"Ġsci":20681,"Bon":20682,"Ġregression":20683,"wegian":20684,"Ġadvisor":20685,"itures":20686,"ĠVo":20687,"example":20688,"ĠInstruct":20689,"Ġsiege":20690,"Ġreductions":20691,"ptr":20692,"Ġstatutory":20693,"Ġremoves":20694,"Ġpuck":20695,"redits":20696,"Ġbee":20697,"Ġsalad":20698,"Ġpromotions":20699,"ĠJoshua":20700,"withstanding":20701,"ETH":20702,"ĠCha":20703,"imus":20704,"Ġexpenditure":20705,"aunting":20706,"Ġdelighted":20707,"Ġ155":20708,"beh":20709,"Ġcarpet":20710,"ĠSpart":20711,"Ġjungle":20712,"lists":20713,"Ġbullying":20714,"ĠNobel":20715,"ĠGlen":20716,"Ġreferenced":20717,"Ġintroduces":20718,"sein":20719,"Ġchopped":20720,"glass":20721,"ĠWrest":20722,"Ġneutrality":20723,"ĠâĻ":20724,"Ġinvestigator":20725,"Ġshelves":20726,"Ġunconstitutional":20727,"Ġreproduction":20728,"Ġmerchant":20729,"mia":20730,"Ġmetrics":20731,"Ġexplosives":20732,"ĠSonia":20733,"Ġbodily":20734,"Ġthickness":20735,"Ġpredominantly":20736,"ĠAbility":20737,"Ġmonitored":20738,"ICH":20739,"Ġ].":20740,"ĠMartinez":20741,"Ġvisibility":20742,"Ġqueries":20743,"Ġgenocide":20744,"ĠWarfare":20745,"Query":20746,"Ġstudios":20747,"Ġembry":20748,"Ġcorridor":20749,"Ġcleaned":20750,"complete":20751,"ĠMH":20752,"Ġenrollment":20753,"INGS":20754,"Ġimpacted":20755,"Ġdisastrous":20756,"ĠYun":20757,"ĠClaire":20758,"ĠBasically":20759,"yt":20760,"usterity":20761,"Ġindirectly":20762,"wik":20763,"Ġdod":20764,"ĠCarr":20765,"Ġamp":20766,"Ġprohibit":20767,"ĠInitial":20768,"ĠRd":20769,"iji":20770,"Ġeducate":20771,"corn":20772,"iott":20773,"ĠBeauty":20774,"Ġdetective":20775,"ĠConn":20776,"since":20777,"Ġstagger":20778,"Ġobese":20779,"Ġbree":20780,"ologic":20781,"isse":20782,"walker":20783,"Ġblades":20784,"Ġlawful":20785,"func":20786,"ĠBehind":20787,"Ġappetite":20788,"Ġ(*":20789,"Ġtennis":20790,"Ġoffspring":20791,"Ġjets":20792,"Ġstructured":20793,"Ġaforementioned":20794,"Nov":20795,"Ġscaling":20796,"fill":20797,"Ġstew":20798,"Ġcurb":20799,"ĠStephan":20800,"edIn":20801,"SF":20802,"obic":20803,"éŃĶ":20804,"oug":20805,"ĠMM":20806,"Ġgenetically":20807,"opez":20808,"136":20809,"Ġumb":20810,"ancers":20811,"Ġcohort":20812,"Ġmerchandise":20813,"Ġimposing":20814,"ĠLegislature":20815,"ĠArchive":20816,"ivia":20817,"ĠNaval":20818,"Ġoffences":20819,"Ġmiracle":20820,"Ġsnapped":20821,"Ġfoes":20822,"Ġextensively":20823,"ĠRaf":20824,"Ġcater":20825,"edience":20826,"Kit":20827,"ĠBin":20828,"Ġrecommends":20829,"ĠCities":20830,"Ġrigid":20831,"ĠREAD":20832,"ĠNoble":20833,"ĠTian":20834,"Ġcertificates":20835,"antis":20836,"oiler":20837,"ĠBuddhist":20838,"did":20839,"Ġsurveyed":20840,"Ġdownward":20841,"Ġprints":20842,"ĠMotion":20843,"ronics":20844,"ĠSans":20845,"ossibly":20846,"uctions":20847,"Ġcolonies":20848,"ĠDanish":20849,"unit":20850,"Ġspoil":20851,"Ġadvisory":20852,"berries":20853,"Plan":20854,"Ġspecification":20855,"ophers":20856,"ĠResource":20857,"Ġshirts":20858,"prisingly":20859,"communications":20860,"Ġtrivial":20861,"Ġmentioning":20862,"isexual":20863,"Ġsupplements":20864,"Ġsupervision":20865,"BP":20866,"vor":20867,"Ġwit":20868,"Ġcooldown":20869,"Ġplaintiff":20870,"ĠReviews":20871,"ĠSri":20872,"ĠMint":20873,"ĠSugar":20874,"Ġafterward":20875,"ĠPriest":20876,"ĠInvestment":20877,"ogene":20878,"ĠTaking":20879,"Ġstretching":20880,"Ġinflammation":20881,"ĠTehran":20882,"Ġlining":20883,"Ġfreezing":20884,"ĠEntity":20885,"Ġinspiring":20886,"special":20887,"price":20888,"Ġsue":20889,"ĠPorter":20890,"ounge":20891,"ETA":20892,"ĠDerek":20893,"ĠLuis":20894,"uo":20895,"ymph":20896,"Ġexterior":20897,"ihil":20898,"ĠAshley":20899,"inator":20900,"Ġnutrients":20901,"ĠThrones":20902,"Ġfinances":20903,"ĠInspect":20904,"Ġspecially":20905,"ĠRequired":20906,"ĠPTS":20907,"ĠViolence":20908,"ointed":20909,"shots":20910,"Ġexcerpt":20911,"coon":20912,"INS":20913,"ĠGri":20914,"Ġrecognised":20915,"Week":20916,"Young":20917,"Ġvom":20918,"isle":20919,"ĠCurry":20920,"ĠBuddh":20921,"Ġnotebook":20922,"Ġdurable":20923,"/?":20924,"ĠGad":20925,"ĠPupp":20926,"Ġforgive":20927,"park":20928,"Ġpersonalities":20929,"analysis":20930,"clamation":20931,"Ġelevator":20932,"Ġwarehouse":20933,"ĠRole":20934,"unn":20935,"Ġillustration":20936,"ĠScan":20937,"Ġatmospheric":20938,"Import":20939,"ANC":20940,"ricted":20941,"fu":20942,"010":20943,"Ġarche":20944,"Ġrewarded":20945,"akespeare":20946,"Ġinternally":20947,"ĠRBI":20948,"alker":20949,"Ġelephant":20950,"owitz":20951,"ĠPizza":20952,"Ġbipartisan":20953,"és":20954,"Ġslowed":20955,"ĠStark":20956,"Ġoverride":20957,"OUS":20958,"Ġ320":20959,"undreds":20960,"ĠDeck":20961,"ĠCensus":20962,"bee":20963,"146":20964,"otor":20965,"Ġip":20966,"Ġub":20967,"ocations":20968,"ĠButton":20969,"rice":20970,"Ġcripp":20971,"fff":20972,"Ġoriginated":20973,"Ġoverwhelmed":20974,"appa":20975,"Ġforemost":20976,"âĢij":20977,"ĠLEG":20978,"release":20979,"eatured":20980,"atches":20981,"Ġreps":20982,"Ġlending":20983,"ĠReference":20984,"ĠClient":20985,"165":20986,"venth":20987,"Complete":20988,"ĠPatrol":20989,"Ġsworn":20990,"cam":20991,"Ġshuttle":20992,"ĠRalph":20993,"Ġhometown":20994,"-,":20995,"onal":20996,"ĠBP":20997,"åı":20998,"Ġpersuade":20999,"ĠAlexand":21000,"Ġcombines":21001,"Ġvivid":21002,"ĠLag":21003,"Ġencoding":21004,"Ġsalvation":21005,"wen":21006,"ĠRecovery":21007,"iya":21008,"University":21009,"ĠBiden":21010,"Ġbudgets":21011,"ĠTexans":21012,"fits":21013,"Ġhonored":21014,"Ġpython":21015,"TD":21016,"###":21017,"clone":21018,"Ġblink":21019,"ĠLiquid":21020,"Ġunemployed":21021,"Ġclashes":21022,"ĠCounsel":21023,"Ġdirecting":21024,"Ġpunct":21025,"ĠFalcons":21026,"Ġshark":21027,"ĠDamascus":21028,"Ġjeans":21029,"Ġembark":21030,"Ġseize":21031,"Ġupwards":21032,"280":21033,"ĠEz":21034,"ĠAnything":21035,"Ġexotic":21036,"lower":21037,"ĠCreator":21038,"ĠUm":21039,"Ġsuburbs":21040,"berger":21041,"ĠWend":21042,"Ġmint":21043,"ĠXX":21044,"ĠDro":21045,"Ġsuffers":21046,"Ġherb":21047,"tree":21048,"Ġfragile":21049,"Ġflooded":21050,"ĠAlcohol":21051,"olean":21052,"nyder":21053,"ĠKO":21054,"Fram":21055,"Ġ136":21056,"Ġowed":21057,"ĠMelee":21058,"ĠHash":21059,"Ġwhisk":21060,"Ġsudo":21061,"rr":21062,"Quick":21063,"appro":21064,"Ġii":21065,"ĠExamples":21066,"hee":21067,"Ġpromotes":21068,"perature":21069,"kar":21070,"ĠHonor":21071,"Ġsodium":21072,"ĠLif":21073,"rosso":21074,"intendent":21075,"Ġcorrespondent":21076,"Found":21077,"secret":21078,"Ġidentifies":21079,"agne":21080,"Ġlou":21081,"ĠPP":21082,"Ġcoincidence":21083,"move":21084,"Ġmilitia":21085,"Ġinfiltr":21086,"ĠPrimary":21087,"Ġpitching":21088,"ĠIb":21089,"ĠGOOD":21090,"ãĤ¸":21091,"ĠWizards":21092,"iral":21093,"ĠVenus":21094,"RR":21095,"ĠâĢķ":21096,"ĠCasey":21097,"Ġsadly":21098,"Ġadmire":21099,"Ġembarrassed":21100,"cb":21101,"Mel":21102,"Ġtubes":21103,"Ġbeautifully":21104,"ĠQueensland":21105,"Below":21106,"rez":21107,"quet":21108,"pleasant":21109,"Ġ«":21110,"Camp":21111,"Ġdecisive":21112,"1998":21113,"ĠLamb":21114,"utton":21115,"hn":21116,"ĠJagu":21117,"aunder":21118,"ĠCord":21119,"Ġclerk":21120,"Ġcaffe":21121,"Ġwiped":21122,"Ġreim":21123,"ĠMountains":21124,"Ġimprisoned":21125,"Ġdevelops":21126,"ĠPra":21127,"Ġmodeling":21128,"Anyone":21129,"ancel":21130,"ĠSit":21131,"Ġshields":21132,"Ġlawn":21133,"Ġcardiovascular":21134,"Ġdemonstrating":21135,"Ġparse":21136,"ĠIsraelis":21137,"Ġeuros":21138,"143":21139,"Ġglorious":21140,"inski":21141,"ecd":21142,"Ġconditioning":21143,"Ġhelpless":21144,"Ġmicrosc":21145,"ĠHarbor":21146,"Ġstakes":21147,"Ġ260":21148,"Ġunequ":21149,"ĠFloyd":21150,"Ġdamp":21151,"Ġapparatus":21152,"ĠLaws":21153,"Ġcounters":21154,"Ġinduce":21155,"atable":21156,"ĠAhmed":21157,"Ġslam":21158,"November":21159,"Ġpersist":21160,"Ġimminent":21161,"án":21162,"Ġshred":21163,"Ġphases":21164,"ĠEdmonton":21165,"ĠArmstrong":21166,"ĠMeet":21167,"ĠKitty":21168,"ÑĢ":21169,"circ":21170,"ĠAdult":21171,"Ġarose":21172,"ĠXen":21173,"Dan":21174,"gow":21175,"Ġsuperf":21176,"ĠAdmir":21177,"Ġendure":21178,"Ġkeyword":21179,"yrus":21180,"Ġyarn":21181,"Ġpathway":21182,"ĠHopkins":21183,"midt":21184,"Ġcensorship":21185,"dependent":21186,"Ġinstructor":21187,"Sources":21188,"Ġtoe":21189,"Ġballoon":21190,"Nob":21191,"Ġswear":21192,"ĠCastro":21193,"Ġgloss":21194,"ĠKavanaugh":21195,"Ġremarkably":21196,"Photos":21197,"ĠNom":21198,"ĠSoutheast":21199,"yers":21200,"Ġvalidation":21201,"Ġcannon":21202,"ĠVictory":21203,"ĠPierre":21204,"Ġcautious":21205,"Audio":21206,"Ġfetch":21207,"ĠGift":21208,"ĠHyp":21209,"Ġremedy":21210,"ZE":21211,"Ġscent":21212,"Ġbeard":21213,"ĠRut":21214,"-\"":21215,"Ġpatents":21216,"Hy":21217,"Ġunjust":21218,"Ġpotato":21219,"Ġforthcoming":21220,"Ġchef":21221,"ĠRift":21222,"affe":21223,"ĠROM":21224,"ĠLaunch":21225,"Ġpads":21226,"ĠNeo":21227,"Ġonset":21228,"Ġsqueeze":21229,"safe":21230,"Ġprefix":21231,"ĠTM":21232,"ĠNearly":21233,"ĠClinical":21234,"ĠMental":21235,"otiation":21236,"ĠUnic":21237,"antry":21238,"ĠCir":21239,"Ġepit":21240,"æ":21241,"Ġextracted":21242,"versely":21243,"riad":21244,"Ġstrains":21245,"Ġtops":21246,"Ġpoem":21247,"ĠRandy":21248,"ĠMaple":21249,"THER":21250,"upiter":21251,"ĠSSD":21252,"ļé":21253,"Ġuncon":21254,"pering":21255,"Ġslept":21256,"iners":21257,"Ġunderwater":21258,"ĠEvidence":21259,"gone":21260,"205":21261,"Ġhistorians":21262,"Ġsynthesis":21263,"Ġfrog":21264,"basketball":21265,"Ġvibrant":21266,"Ġsubord":21267,"Ġ365":21268,"ĠDial":21269,"Ġcooperate":21270,"HAHA":21271,"Ġgreeted":21272,"158":21273,"Ġjazz":21274,"Ġintox":21275,"ĠWalking":21276,"Ġsupervisor":21277,"ĠFusion":21278,"ĠMercedes":21279,"send":21280,"Ham":21281,"sd":21282,"nl":21283,"Ġtours":21284,"ĠFIFA":21285,"Ġculp":21286,"gd":21287,"304":21288,"Ġpleas":21289,"Ġillustrates":21290,"ĠColombia":21291,"Ġhighlighting":21292,"ĠSummary":21293,"Ġexposing":21294,"ĠDru":21295,"Ġirony":21296,"ritional":21297,"ĠCarroll":21298,"ĠEllis":21299,"Pict":21300,"ĠRapt":21301,"Ġadapter":21302,"Ġunm":21303,"Ġcorpse":21304,"Ġcelebrities":21305,"Den":21306,"atum":21307,"ĠApocalypse":21308,"ĠWag":21309,"lining":21310,"Ġhormones":21311,"Rub":21312,"ĠXi":21313,"ĠVaults":21314,"208":21315,"alkyrie":21316,"inosaur":21317,"Ġfeeds":21318,"vity":21319,"Ġdefeating":21320,"Wait":21321,"Ġemphasize":21322,"ĠSteelers":21323,"yrinth":21324,"leys":21325,"ĠWhenever":21326,"Currently":21327,"ĠClock":21328,"Ġcollectively":21329,"anyon":21330,"ĠJP":21331,"Ġmentality":21332,"Ġdownloads":21333,"Ġsurroundings":21334,"ĠBarnes":21335,"Ġflagship":21336,"Ġindicators":21337,"Ġgrapp":21338,"January":21339,"ĠElemental":21340,"ĠAthena":21341,"ibal":21342,"Ġsights":21343,"Ġcapita":21344,"ĠTreaty":21345,"Ġvoiced":21346,"ĠGaz":21347,"lette":21348,"Ġya":21349,"Ġexpired":21350,"Legend":21351,"Hot":21352,"nature":21353,"Ġunstable":21354,"Ġ280":21355,"ú":21356,"Comment":21357,"ALE":21358,"Ġquests":21359,"Ġhandler":21360,"nis":21361,"Ġversatile":21362,"Ġconceal":21363,"engeance":21364,"ĠInteractive":21365,"Ġobsessed":21366,"ĠDogs":21367,"Ġcracked":21368,"Sound":21369,"sv":21370,"ĠDylan":21371,"roads":21372,"fx":21373,"ĠCatholics":21374,"ĠHag":21375,"Ġslammed":21376,"Ġglowing":21377,"sale":21378,"Ġtissues":21379,"ĠChi":21380,"nee":21381,"Ġcher":21382,"sic":21383,"urrection":21384,"Ġbacon":21385,"ulatory":21386,").\"":21387,"Ġirregular":21388,"FORM":21389,"assed":21390,"Ġintentional":21391,"Ġcompensate":21392,"ĠSpeaking":21393,"ĠSets":21394,"153":21395,"Ġconventions":21396,"bands":21397,"emade":21398,"Ġecc":21399,"ĠWinston":21400,"ĠAssassin":21401,"ĠBelgian":21402,"Ġdependence":21403,"Ġniche":21404,"Ġbark":21405,"ĠJazz":21406,"Ġdisadvantage":21407,"Ġgasoline":21408,"Ġ165":21409,"çļĦ":21410,"essa":21411,"module":21412,"angular":21413,"OY":21414,"ĠTreatment":21415,"itas":21416,"olation":21417,"ĠArnold":21418,"Ġfeud":21419,"ĠNest":21420,"Ġtheatre":21421,"ewater":21422,"Ġminors":21423,"olicy":21424,"ĠHaven":21425,"division":21426,"Ġtrunk":21427,"Far":21428,"ĠPull":21429,"Ġcapturing":21430,"Ġ1800":21431,"ĠTeen":21432,"Ġexempl":21433,"Ġclinics":21434,"ĠBurg":21435,"Ġsubstit":21436,"Ġpayload":21437,"ĠLav":21438,"ĠTroy":21439,"ĠWitness":21440,"Ġfragments":21441,"Ġpasswords":21442,"Ġgospel":21443,"ĠGin":21444,"Ġtenants":21445,"olith":21446,"Six":21447,"Previous":21448,"ĠAges":21449,"ĠDarwin":21450,"Ġblat":21451,"Ġempathy":21452,"smith":21453,"bag":21454,"ĠEcho":21455,"ĠCamb":21456,"ĠMadd":21457,"ĠBoo":21458,"Ġrede":21459,"ĠBurning":21460,"Ġsmoothly":21461,"ĠAdrian":21462,"ĠVampire":21463,"ĠMonsters":21464,"steam":21465,"Style":21466,"Ma":21467,"rea":21468,"ĠDwar":21469,"alyst":21470,"ursor":21471,"Ġelimination":21472,"Ġcrypto":21473,"cht":21474,"ĠEternal":21475,"â̦]":21476,"ĠSorce":21477,"Ill":21478,"NER":21479,"Ġuh":21480,"Conclusion":21481,"wage":21482,"Ġrespir":21483,"Ġreminis":21484,"hetical":21485,"Ġgy":21486,"Ġutilized":21487,"icidal":21488,"Ġ1900":21489,"Ġhunters":21490,"ĠSwan":21491,"ĠReact":21492,"Ġvisitor":21493,"ĠThanksgiving":21494,"308":21495,"Posts":21496,"Ġhips":21497,"1997":21498,"omers":21499,"Ġknocking":21500,"ĠVehicle":21501,"Ġtil":21502,"Ġ138":21503,"Ġmi":21504,"ĠInvestigation":21505,"ĠKenya":21506,"Ġcasino":21507,"Ġmotives":21508,"Ġregain":21509,"rex":21510,"Ġweekends":21511,"Ġstabbed":21512,"boro":21513,"Ġexploited":21514,"ĠHAVE":21515,"ĠTelevision":21516,"cock":21517,"Ġpreparations":21518,"Ġendeav":21519,"ĠRemote":21520,"ĠMaker":21521,"ĠProdu":21522,"ĠEvan":21523,"Ġinformational":21524,"ĠLouisville":21525,"154":21526,"ĠDreams":21527,"Ġplots":21528,"ĠRunner":21529,"Ġhurting":21530,"Ġacademy":21531,"ĠMontgomery":21532,"nm":21533,"ĠLanc":21534,"ĠAlz":21535,"210":21536,"elong":21537,"Ġretailer":21538,"Ġarising":21539,"Ġrebellion":21540,"Ġblonde":21541,"played":21542,"Ġinstrumental":21543,"Cross":21544,"Ġretention":21545,"Ġtherapeutic":21546,"Ġseas":21547,"Ġinfantry":21548,"ĠClint":21549,"Ġprompting":21550,"Ġbitch":21551,"Ġstems":21552,"ĠKra":21553,"Ġthesis":21554,"ĠBog":21555,"rued":21556,"Ġkings":21557,"Ġclay":21558,"ificent":21559,"ĠYES":21560,"ĠThing":21561,"ĠCubs":21562,"veyard":21563,"elsh":21564,"inarily":21565,"ĠEy":21566,"ĠRolling":21567,"Ġevolving":21568,"India":21569,"Ġrecognizes":21570,"Ġgraduation":21571,"isers":21572,"Ġfertility":21573,"ĠMilan":21574,"Command":21575,"Ġboxing":21576,"Ġ1943":21577,"Ġgluten":21578,"ĠEmir":21579,"Ġidol":21580,"Ġconceived":21581,"ĠCreation":21582,"Merit":21583,"uddy":21584,"ussions":21585,"ĠLieutenant":21586,"ietal":21587,"Ġunchanged":21588,"ĠScale":21589,"ĠCrimea":21590,"balls":21591,"atorial":21592,"Ġdepths":21593,"Ġempirical":21594,"Ġtransm":21595,"Ġunsafe":21596,"missible":21597,"comfort":21598,"156":21599,"Ġmechanic":21600,"002":21601,"lins":21602,"Ġsmoked":21603,"Pos":21604,"Ġslowing":21605,"Ġlav":21606,"Texas":21607,"Ġcheating":21608,"ĠMetropolitan":21609,"ethyl":21610,"Ġdiscovering":21611,"asse":21612,"Ġpencil":21613,"ĠPyongyang":21614,"Ġcloset":21615,"ĠSheet":21616,"ĠEntry":21617,"oustic":21618,"Ġmyst":21619,"erate":21620,"ariat":21621,"Ġminerals":21622,"Ġmusician":21623,"ĠPul":21624,"ĠMaz":21625,"249":21626,"Ġpermissions":21627,"Ġiv":21628,"enary":21629,"ickers":21630,"ĠBing":21631,"hea":21632,"enable":21633,"Ġgriev":21634,"Ġasserted":21635,"ĠColonel":21636,"Ġaffidav":21637,"wo":21638,"Ġseated":21639,"ĠRide":21640,"Ġpaintings":21641,"ĠPix":21642,"Ġ137":21643,"ishi":21644,"umbai":21645,"gotten":21646,"ĠEarl":21647,"Ġinning":21648,"Ġcensus":21649,"Ġtravelled":21650,"ĠConsult":21651,"185":21652,"bind":21653,"Ġsimplicity":21654,"Ġoverlooked":21655,"ĠHelpful":21656,"Ġmonkey":21657,"Ġoverwhelmingly":21658,"Blood":21659,"ĠFlint":21660,"ĠJama":21661,"ĠPresent":21662,"ĠRage":21663,"ĠTA":21664,"ptive":21665,"Ġturnout":21666,"wald":21667,"ĠDolphins":21668,"ĠVPN":21669,"Ġonion":21670,"Ġcrafting":21671,"mma":21672,"ĠMercury":21673,"Ġarrange":21674,"Ġalerts":21675,"ĠOT":21676,"zbollah":21677,"Ġgases":21678,"ĠRichardson":21679,"sal":21680,"lar":21681,"Ġfrost":21682,"Ġlowering":21683,"Ġacclaim":21684,"Ġstartups":21685,"ĠGain":21686,"essment":21687,"Ġguardian":21688,"人":21689,"ĠPie":21690,"ĠLinks":21691,"Ġmerits":21692,"Ġawake":21693,"Ġparental":21694,"Ġexceeds":21695,"Ġidle":21696,"ĠPilot":21697,"ĠeBay":21698,"ĠAccept":21699,"ipeg":21700,"Cam":21701,"ĠKot":21702,"Ġtraders":21703,"olitics":21704,"unker":21705,"ĠPale":21706,"osi":21707,"anmar":21708,"Ġ1947":21709,"ĠFell":21710,"estial":21711,"itating":21712,"GF":21713,"ĠSr":21714,"ifted":21715,"Ġconnector":21716,"ĠBone":21717,"illes":21718,"260":21719,"hma":21720,"Ġoverlap":21721,"ĠGitHub":21722,"Ġcleaner":21723,"ĠBaptist":21724,"ĠWAS":21725,"Ġlungs":21726,"Ñģ":21727,"ĠBUT":21728,"Ġcite":21729,"Ġpitched":21730,"reatment":21731,"Ġtrophies":21732,"ĠNu":21733,"386":21734,"ĠPride":21735,"Ġattendees":21736,"[]":21737,"179":21738,"Ġspatial":21739,"Ġprizes":21740,"ĠReligion":21741,"Ġshowcase":21742,"ĠCategory":21743,"vidia":21744,"Target":21745,"Property":21746,"?,":21747,"Ġfusion":21748,"pie":21749,"ĠUCLA":21750,"Ġsoundtrack":21751,"Ġprincess":21752,"ĠCaval":21753,"should":21754,"Ġlimbs":21755,"Background":21756,"Ġlonely":21757,"Ġcores":21758,"ĠTail":21759,"sheet":21760,"Ġ132":21761,"Ra":21762,"ãĤ«":21763,"ĠBolt":21764,"Ġbooked":21765,"Ġadminister":21766,"Ġequals":21767,"wy":21768,"Ġobserving":21769,"ĠBaron":21770,"ĠAdobe":21771,"Ġvirgin":21772,"ĠSocialist":21773,"Move":21774,"ghazi":21775,"ĠLinda":21776,"212":21777,"Ġbrewing":21778,"Ġmerchants":21779,"burse":21780,"Ġdivor":21781,"Ġmetals":21782,"ĠNer":21783,"Ġsums":21784,"ĠEnemy":21785,"Ġenvision":21786,"Ġgranting":21787,"ĠHoney":21788,"ĠSkyrim":21789,"Ġsocio":21790,"graded":21791,"Ġselective":21792,"WASHINGTON":21793,"Ġ1948":21794,"ĠSirius":21795,"ĠGross":21796,"activity":21797,"ĠIvan":21798,"Ġfurious":21799,"BSD":21800,"ĠPrevious":21801,"Ġresponsive":21802,"Ġcharitable":21803,"Ġleaning":21804,"ĠPew":21805,"Ġviolates":21806,"\\\\\\\\\\\\\\\\":21807,"ĠComing":21808,"wire":21809,"Ġpoet":21810,"Ġresolutions":21811,"command":21812,"ĠPortuguese":21813,"Ġnickname":21814,"Ġdeaf":21815,"February":21816,"Ġrecognise":21817,"Ġentirety":21818,"Ġseasonal":21819,"placed":21820,"ĠTelegraph":21821,"Ġmicrophone":21822,"ouring":21823,"Ġgrains":21824,"Ġgoverned":21825,"Ġpostp":21826,"ĠWaters":21827,"inement":21828,"Ġundocumented":21829,"ĠComcast":21830,"Ġfox":21831,"Ġassaults":21832,"reon":21833,"many":21834,"ĠJenkins":21835,"ĠAnyway":21836,"Ġassessments":21837,"Ġdowns":21838,"ĠMouse":21839,"Ġsuperb":21840,"kt":21841,"ĠDow":21842,"Ġtaxation":21843,"401":21844,"Ġsmiles":21845,"Ġundertaken":21846,"Ġexh":21847,"Ġenthusiastic":21848,"Ġtwent":21849,"Ġgovernmental":21850,"Ġautonomy":21851,"ĠTechnologies":21852,"ĠChain":21853,"Ġprevalent":21854,"fb":21855,"Ġnicotine":21856,"ogram":21857,"job":21858,"Ġawaiting":21859,"ĠMenu":21860,"Ġdeputies":21861,"kov":21862,"ishops":21863,"Button":21864,"ĠShanghai":21865,"Ġdiesel":21866,"ĠDuck":21867,"Ryan":21868,"ĠPCs":21869,"NF":21870,"jury":21871,"ente":21872,"Ġinaccurate":21873,"eddy":21874,"Whatever":21875,"Ġshowc":21876,"ĠNad":21877,"odus":21878,"etr":21879,"Ġplaintiffs":21880,"ĠWOR":21881,"ĠAssange":21882,"Ġprivat":21883,"Ġpremiums":21884,"Ġtam":21885,"URL":21886,"Ġelites":21887,"ĠRanger":21888,"ottenham":21889,"ĠHoff":21890,"ĠAthens":21891,"Ġdefinite":21892,"Ġsighed":21893,"Ġevenly":21894,"211":21895,"ĠAmber":21896,"akia":21897,"Ġmailing":21898,"Ġcrashing":21899,"ĠConfederate":21900,"rugged":21901,"Wal":21902,"ĠDepths":21903,"Ġjuvenile":21904,"Ġreactor":21905,"Introduction":21906,"ĠDeluxe":21907,"1995":21908,"ĠSanchez":21909,"ĠMead":21910,"ivable":21911,":-":21912,"ĠPlanning":21913,"ĠTrap":21914,"quin":21915,"ĠProtect":21916,"vered":21917,"Information":21918,"Ġkidney":21919,"innamon":21920,"las":21921,"Ġpolicing":21922,"Ġtolerate":21923,"ĠQi":21924,"Ġbiased":21925,"Fort":21926,"ĠKi":21927,"save":21928,"Ġprivileged":21929,"Ġbeasts":21930,"ĠGlas":21931,"ĠCinem":21932,"Ġcomeback":21933,"Sunday":21934,"Ġextinction":21935,"hops":21936,"Ġtransmit":21937,"Ġdoubles":21938,"ĠFlat":21939,"167":21940,"Ġdisputed":21941,"Ġinjustice":21942,"foo":21943,"Vict":21944,"roleum":21945,"ĠJulie":21946,"Context":21947,"ĠRarity":21948,"issue":21949,"Component":21950,"Ġcounseling":21951,"anne":21952,"dark":21953,"Ġobjections":21954,"uilt":21955,"Ġgast":21956,"Ġplac":21957,"Ġunused":21958,"ãĥĩ":21959,"ĠTrial":21960,"ĠJas":21961,"hedral":21962,"obb":21963,"Ġtemporal":21964,"ĠPRO":21965,"ĠNW":21966,"ĠAnniversary":21967,"Large":21968,"Ġtherm":21969,"Ġdavid":21970,"Ġsystemic":21971,"ĠShir":21972,"mut":21973,"ĠNept":21974,"address":21975,"Ġscanning":21976,"Ġunderstandable":21977,"Ġcanvas":21978,"Cat":21979,"ĠZoo":21980,"Ġangels":21981,"LO":21982,"ĠStatement":21983,"ĠSig":21984,"ovable":21985,"ĠAway":21986,"sharing":21987,"ocrats":21988,"stated":21989,"Ġweighing":21990,"Nor":21991,"wild":21992,"Bey":21993,"Ġastonishing":21994,"ĠReynolds":21995,"Ġopener":21996,"Ġtrainer":21997,"Ġsurgical":21998,"pn":21999,"Ġadjusting":22000,"wheel":22001,"Ġfrown":22002,"ervative":22003,"Ġsuspend":22004,"Within":22005,"tein":22006,"Ġobstacle":22007,"Ġliberties":22008,"ymes":22009,"Ġuranium":22010,"ansom":22011,"anol":22012,"uba":22013,"ĠLoss":22014,"Ġarous":22015,"ĠHenderson":22016,"Wow":22017,"spl":22018,"cur":22019,"ĠÂŃ":22020,"Ġtheirs":22021,"Damage":22022,"Ġdownloading":22023,"Ġdiscern":22024,"ĠSto":22025,"ĠFla":22026,"Ġhath":22027,"ĠAj":22028,"Ġunpleasant":22029,"European":22030,"expensive":22031,"Ġscreenshot":22032,"ĠUV":22033,"Ġallied":22034,"ĠPersian":22035,"Ġmonopoly":22036,"Ġatom":22037,"ĠRedskins":22038,"\"><":22039,"Ġcancell":22040,"Ġcinema":22041,"131":22042,"fair":22043,"ĠAlfred":22044,"Ġduck":22045,"args":22046,"223":22047,"ĠISI":22048,"Ġsignaling":22049,"inar":22050,"Ġlaughs":22051,"Ġforwards":22052,"Ġreckless":22053,"Ġlisteners":22054,"ativity":22055,"Ġvastly":22056,"nant":22057,"Less":22058,"ĠHunting":22059,"ĠScientific":22060,"ITED":22061,"Ġknight":22062,"ĠHTC":22063,"usa":22064,"tmp":22065,"Ġrude":22066,"ĠLegendary":22067,"Ġarises":22068,"Bad":22069,"ĠClaim":22070,"peg":22071,"Ġrealities":22072,"Think":22073,"Ġ°":22074,"Ġrode":22075,"Ġstrive":22076,"Ġanecd":22077,"Ġshorts":22078,"Ġhypothes":22079,"Ġcoordinated":22080,"ĠGandhi":22081,"ĠFPS":22082,"RED":22083,"Ġsusceptible":22084,"Ġshrink":22085,"ĠChart":22086,"Help":22087,"Ġion":22088,"deep":22089,"ribes":22090,"ĠKai":22091,"ĠCustomer":22092,"Summary":22093,"Ġcough":22094,"wife":22095,"Ġlend":22096,"Ġpositioning":22097,"Ġlottery":22098,"ĠCanyon":22099,"Ġfade":22100,"Ġbronze":22101,"ĠKenny":22102,"Ġboasts":22103,"ĠEnhanced":22104,"record":22105,"Ġemergence":22106,"Ġakin":22107,"ĠBert":22108,"itous":22109,"âĸij":22110,"Ġstip":22111,"Ġexchanged":22112,"omore":22113,"alsh":22114,"Ġreservoir":22115,"Ġstandpoint":22116,"WM":22117,"Ġinitiate":22118,"Ġdecay":22119,"Ġbrewery":22120,"Ġterribly":22121,"Ġmortal":22122,"levard":22123,"Ġrevis":22124,"NI":22125,"elo":22126,"Ġconfess":22127,"ĠMSNBC":22128,"Ġsubmissions":22129,"Controller":22130,"Ġ202":22131,"ĠRuth":22132,"});":22133,"ĠAzure":22134,"Ġ.\"":22135,"206":22136,"ĠMarketing":22137,"Ġlaund":22138,"iencies":22139,"Ġrenowned":22140,"ĠTrou":22141,"ĠNGO":22142,"blems":22143,"Ġterrified":22144,"Ġwarns":22145,"Ġpert":22146,"Ġunsure":22147,"480":22148,"alez":22149,"ultz":22150,"ĠOutside":22151,"Ġstyl":22152,"ĠUnderground":22153,"Ġpanc":22154,"Ġdictionary":22155,"Ġfoe":22156,"riminal":22157,"ĠNorwegian":22158,"Ġjailed":22159,"Ġmaternal":22160,"ée":22161,"ĠLucy":22162,"cop":22163,"Cho":22164,"Ġunsigned":22165,"ĠZelda":22166,"ĠInsider":22167,"ĠContinued":22168,"Ġ133":22169,"ĠNaruto":22170,"ĠMajority":22171,"169":22172,"ĠWo":22173,"ãĤĵ":22174,"Ġpastor":22175,"Ġinformal":22176,"н":22177,"anthrop":22178,"join":22179,"ãģĹ":22180,"itational":22181,"NP":22182,"ĠWriting":22183,"fn":22184,"ĠBever":22185,"195":22186,"Ġyelling":22187,"Ġdrastically":22188,"Ġeject":22189,"Ġneut":22190,"Ġthrive":22191,"ĠFrequ":22192,"oux":22193,"Ġpossesses":22194,"ĠSenators":22195,"ĠDES":22196,"ĠShakespeare":22197,"ĠFranco":22198,"ĠLB":22199,"uchi":22200,"Ġincarn":22201,"Ġfounders":22202,"Function":22203,"Ġbrightness":22204,"ĠBT":22205,"Ġwhale":22206,"ĠTheater":22207,"mass":22208,"ĠDoll":22209,"Something":22210,"Ġechoed":22211,"ĠHex":22212,"crit":22213,"afia":22214,"Ġgoddess":22215,"Ġeleven":22216,"ĠPreview":22217,"ĠAurora":22218,"Ġ401":22219,"ulsive":22220,"ĠLogan":22221,"inburgh":22222,"ĠCenters":22223,"ĠONLY":22224,"ĠAid":22225,"Ġparadox":22226,"Ġhurd":22227,"ĠLC":22228,"Due":22229,"court":22230,"Ġoffended":22231,"Ġevaluating":22232,"ĠMatthews":22233,"Ġtomb":22234,"Ġpayroll":22235,"Ġextraction":22236,"ĠHands":22237,"ifi":22238,"Ġsupernatural":22239,"ĠCOMM":22240,"]=":22241,"dogs":22242,"Ġ512":22243,"ĠMeeting":22244,"Richard":22245,"ĠMaximum":22246,"Ġideals":22247,"Things":22248,"mand":22249,"ĠRegardless":22250,"Ġhumili":22251,"buffer":22252,"Little":22253,"ĠDani":22254,"ĠNak":22255,"Ġliberation":22256,"ĠAbe":22257,"ĠOL":22258,"Ġstuffed":22259,"aca":22260,"inda":22261,"raphic":22262,"Ġmosqu":22263,"Ġcampaigning":22264,"Ġoccupy":22265,"Squ":22266,"rina":22267,"ĠWel":22268,"ĠVS":22269,"Ġphysic":22270,"Ġpuls":22271,"rint":22272,"oaded":22273,"ETF":22274,"ĠArchives":22275,"Ġvenues":22276,"hner":22277,"ĠTurbo":22278,"Ġlust":22279,"Ġappealed":22280,"quez":22281,"ilib":22282,"ĠTimothy":22283,"Ġomn":22284,"dro":22285,"Ġobsession":22286,"ĠSavage":22287,"1996":22288,"Global":22289,"Jes":22290,"214":22291,"Ġsliding":22292,"Ġdisappro":22293,"ĠMagical":22294,"Ġvoluntarily":22295,"gb":22296,"aney":22297,"Ġprophet":22298,"ĠRein":22299,"ĠJulia":22300,"ĠWorth":22301,"aurus":22302,"Ġbounds":22303,"ieu":22304,")))":22305,"Ġcrore":22306,"ĠCitizen":22307,"Sky":22308,"Ġcolumnist":22309,"Ġseekers":22310,"ondo":22311,"ISA":22312,"ĠLength":22313,"Ġnostalg":22314,"Ġnewcom":22315,"Ġdetrim":22316,"entric":22317,"375":22318,"ĠGE":22319,"Ġautop":22320,"Ġacademics":22321,"AppData":22322,"ĠShen":22323,"Ġidiot":22324,"ĠTransit":22325,"Ġteaspoon":22326,"Wil":22327,"KO":22328,"ĠComedy":22329,">,":22330,"Ġpopulated":22331,"WD":22332,"Ġpigs":22333,"ĠOculus":22334,"Ġsympathetic":22335,"Ġmarathon":22336,"198":22337,"Ġseizure":22338,"sided":22339,"Ġdop":22340,"irtual":22341,"Land":22342,"ĠFloor":22343,"osaurs":22344,"...]":22345,"Ġlos":22346,"Ġsubsidiary":22347,"EY":22348,"ĠParts":22349,"ĠStef":22350,"ĠJudiciary":22351,"Ġ134":22352,"Ġmirrors":22353,"Ġket":22354,"times":22355,"Ġneurolog":22356,"Ġcav":22357,"ĠGuest":22358,"Ġtumor":22359,"scill":22360,"ĠLloyd":22361,"Est":22362,"Ġclearer":22363,"Ġstereotypes":22364,"Ġdur":22365,"nothing":22366,"Reddit":22367,"Ġnegotiated":22368,"------------------------":22369,"235":22370,"Ġflown":22371,"ĠSeoul":22372,"ĠResident":22373,"ĠSCH":22374,"Ġdisappearance":22375,"ĠVince":22376,"grown":22377,"Ġgrabs":22378,"ril":22379,"ĠInfinite":22380,"ĠTwenty":22381,"Ġpedestrian":22382,"Ġjersey":22383,"ĠFur":22384,"ĠInfinity":22385,"ĠElliott":22386,"Ġmentor":22387,"Ġmorally":22388,"Ġobey":22389,"secure":22390,"iffe":22391,"Ġantibiotics":22392,"angled":22393,"ĠFreeman":22394,"ĠIntroduction":22395,"Jun":22396,"Ġmarsh":22397,"icans":22398,"ĠEVENTS":22399,"ochond":22400,"Wall":22401,"iculty":22402,"Ġmisdemeanor":22403,"Ġly":22404,"Thomas":22405,"ĠResolution":22406,"Ġanimations":22407,"ĠDry":22408,"Ġintercourse":22409,"ĠNewcastle":22410,"ĠHog":22411,"ĠEquipment":22412,"177":22413,"Ġterritorial":22414,"Ġarchives":22415,"203":22416,"Filter":22417,"ĠMunich":22418,"Ġcommanded":22419,"ĠWand":22420,"Ġpitches":22421,"ĠCroat":22422,"Ġratios":22423,"ĠMits":22424,"Ġaccumulated":22425,"ĠSpecifically":22426,"Ġgentleman":22427,"acerb":22428,"Ġpenn":22429,"Ġaka":22430,"ĠFuk":22431,"Ġintervene":22432,"ĠRefuge":22433,"ĠAlzheimer":22434,"Ġsuccession":22435,"ohan":22436,"does":22437,"Lord":22438,"Ġseparat":22439,"Ġcorrespondence":22440,"Ġshiny":22441,"Prior":22442,"Ġsulf":22443,"Ġmiserable":22444,"Ġdedication":22445,"().":22446,"Ġspecialists":22447,"Ġdefects":22448,"ĠCult":22449,"ĠXia":22450,"Ġjeopard":22451,"ĠOre":22452,"Ability":22453,"Ġlear":22454,"Ġambitions":22455,"ĠBMI":22456,"ĠArabs":22457,"Ġ1942":22458,"Ġpreservation":22459,"ificate":22460,"Ġashamed":22461,"loss":22462,"ĠRestaur":22463,"Ġresemble":22464,"Ġenrich":22465,"ĠKN":22466,"ĠClan":22467,"float":22468,"Ġplayable":22469,"ITT":22470,"Ġharmony":22471,"arrison":22472,"ĠWeinstein":22473,"were":22474,"Ġpoisoning":22475,"ĠComput":22476,"ĠWordPress":22477,"major":22478,"ĠValve":22479,"Fan":22480,"ĠThrow":22481,"ĠRomans":22482,"ĠDepression":22483,"ados":22484,"Ġtortured":22485,"Ġbalancing":22486,"bottom":22487,"Ġacquiring":22488,"ĠMonte":22489,"ardi":22490,"Ġaura":22491,"Ġ##":22492,"ĠStanding":22493,"ĠAtlas":22494,"CF":22495,"Ġintrins":22496,"ĠBenghazi":22497,"Ġcamping":22498,"Ġtapped":22499,"blade":22500,"strous":22501,"ĠRabb":22502,"ĠWritten":22503,"tip":22504,"ĠNeigh":22505,"sterdam":22506,"ĠAllow":22507,"ĠHealing":22508,"ĠRhod":22509,"num":22510,"Ġcaffeine":22511,"ĠPercent":22512,"Ġboo":22513,"Ġapples":22514,"305":22515,"Ġwelcoming":22516,"Ġapplaud":22517,"Ġausterity":22518,"±":22519,"ĠReality":22520,"efe":22521,"å®":22522,"Ġsucks":22523,"Ġtabs":22524,"ĠPayPal":22525,"Ġbackpack":22526,"Ġgifted":22527,"abulary":22528,"ĠScout":22529,"irteen":22530,"Ġchin":22531,"Ġomitted":22532,"Ġnegatively":22533,"Ġaccessing":22534,"ĠEarn":22535,"Ġambulance":22536,"Ġheadphones":22537,"Ġ205":22538,"ĠRefresh":22539,"president":22540,"ĠKitchen":22541,"ĠEntered":22542,"ĠSnyder":22543,"005":22544,"omical":22545,"Ġborrowed":22546,"ĠNem":22547,"Ġaviation":22548,"Ġstall":22549,"rimination":22550,"Ġuniforms":22551,"itime":22552,"ĠSimmons":22553,"energy":22554,"ablished":22555,"yy":22556,"qualified":22557,"Ġrallies":22558,"ĠStuart":22559,"flight":22560,"Ġgangs":22561,"rag":22562,"Ġvault":22563,"lux":22564,"ĠCompar":22565,"Ġdesignation":22566,"209":22567,"ĠJos":22568,"dollar":22569,"zero":22570,"Ġwells":22571,"303":22572,"Ġconstituents":22573,"Ġheck":22574,"Ġcows":22575,"Ġcommanders":22576,"Ġdifferential":22577,"ĠCatherine":22578,"299":22579,"Ġvalve":22580,"Ġbrace":22581,"Ġperspectives":22582,"cert":22583,"fact":22584,"icularly":22585,"ĠMcN":22586,"planes":22587,"Ġintric":22588,"Ġpeas":22589,"ovan":22590,"Ġtossed":22591,"retch":22592,"ĠLopez":22593,"Ġunfamiliar":22594,"death":22595,"ĠApart":22596,"ĠChang":22597,"Ġrelieved":22598,"rophe":22599,"Ġairports":22600,"Ġfreak":22601,"util":22602,"Mill":22603,"ĠChin":22604,"ĠOwen":22605,"male":22606,"ĠBroken":22607,"ĠWinds":22608,"rob":22609,"rising":22610,"Ġfirefighters":22611,"Ġauthoritarian":22612,"Ġ148":22613,"Bitcoin":22614,"external":22615,"Ġbrowsers":22616,"ichever":22617,"orian":22618,"Ġunb":22619,"Ġpoke":22620,"ĠZot":22621,"Mid":22622,"ĠPopular":22623,"Ġcovert":22624,"Ġcontributes":22625,"Ġ650":22626,"Ġcontention":22627,"Gate":22628,"Ġconsoles":22629,"Ġchromos":22630,"ĠIX":22631,"Ġvisually":22632,"ĠEisen":22633,"Ġjewelry":22634,"Ġdelegation":22635,"Ġaccelerate":22636,"ĠRiley":22637,"Ġslope":22638,"Ġindoor":22639,"itially":22640,"Ġhugely":22641,"Ġtunnels":22642,"Ġfined":22643,"Ġdirective":22644,"Ġforehead":22645,"ustomed":22646,"Ġskate":22647,"Music":22648,"gas":22649,"Ġrecognizing":22650,"ambo":22651,"Ġoverweight":22652,"ĠGrade":22653,"ÙĬ":22654,"Ġsounding":22655,"Ġlocking":22656,"ĠREM":22657,"Store":22658,"Ġexcav":22659,"ĠLikewise":22660,"ĠLights":22661,"Ġelbow":22662,"ĠSupply":22663,"wic":22664,"Ġhandsome":22665,"1994":22666,"Coll":22667,"Ġadequately":22668,"ĠAssociate":22669,"Ġstrips":22670,"Ġcrackdown":22671,"Ġmarvel":22672,"ĠKun":22673,"Ġpassages":22674,"@@@@":22675,"ĠTall":22676,"Ġthoughtful":22677,"namese":22678,"Ġprostitution":22679,"business":22680,"Ġballistic":22681,"personal":22682,"cig":22683,"izational":22684,"Round":22685,"ĠÂłĠÂłĠÂłĠÂł":22686,"ĠColeman":22687,"Ġadmitting":22688,"ĠPlug":22689,"Ġbitcoins":22690,"ĠSuz":22691,"Ġfairness":22692,"Ġsupplier":22693,"Ġcatastrophic":22694,"ĠHelen":22695,"oqu":22696,"Marc":22697,"ĠArticles":22698,"gie":22699,"Ġendangered":22700,"Ġdestiny":22701,"ĠVolt":22702,"olia":22703,"axis":22704,"Ġcheat":22705,"Ġunified":22706,"ICO":22707,"quote":22708,"302":22709,"ĠSed":22710,"Ġsuppression":22711,"Ġanalyzing":22712,"Ġsquat":22713,"Ġfiguring":22714,"Ġcoordinates":22715,"Ġchunks":22716,"Ġ1946":22717,"Ġsubp":22718,"Ġwiki":22719,"ĠForbes":22720,"ĠJupiter":22721,"ĠErik":22722,"imer":22723,"ĠCommercial":22724,"\\)":22725,"Ġlegitimacy":22726,"Ġdental":22727,"ĠMean":22728,"Ġdeficits":22729,"550":22730,"Originally":22731,"ĠHorror":22732,"Ġcontamination":22733,"llah":22734,"Ġconfisc":22735,"ĠClare":22736,"TB":22737,"ĠFailed":22738,"aned":22739,"Ġruler":22740,"ĠController":22741,"Ġfeminists":22742,"Fix":22743,"gay":22744,"207":22745,"Ġrabbit":22746,"Third":22747,"owntown":22748,"Ġglue":22749,"Ġvolatile":22750,"Ġshining":22751,"Ġfoll":22752,"Ġimpaired":22753,"Ġsupers":22754,"æĪ":22755,"Ġclutch":22756,"ļéĨĴ":22757,"Ġprolet":22758,"Ġ(!":22759,"Ġyelled":22760,"ĠKiev":22761,"ĠErn":22762,"ĠShock":22763,"KB":22764,"Ġsituated":22765,"query":22766,"ĠNas":22767,"Ġannex":22768,"character":22769,"ĠHoliday":22770,"Ġautomation":22771,"ĠJill":22772,"ĠRemastered":22773,"Ġlinem":22774,"Ġwilderness":22775,"ĠHorizon":22776,"ĠGuinea":22777,"AZ":22778,"Ġmainland":22779,"Ġsecrecy":22780,"LEASE":22781,"Ġpunk":22782,"ĠProvince":22783,"(),":22784,"Speed":22785,"Ġhanding":22786,"ĠSebast":22787,"Sir":22788,"rase":22789,"Ġjournals":22790,"Ġcongest":22791,"ĠTut":22792,"irrel":22793,"Ġschizophrenia":22794,"Ġmisogyn":22795,"healthy":22796,"Iron":22797,"Ġreacted":22798,"-$":22799,"252":22800,"Ġplural":22801,"Ġplum":22802,"Ġbargain":22803,"Ġgrounded":22804,"finder":22805,"Ġdisse":22806,"ĠLaz":22807,"OOD":22808,"Ġatroc":22809,"Factory":22810,"Ġminions":22811,"Ġori":22812,"ĠBrave":22813,"ĠPRE":22814,"ĠMyanmar":22815,"ĠHod":22816,"Ġexpedition":22817,"Ġexplode":22818,"ĠCoord":22819,"Ġextr":22820,"ĠBrief":22821,"ĠADHD":22822,"Ġhardcore":22823,"feeding":22824,"Ġdile":22825,"ĠFruit":22826,"Ġvaccination":22827,"ĠMao":22828,"osphere":22829,"Ġcontests":22830,"-|":22831,"Ġfren":22832,"isphere":22833,"Rom":22834,"ĠSharp":22835,"ĠTrend":22836,"Ġdisconnect":22837,"âĢ¢âĢ¢":22838,"Ġpersecution":22839,"Earth":22840,"Ġhealthier":22841,"384":22842,"Ġcob":22843,"ĠTrinity":22844,"OWS":22845,"ANN":22846,"Ġspecialty":22847,"Ġgru":22848,"Ġcooperative":22849,"why":22850,"Starting":22851,"ĠIssues":22852,"stre":22853,"ensor":22854,"Ġ185":22855,"Adv":22856,"!?":22857,"ĠRevel":22858,"emia":22859,"ĠHulk":22860,"Ġcelebrations":22861,"ĠSou":22862,"raud":22863,"ĠKlein":22864,"Ġunreal":22865,"context":22866,"Ġpartnerships":22867,"Ġadopting":22868,"tical":22869,"Ġsplash":22870,"ĠHezbollah":22871,"category":22872,"cyclop":22873,"xton":22874,"ĠDot":22875,"urdy":22876,"tz":22877,"Ġenvelope":22878,"ĠNL":22879,"âķ":22880,"Ġwherein":22881,"Spec":22882,"184":22883,"Ġtelev":22884,"aliation":22885,"Ġmyths":22886,"å°":22887,"Ġrigorous":22888,"Ġcommunicating":22889,"Ġobserver":22890,"Ġrehe":22891,"ĠWash":22892,"Ġapologized":22893,"ĠTin":22894,"Ġexpenditures":22895,"workers":22896,"document":22897,"Ġhesitate":22898,"ĠLenin":22899,"Ġunpredictable":22900,"Ġrenewal":22901,"cler":22902,"okia":22903,"ĠCONT":22904,"Ġpostseason":22905,"Tokens":22906,"Ġexacerb":22907,"Ġbetting":22908,"Ġ147":22909,"Ġelevation":22910,"Wood":22911,"ĠSolomon":22912,"194":22913,"004":22914,"output":22915,"Ġredund":22916,"ĠMumbai":22917,"ĠpH":22918,"Ġreproduce":22919,"ĠDuration":22920,"MAX":22921,"Ġbog":22922,"CBS":22923,"ĠBalance":22924,"ĠSgt":22925,"ĠRecent":22926,"Ġcd":22927,"Ġpopped":22928,"Ġincompet":22929,"prop":22930,"ayan":22931,"guy":22932,"Pacific":22933,"Ġtyr":22934,"Ġ{{":22935,"ĠMystic":22936,"ĠDana":22937,"Ġmasturb":22938,"Ġgeometry":22939,"â":22940,"ĠCorrect":22941,"Ġtrajectory":22942,"Ġdistracted":22943,"Ġfoo":22944,"ĠWelsh":22945,"Luc":22946,"mith":22947,"Ġrugby":22948,"Ġrespiratory":22949,"Ġtriangle":22950,"Ġ215":22951,"Ġundergraduate":22952,"ĠSuperior":22953,"changing":22954,"_-":22955,"Ġrightly":22956,"Ġreferee":22957,"Ġlucrative":22958,"Ġunauthorized":22959,"Ġresembles":22960,"ĠGNU":22961,"ĠDerby":22962,"Ġpathways":22963,"ĠLed":22964,"Ġendurance":22965,"Ġstint":22966,"Ġcollector":22967,"Fast":22968,"Ġdots":22969,"Ġnationals":22970,"ĠSecurities":22971,"Ġwhip":22972,"Param":22973,"Ġlearns":22974,"Magic":22975,"Ġdetailing":22976,"moon":22977,"Ġbroadcasting":22978,"Ġbaked":22979,"265":22980,"holm":22981,"ĠSah":22982,"ĠHussein":22983,"ĠCourtesy":22984,"174":22985,"Ġ146":22986,"Ġgeographic":22987,"peace":22988,"Ġjudging":22989,"ĠStern":22990,"Bur":22991,"Ġstoryline":22992,"Gun":22993,"ĠStick":22994,"245":22995,"307":22996,"ãĤ´ãĥ³":22997,"ĠAdministrator":22998,"Ġburnt":22999,"Ġpave":23000,"choes":23001,"Exec":23002,"Ġcampuses":23003,"Result":23004,"Ġmutations":23005,"ĠCharter":23006,"Ġcaptures":23007,"Ġcompares":23008,"Ġbadge":23009,"Scient":23010,"Ġerad":23011,"iery":23012,"oi":23013,"ettes":23014,"ĠEstate":23015,"Ġstrap":23016,"Ġproudly":23017,"Ġfried":23018,"Ġwithdrawn":23019,"ĠVoy":23020,"phony":23021,"Items":23022,"ĠPierce":23023,"bard":23024,"Ġannotation":23025,"anton":23026,"illon":23027,"Impro":23028,"...)":23029,"Ġhappier":23030,"------":23031,"adjust":23032,"Ġstaffers":23033,"Ġactivism":23034,"Ġperf":23035,"Ġalright":23036,"Need":23037,"Ġcommence":23038,"Ġopioid":23039,"ĠAmanda":23040,"Es":23041,"ĠPars":23042,"ĠKaw":23043,"Works":23044,"248":23045,"Ġindo":23046,"tc":23047,"endant":23048,"ĠMoto":23049,"Ġlegalization":23050,"OTE":23051,"Ġtasked":23052,"Ġtsp":23053,"ĠACTIONS":23054,"166":23055,"Ġrefreshing":23056,"ĠNR":23057,"ĠPerez":23058,"Ġinfringement":23059,"SY":23060,"Listen":23061,"inning":23062,"ku":23063,"Ġrotate":23064,"program":23065,"arah":23066,"Design":23067,"Ġ(£":23068,"Ġstoring":23069,"Ġwarrants":23070,"Ġjudgement":23071,"ĠBrist":23072,"usually":23073,"photo":23074,"ĠRan":23075,"ĠPine":23076,"Ġoutrageous":23077,"ĠValentine":23078,"luence":23079,"ĠEverybody":23080,"Altern":23081,"Ġrelevance":23082,"Ġterminated":23083,"Ġdessert":23084,"Ġfulfilled":23085,"Ġprosecuted":23086,"ĠWords":23087,"Ġmigrant":23088,"Ġcultivation":23089,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":23090,"idelity":23091,"ĠVern":23092,"ĠLogin":23093,"Ġmetaphor":23094,"ĠTip":23095,"Ġrecruits":23096,"ĠPig":23097,"ribing":23098,"Ġenthusiasts":23099,"exper":23100,"Ġfrightening":23101,"ĠHair":23102,"anson":23103,"strate":23104,"Ġhi":23105,"Height":23106,"Ġowning":23107,"none":23108,"Ġdislike":23109,"Ġknives":23110,"pherd":23111,"Ġloudly":23112,"ĠAPIs":23113,"Display":23114,"ĠLac":23115,"ĠUSS":23116,"abl":23117,"verages":23118,"Jew":23119,"Ġ172":23120,"ĠHistorical":23121,"atoon":23122,"ĠPhysics":23123,"intern":23124,"Ġwarmth":23125,"Ġtopp":23126,"DM":23127,"Ġgunman":23128,"Ġemperor":23129,"odi":23130,"ãĥ£":23131,"inatory":23132,"ĠRib":23133,"Ġ131":23134,"ĠSaturn":23135,"ĠShining":23136,"Ġwaking":23137,"Quotes":23138,"Ġcomedian":23139,"enberg":23140,"½":23141,"Ġbelievers":23142,"Ġpaperwork":23143,"custom":23144,"Ġlev":23145,"Ġlament":23146,"Ġpouring":23147,"222":23148,"political":23149,"ĠSupplement":23150,"maid":23151,"Ġcruelty":23152,"Ġtread":23153,"ysics":23154,"Aw":23155,"rites":23156,"Ġmodifier":23157,"ĠPosition":23158,"Adam":23159,"lb":23160,"ubs":23161,"Ġimperfect":23162,"Ġclusters":23163,"ĠEngineer":23164,"ĠCherry":23165,"Ġinauguration":23166,"ĠSau":23167,"Ġembodiment":23168,"ĠUncle":23169,"Ġoverr":23170,"Ġexplosions":23171,"cule":23172,"ĠPrinceton":23173,"ĠAndrea":23174,"Ġincorrectly":23175,"Ġearnest":23176,"Ġpilgr":23177,"ĠSprint":23178,"Ġsleeve":23179,"Ġhears":23180,"ĠAmazing":23181,"Ġbrowsing":23182,"agin":23183,"Ġhomeland":23184,"Ġhaw":23185,"Ġdiving":23186,"istered":23187,"178":23188,"Ġbargaining":23189,"ĠArcade":23190,"Ġdelegate":23191,"terson":23192,"................................................................":23193,"ĠJacksonville":23194,"275":23195,"Ġstagn":23196,"Ġadam":23197,"ĠSherman":23198,"CB":23199,"Ġsuburb":23200,"ĠFoods":23201,"Ġconverting":23202,"ĠArist":23203,"Ġchambers":23204,"love":23205,"Ġamino":23206,"ĠGan":23207,"Ġmadness":23208,"mc":23209,"ĠUSE":23210,"defined":23211,"Ġultr":23212,"indust":23213,"Ġwolves":23214,"lance":23215,"Additionally":23216,"Ġcracks":23217,"asia":23218,"ĠReason":23219,"ĠPump":23220,"Ġaccidental":23221,"ĠLaser":23222,"ĠRid":23223,"Ġinitialized":23224,"elli":23225,"Ġunnamed":23226,"Ġnoun":23227,"ĠPassed":23228,"Ġhostage":23229,"ĠEthiop":23230,"shirts":23231,"Ġunrel":23232,"ĠEmbassy":23233,"Ġ1941":23234,"Ġatoms":23235,"Ġpurported":23236,"164":23237,"ĠFi":23238,"Ġgallons":23239,"ĠMonica":23240,"Ġpg":23241,"enment":23242,"Ġsorted":23243,"ĠGospel":23244,"Ġheights":23245,"Ġtraced":23246,"Ġundergoing":23247,"Shell":23248,"Ġsacks":23249,"Ġproportions":23250,"Ġhalluc":23251,"Font":23252,"acet":23253,"Ġwarmer":23254,"ĠINTER":23255,"Ġgrabbing":23256,"Plug":23257,"Ġrealization":23258,"ĠBurke":23259,"Ġenchant":23260,"ATER":23261,"ĠSeed":23262,"Ġabundant":23263,"FM":23264,"Ġcivic":23265,"Vs":23266,"isi":23267,"Ġvow":23268,"Ġreper":23269,"ĠPartnership":23270,"Ġpenetration":23271,"Ġaxe":23272,"Ġshattered":23273,"ĠZombies":23274,"Ġvinyl":23275,"ĠAlert":23276,"eon":23277,"Ġobliged":23278,"ĠIllust":23279,"ĠPlaza":23280,"ĠFrontier":23281,"Ġdavidjl":23282,"ĠSerial":23283,"ĠHav":23284,"ĠNutrition":23285,"Bi":23286,"ĠâĸĪ":23287,"ĠJays":23288,"linux":23289,"Ġhurry":23290,"Ġvoy":23291,"Ġhopeless":23292,"ĠStealth":23293,"Ġãģ":23294,"essors":23295,"ttle":23296,"borg":23297,"ĠSafari":23298,"fell":23299,"Ġwary":23300,"due":23301,"ĠAbove":23302,"Ha":23303,"ELL":23304,"Ġnotor":23305,"ĠWon":23306,"Too":23307,"Ġoccupations":23308,"Ġpossessions":23309,"Ġinviting":23310,"Ġpredators":23311,"Ġaccelerated":23312,"Ġ157":23313,"uterte":23314,"ĠCube":23315,"east":23316,"account":23317,"Give":23318,"Ġtransplant":23319,"redients":23320,"idable":23321,"Ġscreenshots":23322,"ĠGund":23323,"ĠFS":23324,"Ġtravelers":23325,"Ġsensory":23326,"ĠFiat":23327,"ĠRockets":23328,"İĭ":23329,"_{":23330,"Friend":23331,"Ġcharming":23332,"ALS":23333,"Ġenjoyment":23334,"mph":23335,"Ġ5000":23336,"ĠREG":23337,"ÙĨ":23338,"bia":23339,"Ġcompilation":23340,"rost":23341,"ĠVP":23342,"ĠSchne":23343,"2019":23344,"Ġcopying":23345,"MORE":23346,"ĠFlore":23347,"falls":23348,"215":23349,"total":23350,"Ġdisciples":23351,"double":23352,"Ġexceeding":23353,"Ġsmashed":23354,"Ġconceptual":23355,"ĠRomania":23356,"ĠBrent":23357,"ĠICE":23358,"ĠTou":23359,"Ġgrap":23360,"Ġnails":23361,"189":23362,"ãĥĺ":23363,"Ġprocure":23364,"eur":23365,"Ġconfirming":23366,"ĠCec":23367,"awi":23368,"ĠEden":23369,"Ġng":23370,"Ġengineered":23371,"atics":23372,"Ġhooked":23373,"Ġdisgusting":23374,"ĠMurder":23375,"ãĤ¿":23376,"Library":23377,"Ġ168":23378,"Almost":23379,"hematic":23380,"Menu":23381,"ĠNotre":23382,"ĠJur":23383,"Ġkidnapped":23384,"Ġhacker":23385,"ĠJade":23386,"Ġcreepy":23387,"Ġdrawings":23388,"ĠSponsor":23389,"Ġcyclists":23390,"ĠGoblin":23391,"Ġoptimized":23392,"Ġstaged":23393,"ĠMcD":23394,"between":23395,"Age":23396,"eno":23397,"Sex":23398,"ĠWide":23399,"nings":23400,"avis":23401,"Ġincapable":23402,"ĠKob":23403,"Ġrewarding":23404,"ĠLone":23405,"olescent":23406,"Ġcontracted":23407,"Ġsticky":23408,"Jose":23409,"Ball":23410,"fest":23411,"ĠInput":23412,"ĠRecently":23413,"Ġtomat":23414,"square":23415,"Application":23416,"Ġnitrogen":23417,"Ġduplicate":23418,"ĠRecon":23419,"ĠDear":23420,"London":23421,"Ġintra":23422,"Ġdock":23423,"Ġoutreach":23424,"ĠMillion":23425,"Ġmammals":23426,"ampton":23427,"VAL":23428,"Ġsnaps":23429,"Ġdos":23430,"ĠWhole":23431,"ĠReady":23432,"Try":23433,"ĠWinnipeg":23434,"earance":23435,"Ġincurred":23436,"renched":23437,"ĠNSW":23438,"ilot":23439,"raine":23440,"Ġcube":23441,"got":23442,"Ġrunway":23443,"etermined":23444,"ĠHawks":23445,"Ġsurvivor":23446,"ĠWish":23447,"ĠDin":23448,"ĠDEF":23449,"ĠVault":23450,"187":23451,"Ġmushrooms":23452,"Ġcrisp":23453,"bey":23454,"ĠDiscovery":23455,"Ġdevelopmental":23456,"Ġparadigm":23457,"Ġchaotic":23458,"ĠTsu":23459,"Ġ333":23460,"bons":23461,"Ġbacterial":23462,"Ġcommits":23463,"Ġcosmic":23464,"Ġmega":23465,"ocative":23466,"ĠPaint":23467,"ophobic":23468,"Ġvain":23469,"Ġcarved":23470,"ĠThief":23471,"ĠGul":23472,"owship":23473,"Ġcites":23474,"ĠEdinburgh":23475,"Ġdiminished":23476,"Ġacknowledges":23477,"ĠKills":23478,"Ġmicrow":23479,"ĠHera":23480,"Ġseniors":23481,"Ġwhereby":23482,"Hop":23483,"atron":23484,"Ġunavailable":23485,"ĠNate":23486,"Ġ480":23487,"Ġslated":23488,"ĠRebecca":23489,"ĠBattery":23490,"Ġgrammar":23491,"Ġheadset":23492,"Ġcursor":23493,"Ġexcluding":23494,"anye":23495,"aundering":23496,"ebin":23497,"Ġfeasible":23498,"ĠPublishing":23499,"ĠLabs":23500,"ĠCliff":23501,"ĠFerrari":23502,"Ġpac":23503,"visible":23504,"marked":23505,"pell":23506,"Ġpolite":23507,"Ġstaggering":23508,"ĠGalactic":23509,"Ġsuperst":23510,"Ġparan":23511,"ĠOfficers":23512,"ãĢģ":23513,"Ġspecifics":23514,"ulus":23515,"239":23516,"ĠPaste":23517,"AMP":23518,"ĠPanama":23519,"ĠDelete":23520,"anguard":23521,"restrial":23522,"Ġheroic":23523,"ĠDy":23524,"اÙĦ":23525,"Ġincumbent":23526,"Ġcrunch":23527,"tro":23528,"Ġscoop":23529,"Ġblogger":23530,"Ġsellers":23531,"uren":23532,"Ġmedicines":23533,"ĠCaps":23534,"ĠAnimation":23535,"oxy":23536,"Ġoutward":23537,"Ġinquiries":23538,"229":23539,"Ġpsychologist":23540,"ĠSask":23541,"evil":23542,"Ġcontaminated":23543,"ãĤ¨":23544,"herence":23545,"Ġbranded":23546,"ĠAbdul":23547,"zh":23548,"Ġparagraphs":23549,"Ġmins":23550,"Ġcorrelated":23551,"erb":23552,"Ġimpart":23553,"Ġmilestone":23554,"ĠSolutions":23555,"otle":23556,"Ġundercover":23557,"Ġmarched":23558,"ĠChargers":23559,"fax":23560,"ĠSecrets":23561,"Ġruth":23562,"weather":23563,"Ġfeminine":23564,"Ġsham":23565,"Ġprestigious":23566,"iggins":23567,"Ġsung":23568,"history":23569,"ettle":23570,"ggie":23571,"Ġoutdated":23572,"oland":23573,"Ġperceptions":23574,"ĠSession":23575,"ĠDodgers":23576,"uj":23577,"ĠEND":23578,"Doc":23579,"Ġdeficiency":23580,"Grand":23581,"ĠJoker":23582,"Ġretrospect":23583,"Ġdiagnostic":23584,"Ġharmless":23585,"Ġrogue":23586,"ĠAval":23587,"Equ":23588,"Ġtransc":23589,"ĠRobertson":23590,"ĠDepending":23591,"ĠBurns":23592,"ivo":23593,"Ġhostility":23594,"Features":23595,"ĵĺ":23596,"Ġdiscomfort":23597,"ĠLCD":23598,"specified":23599,"ĠExpect":23600,"340":23601,"Ġimperative":23602,"ĠRegular":23603,"Chinese":23604,"Ġstatewide":23605,"Ġsymm":23606,"Ġloops":23607,"Ġautumn":23608,"Nick":23609,"Ġshaping":23610,"Ġquot":23611,"Ġcherry":23612,"ĠCrossref":23613,"è¦ļéĨĴ":23614,"Standard":23615,"heed":23616,"ĠDell":23617,"ĠVietnamese":23618,"Ġost":23619,"ĠValkyrie":23620,"OA":23621,"Assad":23622,"Ġrebound":23623,"ĠTraffic":23624,"places":23625,"æĺ":23626,"ĠBuc":23627,"172":23628,"Ġshelters":23629,"Ġinsisting":23630,"ĠCertainly":23631,"ĠKenneth":23632,"ĠTCP":23633,"Ġpenal":23634,"ĠReplay":23635,"heard":23636,"Ġdialect":23637,"iza":23638,"ĠFY":23639,"itcher":23640,"ĠDL":23641,"Ġspiral":23642,"Ġquarterbacks":23643,"Ġhull":23644,"Ġgoogle":23645,"Ġtodd":23646,"ĠSterling":23647,"ĠPlate":23648,"Ġspying":23649,"mbol":23650,"ĠRealm":23651,"ĠProced":23652,"ĠCrash":23653,"Ġterminate":23654,"Ġprotesting":23655,"Center":23656,"guided":23657,"Ġuncover":23658,"Ġboycott":23659,"Ġrealizes":23660,"sound":23661,"Ġpretending":23662,"ĠVas":23663,"1980":23664,"Ġframed":23665,"Ġ139":23666,"Ġdescended":23667,"Ġrehabilitation":23668,"Ġborrowing":23669,"ĠBuch":23670,"Ġblur":23671,"Ron":23672,"ĠFrozen":23673,"enza":23674,"Chief":23675,"ĠPoor":23676,"Ġtranslates":23677,"MIN":23678,"Ġ212":23679,"JECT":23680,"Ġerupted":23681,"Ġsuccesses":23682,"SEC":23683,"Ġplague":23684,"Ġgems":23685,"doms":23686,"Ġstretches":23687,"ĠSpy":23688,"Ġstorytelling":23689,"Credit":23690,"ĠPush":23691,"Ġtraction":23692,"Ġineffective":23693,"ĠLuna":23694,"Ġtapes":23695,"Ġanalytics":23696,"ercise":23697,"Ġprogrammes":23698,"ĠCarbon":23699,"Ġbehold":23700,"heavy":23701,"ĠConservation":23702,"ĠFIR":23703,"Ġsack":23704,"termin":23705,"ricks":23706,"Ġhoused":23707,"Ġunusually":23708,"Ice":23709,"Ġexecuting":23710,"ĠMoroc":23711,"eday":23712,"Ġeditions":23713,"Ġsmarter":23714,"ĠBA":23715,"Ġoutlaw":23716,"Ġvanished":23717,"iba":23718,"ALSE":23719,"ĠSilva":23720,"238":23721,"Could":23722,"Ġphilosopher":23723,"Ġevacuated":23724,"Secret":23725,"142":23726,"Ġvisas":23727,"ãĤ¬":23728,"ĠMalt":23729,"ĠClearly":23730,"ĠNiger":23731,"ĠCairo":23732,"ĠFist":23733,"380":23734,"ĠXML":23735,"auto":23736,"itant":23737,"Ġreinforced":23738,"Record":23739,"ĠSurvivor":23740,"GHz":23741,"Ġscrews":23742,"parents":23743,"Ġoceans":23744,"mares":23745,"Ġbrakes":23746,"vasive":23747,"Ġhello":23748,"ĠSIM":23749,"rimp":23750,"Ġore":23751,"ĠArmour":23752,"247":23753,"Ġterrific":23754,"Ġtones":23755,"141":23756,"ĠMinutes":23757,"Episode":23758,"Ġcurves":23759,"Ġinflammatory":23760,"Ġbatting":23761,"ĠBeautiful":23762,"Lay":23763,"Ġunpop":23764,"vable":23765,"Ġriots":23766,"ĠTactics":23767,"baugh":23768,"ĠCock":23769,"Ġorgasm":23770,"ĠSas":23771,"Ġconstructor":23772,"etz":23773,"Gov":23774,"Ġantagon":23775,"Ġtheat":23776,"Ġdeeds":23777,"hao":23778,"cuts":23779,"ĠMcCl":23780,"Ġum":23781,"ĠScientists":23782,"Ġgrassroots":23783,"yssey":23784,"\"]=>":23785,"Ġsurfaced":23786,"Ġshades":23787,"Ġneighbours":23788,"Ġadvertis":23789,"oya":23790,"Ġmerged":23791,"Upon":23792,"Ġgad":23793,"Ġanticipate":23794,"Anyway":23795,"Ġslogan":23796,"Ġdisrespect":23797,"Iran":23798,"ĠTB":23799,"acted":23800,"Ġsubpoen":23801,"mediately":23802,"OOOO":23803,"Ġwaiver":23804,"Ġvulnerabilities":23805,"ottesville":23806,"ĠHuffington":23807,"Josh":23808,"ĠDH":23809,"Monday":23810,"ĠEllen":23811,"Know":23812,"xon":23813,"items":23814,"228":23815,"Ġfills":23816,"ĠNike":23817,"Ġcumulative":23818,"andals":23819,"Ir":23820,"Ġì":23821,"Ġfriction":23822,"igator":23823,"Ġscans":23824,"ĠVienna":23825,"ldom":23826,"Ġperformers":23827,"Prim":23828,"Ġbidding":23829,"Mur":23830,"Ġleaned":23831,"ĠPrix":23832,"alks":23833,"Ġ[â̦]":23834,"ĠTwitch":23835,"ĠDeveloper":23836,"ĠGir":23837,"Ġcallback":23838,"Abstract":23839,"Ġaccustomed":23840,"Ġfreedoms":23841,"ĠPG":23842,"uracy":23843,"Ġlump":23844,"isman":23845,",,,,":23846,"1992":23847,"ĠRED":23848,"Ġworm":23849,"Match":23850,"ĠPlatinum":23851,"IJ":23852,"ĠOwner":23853,"Trivia":23854,"compl":23855,"Ġnewborn":23856,"Ġfantas":23857,"Own":23858,"Ġ1959":23859,"Ġsympath":23860,"Ġubiqu":23861,"Ġoutputs":23862,"Ġallev":23863,"Ġprag":23864,"Kevin":23865,"Ġfavors":23866,"Ġburial":23867,"Ġnurt":23868,"solete":23869,"cache":23870,"Ġ156":23871,"Ġunlocks":23872,"techn":23873,"Making":23874,"Ġconquer":23875,"adic":23876,"æĸ":23877,"Ġelf":23878,"Ġelectorate":23879,"ĠKurds":23880,"ĠStack":23881,"ĠSamurai":23882,"Ġâĺħ":23883,"Ġ{}":23884,"ĠSaid":23885,"ĠFallout":23886,"Ġkindness":23887,"ĠCustoms":23888,"ĠBoulevard":23889,"Ġhelicopters":23890,"otics":23891,"ĠVeget":23892,"comment":23893,"Ġcriticised":23894,"Ġpolished":23895,"ĠRemix":23896,"ĠCultural":23897,"Ġrecons":23898,"Ġdoi":23899,"atem":23900,"Screen":23901,"Ġbarred":23902,"Comments":23903,"ĠGenerally":23904,"Ġslap":23905,"720":23906,"Vari":23907,"pine":23908,"Ġempt":23909,"Ġhats":23910,"ĠPlaying":23911,"lab":23912,"average":23913,"forms":23914,"ĠCotton":23915,"Ġcans":23916,"ĠDON":23917,"ĠSomalia":23918,"Crypt":23919,"ĠIncreases":23920,"Ever":23921,"modern":23922,"Ġsurgeon":23923,"3000":23924,"Ġrandomized":23925,"================================================================":23926,"Bern":23927,"impl":23928,"ĠCOR":23929,"Ġproclaim":23930,"thouse":23931,"Ġtoes":23932,"Ġample":23933,"Ġpreserving":23934,"Ġdisbel":23935,"grand":23936,"Besides":23937,"Ġsilk":23938,"ĠPattern":23939,"hm":23940,"Ġenterprises":23941,"Ġaffidavit":23942,"ĠAdvisory":23943,"Ġadvertised":23944,"ĠReligious":23945,"sections":23946,"psych":23947,"ĠFields":23948,"aways":23949,"Ġhashtag":23950,"ĠNightmare":23951,"Ġvampire":23952,"Ġforensic":23953,"rossover":23954,"nar":23955,"Ġnavy":23956,"Ġvacant":23957,"ĠDuel":23958,"Ġhallway":23959,"Ġfacebook":23960,"identally":23961,"ĠNRA":23962,"Ġmatt":23963,"Ġhurricane":23964,"ĠKirby":23965,"ĠPuzzle":23966,"Ġskirt":23967,"oust":23968,"dullah":23969,"Ġanalogy":23970,"inion":23971,"Ġtomatoes":23972,"ĠNV":23973,"ĠPeak":23974,"ĠMeyer":23975,"Ġappointments":23976,"Ġmasc":23977,"Ġalley":23978,"rehend":23979,"Ġcharities":23980,"Ġundo":23981,"Ġdestinations":23982,"ĠTesting":23983,"\">\"":24618,"cats":24619,"*.":24620,"Ġgestures":24621,"general":24622,"League":24623,"Ġpackets":24624,"ĠInspector":24625,"ĠBerg":24626,"Ġfraudulent":24627,"Ġcriticize":24628,"Fun":24629,"Ġblaming":24630,"ndra":24631,"Ġslash":24632,"ĠEston":24633,"Ġproposing":24634,"Ġwhales":24635,"Ġtherapist":24636,"Ġsubset":24637,"Ġleisure":24638,"ELD":24639,"ĠCVE":24640,"ĠActivity":24641,"Ġculmin":24642,"shop":24643,"ĠDAY":24644,"ischer":24645,"ĠAdmiral":24646,"ĠAttacks":24647,"Ġ1958":24648,"Ġmemoir":24649,"Ġfolded":24650,"Ġsexist":24651,"Ġ153":24652,"ĠLI":24653,"Ġreadings":24654,"Ġembarrassment":24655,"ĠEmployment":24656,"wart":24657,"chin":24658,"Ġcontinuation":24659,"lia":24660,"Recently":24661,"Ġduel":24662,"Ġevacuation":24663,"ĠKashmir":24664,"Ġdisposition":24665,"ĠRig":24666,"Ġbolts":24667,"Ġinsurers":24668,"467":24669,"Mex":24670,"Ġretaliation":24671,"Ġmisery":24672,"Ġunreasonable":24673,"raining":24674,"Imm":24675,"ĠPU":24676,"emer":24677,"Ġgenital":24678,"ãĤ³":24679,"ĠCandy":24680,"Ġonions":24681,"ĠPatt":24682,"liner":24683,"Ġconceded":24684,"Ġfa":24685,"Ġforc":24686,"ĠHernandez":24687,"ĠGeoff":24688,"debian":24689,"ĠTeams":24690,"Ġcries":24691,"Ġhomeowners":24692,"237":24693,"ABC":24694,"Ġstitch":24695,"Ġstatistic":24696,"Ġheaders":24697,"ĠBiology":24698,"Ġmotors":24699,"ĠGEN":24700,"ĠLip":24701,"Ġhates":24702,"Ġheel":24703,"Self":24704,"ipl":24705,"EDIT":24706,"orting":24707,"Ġannot":24708,"ĠSpeech":24709,"oldemort":24710,"ĠJavascript":24711,"ĠLeBron":24712,"Ġfootprint":24713,"Ġfn":24714,"Ġseizures":24715,"nas":24716,"hide":24717,"Ġ1954":24718,"ĠBee":24719,"ĠDeclaration":24720,"ĠKatie":24721,"Ġreservations":24722,"NR":24723,"female":24724,"Ġsaturated":24725,"Ġbiblical":24726,"Ġtrolls":24727,"Device":24728,"photos":24729,"Ġdrums":24730,"ãĥīãĥ©ãĤ´ãĥ³":24731,"Night":24732,"fighter":24733,"ĠHak":24734,"riber":24735,"Ġcush":24736,"Ġdisciplinary":24737,"baum":24738,"ĠGH":24739,"ĠSchmidt":24740,"ilibrium":24741,"Ġsixty":24742,"ĠKushner":24743,"rots":24744,"Ġpund":24745,"ĠRac":24746,"Ġsprings":24747,"Ġconve":24748,"Business":24749,"Fall":24750,"Ġqualifications":24751,"Ġverses":24752,"Ġnarciss":24753,"ĠKoh":24754,"ĠWow":24755,"ĠCharlottesville":24756,"edo":24757,"Ġinterrogation":24758,"ĠWool":24759,"365":24760,"Brian":24761,"Ġâľĵ":24762,"Ġalleges":24763,"onds":24764,"idation":24765,"ĠJackie":24766,"yu":24767,"Ġlakes":24768,"Ġworthwhile":24769,"Ġcrystals":24770,"ĠJuda":24771,"Ġcomprehend":24772,"Ġflush":24773,"Ġabsorption":24774,"ĠOC":24775,"Ġfrightened":24776,"ĠChocolate":24777,"Martin":24778,"Ġbuys":24779,"Ġbucks":24780,"Ġappell":24781,"ĠChampionships":24782,"Ġlistener":24783,"ĠDefensive":24784,"Ġcz":24785,"uds":24786,"ĠMate":24787,"Ġreplay":24788,"Ġdecorated":24789,"Ġsunk":24790,"ĠVIP":24791,"ĠAnk":24792,"Ġ195":24793,"aaaa":24794,"Nobody":24795,"ĠMilk":24796,"ĠGur":24797,"ĠMk":24798,"ĠSara":24799,"Ġseating":24800,"ĠWid":24801,"Track":24802,"Ġemploys":24803,"Ġgigantic":24804,"APP":24805,"ãĤ§":24806,"inventory":24807,"Ġtowel":24808,"atche":24809,"lasting":24810,"ĠTL":24811,"Ġlatency":24812,"Ġkne":24813,"Ber":24814,"meaning":24815,"Ġupheld":24816,"Ġplayground":24817,"Ġmant":24818,"Side":24819,"Ġstereo":24820,"Ġnorthwest":24821,"Ġexceptionally":24822,"Ġrays":24823,"Ġrecurring":24824,"Drive":24825,"Ġupright":24826,"Ġabduct":24827,"ĠMarathon":24828,"Ġgoodbye":24829,"Ġalphabet":24830,"hp":24831,"Ġcourtroom":24832,"rington":24833,"othing":24834,"Tag":24835,"Ġdiplomats":24836,"Ġbarbar":24837,"ĠAqua":24838,"183":24839,"3333":24840,"Ġmaturity":24841,"Ġinstability":24842,"ĠApache":24843,"Ġ===":24844,"Ġfasting":24845,"ĠGrid":24846,"ModLoader":24847,"Ġ152":24848,"Abs":24849,"ĠOperating":24850,"etti":24851,"Ġacquaint":24852,"Donnell":24853,"ĠKem":24854,"ĠForge":24855,"Ġarmored":24856,"Mil":24857,"Ġphilosophers":24858,"invest":24859,"Players":24860,"âĪ":24861,"Ġmyriad":24862,"Ġcomrades":24863,"Rot":24864,"Ġremembering":24865,"Ġcorresponds":24866,"Ġprogrammers":24867,"ĠLynn":24868,"Ġolig":24869,"Ġcoherent":24870,"ynchron":24871,"ĠChemical":24872,"Ġjugg":24873,"pair":24874,"posts":24875,"Eye":24876,"ĠInner":24877,"Ġsemester":24878,"ottest":24879,"ĠEmirates":24880,"ricanes":24881,"orously":24882,"mits":24883,"ĠWis":24884,"Ġdodge":24885,"location":24886,"Ġfaded":24887,"Amazon":24888,"ĠProceed":24889,"ĠINFO":24890,"journal":24891,"ĠTruck":24892,"Ten":24893,"Ġ217":24894,"Ġstatutes":24895,"mobile":24896,"ĠTypes":24897,"Recomm":24898,"buster":24899,"pex":24900,"Ġlegends":24901,"Ġheadache":24902,"faced":24903,"ĠWiFi":24904,"ifty":24905,"ĠHER":24906,"Ġcircuits":24907,"ERROR":24908,"226":24909,"olin":24910,"Ġcylinder":24911,"ospace":24912,"ikers":24913,"Prem":24914,"Quant":24915,"Ġconflicting":24916,"Ġslightest":24917,"Ġforged":24918,"ionage":24919,"Stephen":24920,"ĠKub":24921,"ĠOpportun":24922,"ĠHeal":24923,"Ġblo":24924,"Ġrulers":24925,"Ġhuh":24926,"Ġsubmarine":24927,"fy":24928,"asser":24929,"Ġallowance":24930,"ĠKasich":24931,"ĠTas":24932,"ĠAustralians":24933,"ForgeModLoader":24934,"ĠâĨij":24935,"ĠMatrix":24936,"amins":24937,"Ġ1200":24938,"ĠAcqu":24939,"236":24940,"Document":24941,"ĠBreaking":24942,"193":24943,"ĠSubst":24944,"ĠRoller":24945,"ĠProperties":24946,"ĠNI":24947,"tier":24948,"Ġcrushing":24949,"Ġadvocating":24950,"Furthermore":24951,"keepers":24952,"Ġsexism":24953,"xd":24954,"Ġcaller":24955,"ĠSense":24956,"chieve":24957,"ĠTF":24958,"Ġfueled":24959,"Ġreminiscent":24960,"Ġobsess":24961,"urst":24962,"Ġuphold":24963,"ĠFans":24964,"hetics":24965,"ĠâĹ":24966,"ĠBath":24967,"Ġbeverage":24968,"Ġoscill":24969,"254":24970,"Ġpoles":24971,"Ġgradual":24972,"Ġexting":24973,"ĠSuff":24974,"ĠSuddenly":24975,"Ġliking":24976,"Ġ1949":24977,"unciation":24978,"amination":24979,"ĠOmar":24980,"ĠLV":24981,"ĠConsequently":24982,"Ġsynthes":24983,"ĠGIF":24984,"Ġpains":24985,"Ġinteracting":24986,"uously":24987,"incre":24988,"Ġrumor":24989,"ĠScientology":24990,"197":24991,"ĠZig":24992,"Ġspelling":24993,"ĠASS":24994,"Ġextingu":24995,"mson":24996,"Ġgh":24997,"Ġremarked":24998,"ĠStrategic":24999,"ĠMON":25000,"å¥":25001,"gae":25002,"ĠWHAT":25003,"Eric":25004,"ĠCampus":25005,"Ġmethane":25006,"Ġimagin":25007,"JUST":25008,"ĠAlm":25009,"XT":25010,"iq":25011,"ĠRSS":25012,"Ġwrongdoing":25013,"atta":25014,"Ġbigot":25015,"Ġdemonstrators":25016,"ĠCalvin":25017,"ĠVilla":25018,"Ġmembrane":25019,"ĠAwesome":25020,"Ġbenefic":25021,"268":25022,"Ġmagnificent":25023,"ĠLots":25024,"Greg":25025,"ĠBoris":25026,"Ġdetainees":25027,"ĠHerman":25028,"Ġwhispered":25029,"Ġawe":25030,"Professor":25031,"funding":25032,"Ġphysiological":25033,"ĠDestruction":25034,"Ġlimb":25035,"Ġmanipulated":25036,"Ġbubbles":25037,"Ġpseud":25038,"Ġhydra":25039,"ĠBristol":25040,"Ġstellar":25041,"ĠExpansion":25042,"ĠKell":25043,"ĠInterestingly":25044,"Ġmans":25045,"Ġdragging":25046,"Ġecological":25047,"ĠFit":25048,"Ġgent":25049,"Ġbenefited":25050,"ĠHaiti":25051,"Ġpolyg":25052,"ãĥİ":25053,"Ġ2030":25054,"Ġprow":25055,"Ġreconstruction":25056,"Ġwast":25057,"Ġpsychic":25058,"ĠGreeks":25059,"Handler":25060,"162":25061,"ĠPulse":25062,"Ġsolicit":25063,"Ġsys":25064,"Ġinflux":25065,"ĠGentle":25066,"percent":25067,"Ġproliferation":25068,"Ġtaxable":25069,"Ġdisregard":25070,"Ġescaping":25071,"Ġginger":25072,"Ġwithstand":25073,"Ġdevastated":25074,"ĠDew":25075,"series":25076,"Ġinjected":25077,"elaide":25078,"Ġturnover":25079,"heat":25080,"ĻĤ":25081,"Happy":25082,"ĠSilent":25083,"ãĤŃ":25084,"ivism":25085,"Ġirrational":25086,"AMA":25087,"Ġreef":25088,"rub":25089,"Ġ162":25090,"Ġbankers":25091,"ĠEthics":25092,"vv":25093,"Ġcriticisms":25094,"Kn":25095,"186":25096,"Movie":25097,"ĠTories":25098,"Ġnood":25099,"Ġdistortion":25100,"False":25101,"odore":25102,"Ġtasty":25103,"Research":25104,"ĠUID":25105,"-)":25106,"Ġdivorced":25107,"ĠMU":25108,"ĠHayes":25109,"ĠIsn":25110,"iani":25111,"ĠHQ":25112,"Ġ\"#":25113,"ignant":25114,"Ġtraumatic":25115,"ĠLing":25116,"Hun":25117,"Ġsabot":25118,"online":25119,"random":25120,"Ġrenamed":25121,"rared":25122,"KA":25123,"dead":25124,"ét":25125,"ĠAssistance":25126,"Ġseaf":25127,"++++++++":25128,"Ġseldom":25129,"ĠWebb":25130,"Ġboolean":25131,"ulet":25132,"Ġrefrain":25133,"ĠDIY":25134,"rule":25135,"Ġshutting":25136,"Ġutilizing":25137,"loading":25138,"ĠParam":25139,"coal":25140,"ooter":25141,"Ġattracting":25142,"ĠDol":25143,"Ġhers":25144,"agnetic":25145,"ĠReach":25146,"imo":25147,"Ġdiscarded":25148,"ĠPip":25149,"015":25150,"ür":25151,"Ġmug":25152,"Imagine":25153,"COL":25154,"Ġcursed":25155,"ĠShows":25156,"ĠCurtis":25157,"ĠSachs":25158,"speaking":25159,"ĠVista":25160,"ĠFramework":25161,"ongo":25162,"Ġsubreddit":25163,"Ġcrus":25164,"ĠOval":25165,"Row":25166,"growing":25167,"Ġinstallment":25168,"Ġglac":25169,"ĠAdvance":25170,"ECK":25171,"ĠLGBTQ":25172,"LEY":25173,"Ġacet":25174,"Ġsuccessive":25175,"ĠNicole":25176,"Ġ1957":25177,"Quote":25178,"Ġcircumstance":25179,"ackets":25180,"Ġ142":25181,"ortium":25182,"Ġguessed":25183,"ĠFrame":25184,"Ġperpetrators":25185,"ĠAviation":25186,"ĠBench":25187,"Ġhandc":25188,"Ap":25189,"Ġ1956":25190,"259":25191,"rand":25192,"NetMessage":25193,"din":25194,"urtles":25195,"hig":25196,"ĠVIII":25197,"ffiti":25198,"ĠSwords":25199,"bial":25200,"Ġkidnapping":25201,"device":25202,"Ġbarn":25203,"ĠEli":25204,"aucas":25205,"Send":25206,"Constructed":25207,"Ġ½":25208,"Ġneedles":25209,"Ġadvertisements":25210,"Ġvou":25211,"Ġexhibited":25212,"ĠFortress":25213,"Ask":25214,"Berry":25215,"TYPE":25216,"Ġcancers":25217,"umping":25218,"ĠTerritory":25219,"Ġprud":25220,"Ġnas":25221,"Ġatheist":25222,"Ġbalances":25223,"ãģŁ":25224,"ĠShawn":25225,"&&":25226,"Ġlandsc":25227,"ĠRGB":25228,"Ġpetty":25229,"Ġexcellence":25230,"Ġtranslations":25231,"Ġparcel":25232,"ĠChev":25233,"East":25234,"ĠOutput":25235,"imi":25236,"Ġambient":25237,"ĠThreat":25238,"Ġvillains":25239,"Ġ550":25240,"ICA":25241,"Ġtaller":25242,"Ġleaking":25243,"cup":25244,"Ġpolish":25245,"Ġinfectious":25246,"ĠKC":25247,"Ġ@@":25248,"background":25249,"Ġbureaucracy":25250,"ĠSai":25251,"unless":25252,"itious":25253,"ĠSkype":25254,"Atl":25255,"IDENT":25256,"008":25257,"Ġhypocr":25258,"Ġpitchers":25259,"Ġguessing":25260,"ĠFINAL":25261,"Between":25262,"Ġvillagers":25263,"Ġ252":25264,"fashion":25265,"ĠTunis":25266,"Beh":25267,"ĠExc":25268,"ĠMID":25269,"288":25270,"ĠHaskell":25271,"196":25272,"ĠNOR":25273,"Ġspecs":25274,"Ġinvari":25275,"Ġglut":25276,"ĠCars":25277,"Ġimpulse":25278,"Ġhonors":25279,"gel":25280,"Ġjurisdictions":25281,"ĠBundle":25282,"ulas":25283,"California":25284,"ĠIncrease":25285,"Ġpear":25286,"Ġsingles":25287,"Ġcues":25288,"Ġunderwent":25289,"ĠWS":25290,"Ġexaggerated":25291,"Ġdubious":25292,"Ġflashing":25293,"LOG":25294,")].":25295,"Journal":25296,"tg":25297,"Van":25298,"ĠIstanbul":25299,"ĠInsp":25300,"ĠFranken":25301,"Draw":25302,"Ġsadness":25303,"Ġironic":25304,"ĠFry":25305,"xc":25306,"Ġ164":25307,"isch":25308,"Way":25309,"ĠProtestant":25310,"horn":25311,"Ġunaff":25312,"ĠViv":25313,"illas":25314,"ĠProductions":25315,"ĠHogan":25316,"Ġperimeter":25317,"ĠSisters":25318,"Ġspontaneous":25319,"Ġdownside":25320,"Ġdescendants":25321,"Ġorn":25322,"worm":25323,"Japanese":25324,"Ġ1955":25325,"Ġ151":25326,"ĠDoing":25327,"elsen":25328,"umbles":25329,"Ġradically":25330,"ĠDrum":25331,"ĠBach":25332,"Ġliabilities":25333,"ĠOB":25334,"ĠElementary":25335,"Ġmeme":25336,"ynes":25337,"Ġfingerprint":25338,"ĠGrab":25339,"Ġundertake":25340,"Members":25341,"ĠReader":25342,"ĠSims":25343,"god":25344,"Ġhypothetical":25345,"scient":25346,"ĠAJ":25347,"Ġcharism":25348,"Ġadmissions":25349,"ĠMissile":25350,"trade":25351,"Ġexercising":25352,"ĠBackground":25353,"Written":25354,"Ġvocals":25355,"whether":25356,"Ġvi":25357,"ĠWinner":25358,"Ġlitter":25359,"ĠShooting":25360,"STEM":25361,"ãĤ¡":25362,"ĠAFL":25363,"Ġvariability":25364,"Ġeats":25365,"ĠDPS":25366,"brow":25367,"Ġelephants":25368,"Ġstrat":25369,"ĠÅ":25370,"Ġsettlers":25371,"Matthew":25372,"Ġinadvert":25373,"HI":25374,"ĠIMF":25375,"ĠGoal":25376,"Ġnerves":25377,"Johnson":25378,"eye":25379,"ablishment":25380,"Thursday":25381,"BILITY":25382,"Had":25383,"amoto":25384,"hetamine":25385,"eps":25386,"Ġmitochond":25387,"Ġcompressed":25388,"ĠTrevor":25389,"ĠAnimals":25390,"Tool":25391,"Lock":25392,"Ġtweak":25393,"Ġpinch":25394,"Ġcancellation":25395,"Pot":25396,"Ġfocal":25397,"ĠAstron":25398,"173":25399,"ĠASC":25400,"ĠOTHER":25401,"umni":25402,"Ġdemise":25403,"dl":25404,"Ùħ":25405,"Semitism":25406,"Ġcracking":25407,"Ġcollaborative":25408,"Ġexplores":25409,"sql":25410,"Ġherbs":25411,"Ġconfigurations":25412,"mis":25413,"ĠResult":25414,"acey":25415,"ĠSmoke":25416,"Ġsanct":25417,"elia":25418,"Ġdegener":25419,"Ġdeepest":25420,"Ġscreamed":25421,"Ġnap":25422,"Software":25423,"ĠSTAR":25424,"EF":25425,"ĠXin":25426,"sponsored":25427,"manship":25428,"233":25429,"Ġprimaries":25430,"Ġfiltering":25431,"Ġassemble":25432,"mil":25433,"ĠMyers":25434,"bows":25435,"Ġpunched":25436,"Mic":25437,"Ġinnovations":25438,"Ġfunc":25439,"ando":25440,"Ġfracking":25441,"ĠVul":25442,"оÐ":25443,"oshop":25444,"ĠImmun":25445,"Ġsettling":25446,"Ġadolescents":25447,"Ġrebuilding":25448,"Ġtransforming":25449,"Ġparole":25450,"Ġharbor":25451,"Ġbooking":25452,"otional":25453,"ongevity":25454,"ĠYo":25455,"bug":25456,"Ġemerges":25457,"ĠMethods":25458,"ĠChu":25459,"Pres":25460,"ĠDungeons":25461,"Ġtrailing":25462,"ĠRum":25463,"ĠHugh":25464,"天":25465,"ĠEra":25466,"ĠBattles":25467,"Results":25468,"ĠTrading":25469,"Ġversa":25470,"css":25471,"axies":25472,"heet":25473,"Ġgreed":25474,"1989":25475,"Ġgardens":25476,"Ġcontingent":25477,"Park":25478,"ĠLeafs":25479,"hook":25480,"robe":25481,"Ġdiplomacy":25482,"ĠFuel":25483,"ĠInvasion":25484,"Ġupgrading":25485,"Male":25486,"Ġelic":25487,"Ġrelentless":25488,"ĠCovenant":25489,"apesh":25490,"ĠTrop":25491,"Ty":25492,"production":25493,"arty":25494,"Ġpunches":25495,"ako":25496,"cyclopedia":25497,"ĠRabbit":25498,"ĠHDMI":25499,"Ġ141":25500,"Ġfoil":25501,"ItemImage":25502,"ĠFG":25503,"Ġimplementations":25504,"ĠPom":25505,"ixtures":25506,"Ġawait":25507,"Ġ330":25508,"amus":25509,"Ġumbrella":25510,"Ġforesee":25511,"separ":25512,"Ġcircumcision":25513,"Ġperipheral":25514,"Say":25515,"ĠExpert":25516,"Inc":25517,"Ġwithdrew":25518,"ĠAnders":25519,"fried":25520,"Ġradioactive":25521,"ĠOpening":25522,"Ġboarding":25523,"ĠND":25524,"Ġoverthrow":25525,"Activ":25526,"WP":25527,"ĠActs":25528,"×Ļ":25529,"Ġmotions":25530,"vic":25531,"ĠMighty":25532,"ĠDefender":25533,"aer":25534,"Ġthankful":25535,"ĠKilling":25536,"ĠBris":25537,"moil":25538,"Ġpredicting":25539,"266":25540,"choice":25541,"Ġkillers":25542,"Ġincub":25543,"ĠChest":25544,"athering":25545,"Ġproclaimed":25546,"flower":25547,"ossom":25548,"umbledore":25549,"ĠCycling":25550,"ĠOccupy":25551,"AGES":25552,"Pen":25553,"ĠYug":25554,"Ġpackaged":25555,"Ġheightened":25556,"cot":25557,"stack":25558,"Cond":25559,"Ġstamps":25560,"mage":25561,"Ġpersuaded":25562,"Ġensl":25563,"ĠCardinal":25564,"Ġsolitary":25565,"Ġpossessing":25566,"ĠCork":25567,"Ġevid":25568,"ĠTay":25569,"Ġblues":25570,"Ġextremism":25571,"Ġlunar":25572,"Ġclown":25573,"Techn":25574,"Ġfestivals":25575,"ĠPvP":25576,"ĠLar":25577,"Ġconsequently":25578,"present":25579,"Ġsomeday":25580,"çİĭ":25581,"ĠMeteor":25582,"Ġtouring":25583,"culture":25584,"Ġbeaches":25585,"Ship":25586,"cause":25587,"ĠFlood":25588,"ãĥ¯":25589,"Ġpurity":25590,"those":25591,"Ġemission":25592,"bolt":25593,"Ġchord":25594,"ĠScripture":25595,"Lu":25596,"Ġ${":25597,"created":25598,"Others":25599,"258":25600,"Ġelemental":25601,"Ġannoyed":25602,"ĠAE":25603,"dan":25604,"ĠSag":25605,"Researchers":25606,"Ġfairy":25607,"âĢĵâĢĵ":25608,"============":25609,"Smart":25610,"GGGG":25611,"Ġskeletons":25612,"Ġpupils":25613,"linked":25614,"Ġurgency":25615,"enabled":25616,"ĠFuck":25617,"Ġcouncill":25618,"rab":25619,"UAL":25620,"TI":25621,"Ġlifes":25622,"Ġconfessed":25623,"Bug":25624,"Ġharmon":25625,"ĠCONFIG":25626,"ĠNeutral":25627,"Double":25628,"Ġstaple":25629,"ĠSHA":25630,"British":25631,"ĠSNP":25632,"ATOR":25633,"oco":25634,"Ġswinging":25635,"gex":25636,"oleon":25637,"plain":25638,"ĠMissing":25639,"ĠTrophy":25640,"vari":25641,"ranch":25642,"Ġ301":25643,"440":25644,"0000000000000000":25645,"Ġrestoring":25646,"Ġhaul":25647,"ucing":25648,"nerg":25649,"Ġfutures":25650,"Ġstrategist":25651,"question":25652,"Ġlateral":25653,"ĠBard":25654,"Ġsor":25655,"ĠRhodes":25656,"ĠDowntown":25657,"?????-":25658,"ĠLit":25659,"ĠBened":25660,"Ġcoil":25661,"street":25662,"ĠPortal":25663,"FILE":25664,"ĠGru":25665,"*,":25666,"231":25667,"neum":25668,"Ġsucked":25669,"Ġrapper":25670,"Ġtendencies":25671,"ĠLauren":25672,"cellaneous":25673,"267":25674,"Ġbrowse":25675,"Ġoverc":25676,"header":25677,"oise":25678,"Ġbeet":25679,"ĠGle":25680,"Stay":25681,"Ġmum":25682,"Ġtyped":25683,"Ġdiscounts":25684,"Talk":25685,"ĠOg":25686,"existing":25687,"ĠSell":25688,"uph":25689,"CI":25690,"ĠAustrian":25691,"ĠWarm":25692,"Ġdismissal":25693,"Ġaverages":25694,"camera":25695,"Ġallegiance":25696,"LAN":25697,"=\"#":25698,"Ġcommentators":25699,"ĠSetting":25700,"ĠMidwest":25701,"Ġpharmac":25702,"ĠEXP":25703,"Ġstainless":25704,"Chicago":25705,"Ġtan":25706,"244":25707,"Ġcountryside":25708,"ĠVac":25709,"295":25710,"Ġpinned":25711,"Ġcrises":25712,"Ġstandardized":25713,"Task":25714,"ĠJail":25715,"ĠDocker":25716,"colored":25717,"forth":25718,"\"},":25719,"Ġpatrons":25720,"Ġspice":25721,"Ġmourn":25722,"ĠMood":25723,"Ġlaundry":25724,"Ġequip":25725,"ĠMole":25726,"yll":25727,"ĠTHC":25728,"nation":25729,"ĠSherlock":25730,"Ġissu":25731,"ĠKre":25732,"ĠAmericas":25733,"ĠAAA":25734,"Ġsystematically":25735,"Ġcontra":25736,"ĠSally":25737,"Ġrationale":25738,"Ġcarriage":25739,"Ġpeaks":25740,"Ġcontradiction":25741,"ensation":25742,"ĠFailure":25743,"Ġprops":25744,"Ġnamespace":25745,"Ġcove":25746,"fields":25747,"ãĤĭ":25748,"Ġwool":25749,"ĠCatch":25750,"Ġpresumed":25751,"ĠDiana":25752,"ragon":25753,"igi":25754,"Ġhamm":25755,"Ġstunt":25756,"ĠGUI":25757,"ĠObservatory":25758,"ĠShore":25759,"Ġsmells":25760,"annah":25761,"Ġcockpit":25762,"ĠDuterte":25763,"850":25764,"Ġoppressed":25765,"breaker":25766,"ĠContribut":25767,"ĠPeru":25768,"ĠMonsanto":25769,"ĠAttempt":25770,"Ġcommanding":25771,"Ġfridge":25772,"ĠRin":25773,"ĠChess":25774,"uality":25775,"Ġol":25776,"Republican":25777,"ĠGlory":25778,"ĠWIN":25779,".......":25780,"agent":25781,"reading":25782,"Ġinh":25783,"Jones":25784,"Ġclicks":25785,"alan":25786,"Ġ[];":25787,"ĠMajesty":25788,"ĠCed":25789,"opus":25790,"atel":25791,"ê":25792,"ARC":25793,"ĠEcuador":25794,"ãĥł":25795,"ĠKuro":25796,"Ġrituals":25797,"Ġcaptive":25798,"Ġounce":25799,"Ġdisagreement":25800,"Ġslog":25801,"fuel":25802,"Pet":25803,"Mail":25804,"Ġexercised":25805,"Ġsolic":25806,"Ġrainfall":25807,"Ġdevotion":25808,"ĠAssessment":25809,"Ġrobotic":25810,"options":25811,"ĠRP":25812,"ĠFamilies":25813,"ĠFlames":25814,"Ġassignments":25815,"007":25816,"akedown":25817,"Ġvocabulary":25818,"Reilly":25819,"Ġcaval":25820,"gars":25821,"Ġsuppressed":25822,"ĠSET":25823,"ĠJohns":25824,"Ġwarp":25825,"broken":25826,"Ġstatues":25827,"Ġadvocated":25828,"Ġ275":25829,"Ġperil":25830,"omorph":25831,"ĠFemin":25832,"perfect":25833,"Ġhatch":25834,"Lib":25835,"512":25836,"Ġlifelong":25837,"313":25838,"Ġcheeks":25839,"Ġnumbered":25840,"ĠMug":25841,"Body":25842,"ravel":25843,"Weight":25844,"ĠJak":25845,"ĠHeath":25846,"Ġkissing":25847,"ĠJUST":25848,"Ġwaving":25849,"upload":25850,"Ġinsider":25851,"ĠProgressive":25852,"ĠFilter":25853,"tta":25854,"ĠBeam":25855,"Ġviolently":25856,"ipation":25857,"Ġskepticism":25858,"Ġ1918":25859,"ĠAnnie":25860,"ĠSI":25861,"Ġgenetics":25862,"Ġonboard":25863,"atl":25864,"ĠFriedman":25865,"ĠBri":25866,"ceptive":25867,"Ġpirate":25868,"ĠReporter":25869,"278":25870,"Ġmythology":25871,"Ġeclipse":25872,"Ġskins":25873,"Ġglyph":25874,"ingham":25875,"Files":25876,"Cour":25877,"women":25878,"Ġregimes":25879,"Ġphotographed":25880,"Kat":25881,"ĠMAX":25882,"Officials":25883,"Ġunexpectedly":25884,"Ġimpressions":25885,"Front":25886,";;;;;;;;":25887,"Ġsupremacy":25888,"Ġsang":25889,"Ġaggravated":25890,"Ġabruptly":25891,"ĠSector":25892,"Ġexcuses":25893,"Ġcosting":25894,"idepress":25895,"Stack":25896,"ĠRNA":25897,"obil":25898,"Ġghosts":25899,"ldon":25900,"atibility":25901,"Topics":25902,"Ġreimburse":25903,"ĠHM":25904,"ĠDeg":25905,"Ġthief":25906,"yet":25907,"ogenesis":25908,"leaning":25909,"ĠKol":25910,"ĠBasketball":25911,"Ġfi":25912,"ĠSeeing":25913,"Ġrecycling":25914,"Ġ[-":25915,"Congress":25916,"Ġlectures":25917,"Psy":25918,"Ġnep":25919,"Ġmaid":25920,"Ġoriented":25921,"AX":25922,"Ġrespectful":25923,"rene":25924,"flush":25925,"ĠUnloaded":25926,"request":25927,"grid":25928,"ĠAlternatively":25929,"ĠHugo":25930,"Ġdecree":25931,"ĠBuddhism":25932,"andum":25933,"Android":25934,"ĠCongo":25935,"ĠJoyce":25936,"Ġacknowledging":25937,"hesive":25938,"ĠTomorrow":25939,"ĠHiro":25940,"thren":25941,"ĠMaced":25942,"Ġhoax":25943,"ĠIncreased":25944,"ĠPradesh":25945,"Wild":25946,"______":25947,"161":25948,"Ġaunt":25949,"Ġdistributing":25950,"ĠTucker":25951,"ĠSSL":25952,"ĠWolves":25953,"Building":25954,"oult":25955,"ĠLuo":25956,"ĠYas":25957,"ĠSpir":25958,"ĠShape":25959,"ĠCambod":25960,"ĠIPv":25961,"Ġml":25962,"Ġextrad":25963,"390":25964,"ĠPenny":25965,"dream":25966,"Ġstationed":25967,"optional":25968,"eworthy":25969,".":26700,"ĠWorkshop":26701,"ĠRetail":26702,"ĠAvatar":26703,"625":26704,"Na":26705,"ĠVC":26706,"ĠSecure":26707,"MY":26708,"1988":26709,"ossip":26710,"Ġprostate":26711,"Ġunden":26712,"Ġgamer":26713,"ĠContents":26714,"ĠWarhammer":26715,"ĠSentinel":26716,"310":26717,"Ġsegregation":26718,"ĠFlex":26719,"ĠMAY":26720,"Ġdrills":26721,"ĠDrugs":26722,"Islamic":26723,"Ġspur":26724,"Ġcafe":26725,"Ġimaginary":26726,"Ġguiding":26727,"Ġswings":26728,"ĠTheme":26729,"oby":26730,"Ġnud":26731,"Ġbegging":26732,"Ġstrongh":26733,"Ġrejecting":26734,"Ġpedestrians":26735,"ĠProspect":26736,"Rare":26737,"sle":26738,"Ġconcessions":26739,"ĠConstitutional":26740,"Ġbeams":26741,"Ġfibers":26742,"poon":26743,"Ġinstincts":26744,"property":26745,"ĠBIG":26746,"Sanders":26747,"imates":26748,"Ġcoating":26749,"Ġcorpses":26750,"ĠTRUE":26751,"checked":26752,"Ġ166":26753,"Ash":26754,"ĠJS":26755,"ĠFiction":26756,"Ġcommunal":26757,"Ġenergetic":26758,"oooooooo":26759,"Ġnowadays":26760,"ILD":26761,"ibo":26762,"ĠSUV":26763,"Ren":26764,"Ġdwelling":26765,"Silver":26766,"Ġtally":26767,"ĠMoving":26768,"Ġcoward":26769,"Ġgenerals":26770,"Ġhorns":26771,"Ġcirculated":26772,"Ġrobbed":26773,"ĠUnlimited":26774,"Ġharassed":26775,"Ġinhibit":26776,"Ġcomposer":26777,"ĠSpotify":26778,"Ġspreads":26779,"364":26780,"Ġsuicidal":26781,"Ġnoises":26782,"ĠStur":26783,"Ġsaga":26784,"ĠKag":26785,"iso":26786,"Ġtheoretically":26787,"Money":26788,"Ġsimilarity":26789,"Ġsliced":26790,"utils":26791,"inges":26792,"\"-":26793,"Ġanth":26794,"Ġimped":26795,"Module":26796,"Throughout":26797,"Ġmenus":26798,"committee":26799,"andi":26800,"obj":26801,"inav":26802,"fired":26803,"ĠAbdullah":26804,"Ġundead":26805,"Ġfonts":26806,"Hold":26807,"ENG":26808,"Ġsustainability":26809,"Ġflick":26810,"Ġrazor":26811,"ĠFest":26812,"ĠCharacters":26813,"Ġwording":26814,"Ġpopulist":26815,"Ġcriticizing":26816,"Ġmuse":26817,"vine":26818,"Ġcardboard":26819,"Ġkindly":26820,"Ġfringe":26821,"ĠTheft":26822,"icultural":26823,"Ġgovernors":26824,"Ġ����":26825,"Ġ163":26826,"Ġtimeout":26827,"ĠAuth":26828,"Children":26829,"AU":26830,"Ġredemption":26831,"ĠAlger":26832,"Ġ1914":26833,"Ġwaved":26834,"Ġastronauts":26835,"ograms":26836,"Ġswamp":26837,"ĠFinnish":26838,"Ġcandle":26839,"Ġtonnes":26840,"utm":26841,"Ġray":26842,"Ġspun":26843,"Ġfearful":26844,"articles":26845,"Ġcaus":26846,"orically":26847,"ĠRequires":26848,"ĠGol":26849,"Ġpope":26850,"Ġinaugural":26851,"Ġgle":26852,"ADA":26853,"ĠISIL":26854,"ĠOffensive":26855,"Ġwatchdog":26856,"Ġbalcon":26857,"entity":26858,"ĠHoo":26859,"Ġgallon":26860,"ACC":26861,"Ġdoubling":26862,"Ġimplication":26863,"ĠSight":26864,"Ġdoctr":26865,"-------":26866,"Ġ\\\\":26867,"Ġmalt":26868,"Roll":26869,"Ġâī¥":26870,"Ġrecap":26871,"adding":26872,"uces":26873,"ĠBend":26874,"figure":26875,"Ġturkey":26876,"Ġsocietal":26877,"ĠTickets":26878,"Ġcommercially":26879,"Ġspicy":26880,"Ġ216":26881,"ĠRamp":26882,"Ġsuperiority":26883,"ï":26884,"ĠTracker":26885,"Carl":26886,"ĠCoy":26887,"ĠPatriot":26888,"Ġconsulted":26889,"Ġlistings":26890,"Ġslew":26891,"reenshot":26892,"ĠGone":26893,"Ġ[...]":26894,"309":26895,"Ġhottest":26896,"ر":26897,"Ġrocky":26898,"ĠDiaz":26899,"Ġmassage":26900,"Ġparaly":26901,"Ġpony":26902,"Az":26903,"Ġcartridge":26904,"ĠNZ":26905,"Ġsnack":26906,"ĠLamar":26907,"plement":26908,"ĠLeslie":26909,"Ġmater":26910,"Ġsnipp":26911,"246":26912,"Ġjointly":26913,"ĠBrisbane":26914,"ĠiPod":26915,"Ġpumping":26916,"Ġgoat":26917,"ĠSharon":26918,"ealing":26919,"Ġcoron":26920,"Ġanomal":26921,"rahim":26922,"ĠConnection":26923,"Ġsculpture":26924,"Ġscheduling":26925,"ĠDaddy":26926,"athing":26927,"Ġeyebrows":26928,"Ġcurved":26929,"Ġsentiments":26930,"Ġdrafting":26931,"Drop":26932,"([":26933,"Ġnominal":26934,"ĠLeadership":26935,"ĠGrow":26936,"Ġ176":26937,"Ġconstructive":26938,"ivation":26939,"Ġcorrupted":26940,"gerald":26941,"ĠCros":26942,"ĠChester":26943,"ĠLap":26944,"ãģª":26945,"OTH":26946,"DATA":26947,"Ġalmond":26948,"probably":26949,"Imp":26950,"Ġfeast":26951,"ĠWarcraft":26952,"Flor":26953,"Ġcheckpoint":26954,"Ġtranscription":26955,"Ġ204":26956,"Ġtweaks":26957,"Ġrelieve":26958,"Science":26959,"Ġperformer":26960,"Zone":26961,"Ġturmoil":26962,"igated":26963,"hibit":26964,"ĠCafe":26965,"themed":26966,"Ġfluor":26967,"bench":26968,"Ġdecom":26969,"ĠUnt":26970,"ĠBarrett":26971,"ĠFacts":26972,"Ġtasting":26973,"ĠPTSD":26974,"ĠSeal":26975,"ĠJudaism":26976,"ĠDynamic":26977,"ĠCors":26978,"Ve":26979,"ĠMing":26980,"ĠTransform":26981,"von":26982,"ĠDefenders":26983,"ĠTactical":26984,"ĠVon":26985,"ĠUnivers":26986,"Ġdistorted":26987,"ĠBreath":26988,"?'\"":26989,"Ġagon":26990,"ĠDeadly":26991,"Ġlan":26992,"ĠCycle":26993,"orned":26994,"Ġreliably":26995,"Ġglor":26996,"ĠMonkey":26997,"ãĥ¡":26998,"Ġadren":26999,"Ġmicrowave":27000,"ĠAlban":27001,"ircraft":27002,"digit":27003,"smart":27004,"ĠDread":27005,"¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯":27006,"{{":27007,"ĠRochester":27008,"Ġsimplified":27009,"Ġinflicted":27010,"Ġtakeover":27011,"Ġyourselves":27012,"aditional":27013,"Ġmuscular":27014,"KS":27015,"Ġingen":27016,"Tax":27017,"ĠFeature":27018,"277":27019,"Ġcruc":27020,"Ġcrate":27021,"Ġunidentified":27022,"Ġacclaimed":27023,"ĠManga":27024,"ĠFrances":27025,"ĠNepal":27026,"ĠGerald":27027,"ĠKuwait":27028,"Ġslain":27029,"ĠHeb":27030,"ĠGoku":27031,"ã쮿":27032,"286":27033,"Mrs":27034,"ĠCody":27035,"ĠSanctuary":27036,"016":27037,"Ġdismant":27038,"Ġdataset":27039,"ĠHond":27040,"buck":27041,"ĠPatterson":27042,"Ġpalette":27043,"ĠGD":27044,"icol":27045,"ĠLodge":27046,"Ġplanetary":27047,"akin":27048,"ĠRegistered":27049,"abwe":27050,"ĠPetersburg":27051,"Ġhailed":27052,"ĠPiece":27053,"Sche":27054,"ĠDOJ":27055,"Ġenumer":27056,"181":27057,"ĠObserver":27058,"ĠBold":27059,"founded":27060,"commerce":27061,"Ġexploits":27062,"ĠFinding":27063,"URN":27064,"ĠSne":27065,"ĠAcid":27066,"ayette":27067,"ĠValues":27068,"Ġdrastic":27069,"Ġarchitectural":27070,"Ġ\".":27071,"×ķ":27072,"umped":27073,"Ġwrapping":27074,"Ġwidow":27075,"ĠSlayer":27076,"lace":27077,"once":27078,"Germany":27079,"avoid":27080,"Ġtemples":27081,"PAR":27082,"ô":27083,"ĠLucifer":27084,"ĠFlickr":27085,"lov":27086,"forces":27087,"Ġscouting":27088,"Ġlouder":27089,"tesy":27090,"Ġbeforehand":27091,"Äĵ":27092,"ĠNeon":27093,"ĠWol":27094,"ĠTypically":27095,"ĠPolitico":27096,"-+-+":27097,"Ġbuilder":27098,"Ġderive":27099,"Kill":27100,"Ġpoker":27101,"Ġambiguous":27102,"Ġlifts":27103,"Ġcyt":27104,"Ġribs":27105,"oodle":27106,"ĠSounds":27107,"hair":27108,"ĠSyndrome":27109,"tf":27110,"Ġproportional":27111,"uid":27112,"Ġpertaining":27113,"ĠKindle":27114,"ĠNegro":27115,"Ġreiterated":27116,"ĠTonight":27117,"oths":27118,"ĠCornell":27119,"Ġowing":27120,"Ġ208":27121,"elfare":27122,"ocating":27123,"ĠBirds":27124,"Subscribe":27125,"Ġessays":27126,"Ġburdens":27127,"Ġillustrations":27128,"arious":27129,"ERAL":27130,"ĠCalcul":27131,"Ġxen":27132,"ĠLinkedIn":27133,"ĠJung":27134,"Ġredesign":27135,"Connor":27136,"296":27137,"Ġreversal":27138,"ĠAdelaide":27139,"ĠLL":27140,"Ġsinking":27141,"Ġgum":27142,"USH":27143,"capt":27144,"ĠGrimm":27145,"Ġfootsteps":27146,"ĠCBD":27147,"ispers":27148,"Ġprose":27149,"Wednesday":27150,"ĠMovies":27151,"edin":27152,"Ġoverturned":27153,"Ġcontentious":27154,"USB":27155,"~~~~~~~~~~~~~~~~":27156,"ĠCopper":27157,"Ġpointless":27158,"NV":27159,"values":27160,"olphin":27161,"dain":27162,"Ġdeposited":27163,"ĠGW":27164,"Ġpreceded":27165,"ĠCla":27166,"ĠGolem":27167,"ĠNim":27168,"Ġβ":27169,"ĠEngineers":27170,"middle":27171,"Ġflatt":27172,"operative":27173,"Ġcouncils":27174,"imbabwe":27175,"elin":27176,"Ġstressful":27177,"ĠLD":27178,"Ġresh":27179,"lake":27180,"Ġwheelchair":27181,"ĠAlternative":27182,"Ġoptimize":27183,"operation":27184,"Ġpeek":27185,"Ġoneself":27186,"igil":27187,"Ġtransitions":27188,"opathy":27189,"blank":27190,"Ġ169":27191,"171":27192,"________________________________________________________________":27193,"Ġlaundering":27194,"Enc":27195,"ĠDEC":27196,"Ġworkouts":27197,"Ġspikes":27198,"Ġdinosaurs":27199,"Ġdiscriminatory":27200,"Pool":27201,"Rather":27202,"385":27203,"RNA":27204,"testers":27205,"eto":27206,"ĠIdentity":27207,"Ġvein":27208,"ĠBurton":27209,"Ġarcade":27210,"420":27211,"Ultimately":27212,"ĠSadly":27213,"ð":27214,"pill":27215,"Ġcubic":27216,"ĠSpectrum":27217,"these":27218,"states":27219,"Ġunofficial":27220,"hawks":27221,"ĠEVERY":27222,"Ġrainbow":27223,"Ġincarceration":27224,"anding":27225,"Ġsyll":27226,"ĠEverton":27227,"Ġ179":27228,"ĠSerbia":27229,"Ġ189":27230,"meter":27231,"ĠMickey":27232,"Ġantiqu":27233,"Ġfactual":27234,"neck":27235,"ĠNare":27236,"norm":27237,"must":27238,"Ġhighways":27239,"Ġglam":27240,"Ġdividing":27241,"ĠSquadron":27242,"ĠMartha":27243,"Ġbirths":27244,"Cover":27245,"////////////////":27246,"ĠWong":27247,"Phot":27248,"ĠALS":27249,"rio":27250,"ĠNonetheless":27251,"ĠLemon":27252,"Ġ206":27253,"ĠEE":27254,"Ġderivative":27255,"ĠWWII":27256,"vote":27257,"Ġtherein":27258,"Ġseparating":27259,"446":27260,"sync":27261,"ĠStreets":27262,"Ġratt":27263,"Ġmunicipality":27264,"ĠShortly":27265,"Ġmonk":27266,"),\"":27267,"Ġscrub":27268,"Ġoperatives":27269,"Neither":27270,"Place":27271,"ĠLimit":27272,"Female":27273,"ĠActor":27274,"Character":27275,"Ġconstituted":27276,"357":27277,"Ġprotested":27278,"ĠStraw":27279,"ĠHeight":27280,"ilda":27281,"ĠTyph":27282,"Ġfloods":27283,"Ġcosmetic":27284,"WAY":27285,"perture":27286,"upon":27287,"tons":27288,"essing":27289,"ĠPocket":27290,"Ġrooft":27291,"ĠCaucas":27292,"Ġantidepress":27293,"Ġincompatible":27294,"ECD":27295,"Ġopera":27296,"ĠContest":27297,"Ġgenerators":27298,"lime":27299,"Defense":27300,"1987":27301,"forum":27302,"Ġsavage":27303,"ĠHungarian":27304,"nz":27305,"Ġmetallic":27306,"Ġexpelled":27307,"Ġresidency":27308,"Ġdresses":27309,"666":27310,"ĠClement":27311,"fires":27312,"Category":27313,"Ġgeek":27314,"alis":27315,"Ġcemetery":27316,"educated":27317,"Ġcrawl":27318,"ĠUnable":27319,"ĠTyson":27320,"akis":27321,"Ġpardon":27322,"ĠWra":27323,"Ġstrengthened":27324,"ĠFors":27325,"335":27326,"ĠHC":27327,"ĠMond":27328,"Ġvisuals":27329,"ĠBeatles":27330,"ettlement":27331,"Ġï":27332,"gro":27333,"Ġbash":27334,"Ġpoorest":27335,"Ġexcel":27336,"Ġaspirations":27337,"ĠMunicip":27338,"ensible":27339,"Ġceremonies":27340,"Ġintimidation":27341,"ĠCONTR":27342,"beck":27343,"ĠKap":27344,"asu":27345,"Ġtrademarks":27346,"ĠSew":27347,"ĠCompetition":27348,"network":27349,"ĠArri":27350,"ĠTet":27351,"Roaming":27352,"WC":27353,"Dat":27354,"Ġsob":27355,"Ġpairing":27356,"Ġoverdose":27357,"SAY":27358,"aber":27359,"Ġrevolt":27360,"ĠFah":27361,"acting":27362,"eq":27363,"estation":27364,"Fight":27365,"ĠMarks":27366,"273":27367,"Ġ178":27368,"Raw":27369,"ãģĭ":27370,"349":27371,"blocks":27372,"Ġverge":27373,"estine":27374,"ĠPodesta":27375,"Ġinvasive":27376,"Ġprofoundly":27377,"ĠAo":27378,"each":27379,"Ġlest":27380,"interpret":27381,"Ġshrinking":27382,"Ġerrone":27383,"Ġchees":27384,"lys":27385,"ĠIvy":27386,"ĠDirectory":27387,"Ġhinted":27388,"VICE":27389,"Ġcontacting":27390,"ĠGent":27391,"hei":27392,"Ġlabeling":27393,"Ġmercury":27394,"ĠLite":27395,"Ġexpires":27396,"Ġdestabil":27397,"ritis":27398,"cu":27399,"Ġfeathers":27400,"Ġsteer":27401,"Ġprogrammed":27402,"ĠVader":27403,"Going":27404,"ĠElim":27405,"Ġyo":27406,"ĠMiche":27407,"Ġ203":27408,"Ġsleeves":27409,"Ġbully":27410,"ĠHumans":27411,"368":27412,"Ġcompress":27413,"ĠBanner":27414,"ARS":27415,"Ġawhile":27416,"Ġcalib":27417,"Ġsponsorship":27418,"ĠDifficulty":27419,"ĠPapers":27420,"Ġidentifier":27421,"}.":27422,"Ġyog":27423,"ĠShia":27424,"Ġcleanup":27425,"Ġvibe":27426,"introdu":27427,"imming":27428,"Australia":27429,"Ġoutlines":27430,"ĠYoutube":27431,"train":27432,"ĠMakes":27433,"Ġdeported":27434,"Ġcentr":27435,"ĠDug":27436,"ĠBoulder":27437,"ĠBuffy":27438,"Ġinjunction":27439,"ĠHarley":27440,"ĠGroups":27441,"ĠDumbledore":27442,"ĠClara":27443,"Ġ\"-":27444,"Ġsacrificed":27445,"eph":27446,"Shadow":27447,"ibling":27448,"Ġfreelance":27449,"Ġevidently":27450,"phal":27451,"Ġretains":27452,"Mir":27453,"Ġfinite":27454,"dar":27455,"ĠCous":27456,"Ġrepaired":27457,"Ġperiodic":27458,"Ġchampionships":27459,"Ġasteroid":27460,"blind":27461,"Ġexpressly":27462,"ĠAstros":27463,"Ġscaled":27464,"Ġgeographical":27465,"ĠRapids":27466,"Enjoy":27467,"Ġelastic":27468,"ĠMohamed":27469,"Market":27470,"begin":27471,"Ġdiscovers":27472,"Ġtelecommunications":27473,"Ġscanner":27474,"Ġenlarge":27475,"Ġsharks":27476,"Ġpsychedel":27477,"ĠRouge":27478,"Ġsnapshot":27479,"isine":27480,"XP":27481,"Ġpesticides":27482,"ĠLSD":27483,"ĠDistribution":27484,"really":27485,"Ġdegradation":27486,"Ġdisguise":27487,"Ġbiom":27488,"ĠEXT":27489,"Ġequations":27490,"Ġhazards":27491,"ĠCompared":27492,")*":27493,"Ġvirtues":27494,"Ġelders":27495,"Ġenhancing":27496,"ĠAcross":27497,"eros":27498,"angling":27499,"Ġcombust":27500,"ucci":27501,"Ġconcussion":27502,"Ġcontraception":27503,"ĠKang":27504,"Ġexpresses":27505,"Ġaux":27506,"ĠPione":27507,"Ġexhibits":27508,"Debug":27509,"OTAL":27510,"ĠAlready":27511,"ĠWheeler":27512,"Ġexpands":27513,"?:":27514,"Ġreconciliation":27515,"Ġpirates":27516,"Ġpurse":27517,"Ġdiscourage":27518,"Ġspectacle":27519,"Rank":27520,"Ġwraps":27521,"ĠThought":27522,"Ġimpending":27523,"Opp":27524,"ĠAnglo":27525,"ĠEUR":27526,"Ġscrewed":27527,"retched":27528,"Ġencouragement":27529,"models":27530,"Ġconfuse":27531,"mmm":27532,"ĠVitamin":27533,"âĸijâĸij":27534,"Cru":27535,"Ġknights":27536,"Ġdiscard":27537,"Ġbishops":27538,"ĠWear":27539,"ĠGarrett":27540,"kan":27541,"ãĥŁ":27542,"Ġmasculine":27543,"capital":27544,"ĠAus":27545,"Ġfatally":27546,"thanks":27547,"ĠAU":27548,"ĠGut":27549,"1200":27550,"Ġ00000000":27551,"Ġsurrog":27552,"ĠBIOS":27553,"raits":27554,"ĠWatts":27555,"Ġresurrection":27556,"ĠElectoral":27557,"ĠTips":27558,"4000":27559,"Ġnutrient":27560,"Ġdepicting":27561,"Ġsprink":27562,"Ġmuff":27563,"ĠLIM":27564,"ĠSample":27565,"psc":27566,"ibi":27567,"generated":27568,"Ġspecimens":27569,"Ġdissatisf":27570,"Ġtailored":27571,"Ġholdings":27572,"ĠMonthly":27573,"ĠEat":27574,"poons":27575,"Ġnec":27576,"ĠCage":27577,"ĠLotus":27578,"ĠLantern":27579,"Ġfrontier":27580,"Ġpensions":27581,"Ġjoked":27582,"ĠHardy":27583,"=-=-=-=-":27584,"rade":27585,"UID":27586,"Ġrails":27587,"Ġemit":27588,"Ġslate":27589,"Ġsmug":27590,"Ġspit":27591,"ĠCalls":27592,"ĠJacobs":27593,"feat":27594,"ĠUE":27595,"Ġrestruct":27596,"Ġregeneration":27597,"Ġenergies":27598,"ĠConnor":27599,"OHN":27600,"ĠCheese":27601,"Ġger":27602,"Ġresurrect":27603,"management":27604,"NW":27605,"Ġpresently":27606,"ĠBruins":27607,"Member":27608,"ĠMang":27609,"idan":27610,"Ġboosting":27611,"wyn":27612,"+.":27613,"requisite":27614,"ĠNYPD":27615,"ĠMegan":27616,"ĠConditions":27617,"Ġpics":27618,"nesium":27619,"ĠRash":27620,"Ġ174":27621,"ĠDucks":27622,"Ġembro":27623,"zu":27624,"onian":27625,"religious":27626,"Ġcraz":27627,"ĠACA":27628,"ĠZucker":27629,"EMA":27630,"ĠPros":27631,"Weapon":27632,"ĠKnox":27633,"ĠArduino":27634,"Ġstove":27635,"Ġheavens":27636,"ĠPurchase":27637,"Ġherd":27638,"Ġfundraiser":27639,"Digital":27640,"5000":27641,"Ġproponents":27642,"/âĢĭ":27643,"Ġjelly":27644,"ĠVisa":27645,"Ġmonks":27646,"Ġadvancement":27647,"ĠWer":27648,"Ġ187":27649,"eus":27650,"ertility":27651,"Ġfetal":27652,"Ġ1936":27653,"Lo":27654,"Ġoutfits":27655,"Ġstaircase":27656,"bomb":27657,"Ġcustomized":27658,"clair":27659,"Tree":27660,"Ġmapped":27661,"ĠConsidering":27662,"ĠTorres":27663,"Ġmethyl":27664,"Ġapproximate":27665,"Ġdoom":27666,"ĠHansen":27667,"Ġcrossover":27668,"Ġstandalone":27669,"ä¼":27670,"Ġinvites":27671,"Ġgraveyard":27672,"Ġhp":27673,"DonaldTrump":27674,"Ġescort":27675,"Gar":27676,"Ġpredecessors":27677,"Ġhay":27678,"Ġenzyme":27679,"ĠStraight":27680,"visors":27681,"Ing":27682,"aneously":27683,"ĠApplied":27684,"Ġfec":27685,"ĠDurant":27686,"Ġoutspoken":27687,"orb":27688,"Ġzeal":27689,"Ġdisgrace":27690,"').":27691,"ĠCheng":27692,"289":27693,"ĠRena":27694,"ĠSuicide":27695,"294":27696,"Ġoutraged":27697,"ĠNewman":27698,"ĠNvidia":27699,"ĠAber":27700,"ĠBers":27701,"Ġrecreation":27702,"Window":27703,"ĠDP":27704,"xe":27705,"Ġpedoph":27706,"Ġfallout":27707,"amboo":27708,"Ġpresentations":27709,"ĠApps":27710,"Ġhtml":27711,"345":27712,"ĠXXX":27713,"Ġrubbing":27714,"ĠLeather":27715,"Ġhumidity":27716,"seys":27717,"established":27718,"ĠUnits":27719,"646":27720,"Ġrespectable":27721,"Auto":27722,"Ġthriving":27723,"ĠInnovation":27724,"angs":27725,"Extra":27726,"regulation":27727,"298":27728,"pick":27729,"Examples":27730,"ĠCJ":27731,"Attack":27732,"Ġdracon":27733,"LT":27734,"Ġsticker":27735,"rers":27736,"Ġsunny":27737,"Iss":27738,"regulated":27739,"dim":27740,"ĠAbstract":27741,"Ġhusbands":27742,"Office":27743,"omination":27744,"itars":27745,"ANGE":27746,"ascal":27747,"ĠKris":27748,"ĠInfantry":27749,"Ġmalf":27750,"ĠAthe":27751,"ĠRally":27752,"balanced":27753,"........................":27754,"OUP":27755,"Ġmolecule":27756,"metics":27757,"ĠSplit":27758,"ĠInstructions":27759,"ĠNights":27760,"cards":27761,"Ġtug":27762,"Ġcone":27763,"åŃ":27764,"Ġtx":27765,"ĠDiscussion":27766,"Ġcatastrophe":27767,"ppe":27768,"gio":27769,"Ġcommunism":27770,"Ġhalted":27771,"ĠGuant":27772,"clean":27773,"ĠSched":27774,"ĠKanye":27775,"Ġwander":27776,"ĠSeriously":27777,"Ġ188":27778,"ennial":27779,"follow":27780,"productive":27781,"ĠFlow":27782,"ĠSail":27783,"Ġcraw":27784,"Ġsimulations":27785,"oru":27786,"angles":27787,"ĠNolan":27788,"Ġmenstru":27789,"470":27790,"Ġ207":27791,"aja":27792,"Ġcasually":27793,"boarding":27794,"Ġ222":27795,"ovy":27796,"ĠNumbers":27797,"umat":27798,"OE":27799,"287":27800,"ĠClemson":27801,"Ġcerts":27802,"Ġslid":27803,"ĠTribe":27804,"Ġtoast":27805,"Ġfortunes":27806,"Ġfals":27807,"ĠCommittees":27808,"Ġgp":27809,"Ġfiery":27810,"ĠNets":27811,"ĠAnime":27812,"Package":27813,"ĠCompare":27814,"laughter":27815,"infect":27816,"Ġatrocities":27817,"Ġjustices":27818,"Ġinsults":27819,"ĠVernon":27820,"Ġshaken":27821,"Ġpersona":27822,"estamp":27823,"367":27824,"brain":27825,"Ġexperimenting":27826,"Ken":27827,"ĠElectronics":27828,"Ġ161":27829,"domain":27830,"Ġgraphical":27831,"bishop":27832,"Ġwhopping":27833,"ĠEvangel":27834,"Ġadvertisers":27835,"ĠSpear":27836,"Ġbids":27837,"Ġdestroys":27838,"utz":27839,"Ġundersc":27840,"ĠADD":27841,"Ġants":27842,"ĠCum":27843,"ipples":27844,"ĠFill":27845,"Ġglanced":27846,"Ġindicted":27847,"ĠEff":27848,"Ġmiscon":27849,"ĠDesktop":27850,"Ġabide":27851,"ãĥĢ":27852,"ĠIo":27853,"ĠCoul":27854,"Ġcapsule":27855,"ĠChrys":27856,"MON":27857,"Ġundes":27858,"ĠIRA":27859,"Ġcitation":27860,"Ġdictate":27861,"ĠNetworks":27862,"ĠConflict":27863,"ĠStuff":27864,"xa":27865,"isec":27866,"ĠChemistry":27867,"Ġquarterly":27868,"Williams":27869,"anan":27870,"Opt":27871,"ĠAlexandria":27872,"outheastern":27873,"ĠSpringfield":27874,"ĠBlacks":27875,"Ġgeography":27876,"242":27877,"Ġutmost":27878,"ĠExxon":27879,"abouts":27880,"EVA":27881,"ĠEnable":27882,"ĠBarr":27883,"Ġdisagreed":27884,"ĠCyprus":27885,"Ġdementia":27886,"Ġlabs":27887,"Ġubiquitous":27888,"ĠLOVE":27889,"Ġconsolidated":27890,"sr":27891,"Ġcreamy":27892,"ĠTimber":27893,"Regardless":27894,"ĠCertificate":27895,"Ġ\"...":27896,"ogenous":27897,"Captain":27898,"Ġinsulting":27899,"ĠSoros":27900,"ĠInstr":27901,"ĠBulgaria":27902,"better":27903,"Ġsucking":27904,"ĠDavidson":27905,"atz":27906,"Ġcollateral":27907,"gif":27908,"Ġplagued":27909,"ĠCancel":27910,"ĠGardner":27911,"RB":27912,"Ġsixteen":27913,"Remove":27914,"uristic":27915,"cook":27916,"Rod":27917,"Ġcomprising":27918,"fle":27919,")âĢĶ":27920,"ĠViking":27921,"growth":27922,"agonal":27923,"Ġsrf":27924,"afety":27925,"mot":27926,"Nearly":27927,"stown":27928,"ĠFactor":27929,"Ġautomobile":27930,"Ġprocedural":27931,"mask":27932,"ampires":27933,"Ġdisappears":27934,"jab":27935,"315":27936,"Ġ1951":27937,"needed":27938,"Ġdaring":27939,"leader":27940,"Ġpodium":27941,"Ġunhealthy":27942,"Ġmund":27943,"Ġpyramid":27944,"ocre":27945,"Ġkissed":27946,"Ġdreamed":27947,"ĠFantastic":27948,"ĠGly":27949,"åĬ":27950,"Ġgreatness":27951,"Ġspices":27952,"Ġmetropolitan":27953,"Ġcompuls":27954,"iets":27955,"1016":27956,"ĠSham":27957,"ĠPyr":27958,"flies":27959,"ĠMidnight":27960,"Ġswallowed":27961,"Ġgenres":27962,"ĠLucky":27963,"ĠRewards":27964,"Ġdispatch":27965,"ĠIPA":27966,"ĠApply":27967,"Ġaven":27968,"alities":27969,"312":27970,"things":27971,"Ġ().":27972,"Ġmates":27973,"ĠSz":27974,"ĠCOP":27975,"olate":27976,"OFF":27977,"Ġrecharge":27978,"caps":27979,"ĠYorker":27980,"icone":27981,"Ġgalaxies":27982,"ileaks":27983,"Dave":27984,"ĠPuzz":27985,"ĠCeltic":27986,"ĠAFC":27987,"276":27988,"ĠSons":27989,"Ġaffirmative":27990,"Hor":27991,"Ġtutorials":27992,"ĠCITY":27993,"ĠRosa":27994,"ĠExtension":27995,"Series":27996,"Ġfats":27997,"Ġrab":27998,"lis":27999,"Ġunic":28000,"Ġeve":28001,"ĠSpin":28002,"Ġadulthood":28003,"typ":28004,"Ġsectarian":28005,"Ġcheckout":28006,"ĠCycl":28007,"Single":28008,"Ġmartyr":28009,"Ġchilling":28010,"888":28011,"oufl":28012,"Ġ];":28013,"Ġcongestion":28014,"mk":28015,"ĠWhereas":28016,"Ġ1938":28017,"urrencies":28018,"erion":28019,"Ġboast":28020,"ĠPatients":28021,"Ġchap":28022,"ĠBD":28023,"realDonaldTrump":28024,"Ġexamines":28025,"hov":28026,"Ġstartling":28027,"ĠBabylon":28028,"wid":28029,"omew":28030,"brance":28031,"ĠOdyssey":28032,"wig":28033,"Ġtorch":28034,"ĠVox":28035,"ĠMoz":28036,"ĠTroll":28037,"ĠAns":28038,"Similarly":28039,"ĠFul":28040,"006":28041,"Unless":28042,"ĠAlone":28043,"stead":28044,"ĠPublisher":28045,"rights":28046,"tu":28047,"ĠDoesn":28048,"Ġprofessionally":28049,"Ġclo":28050,"icz":28051,"Ġsteals":28052,"Ġá":28053,"1986":28054,"Ġsturdy":28055,"ĠJohann":28056,"Ġmedals":28057,"Ġfilings":28058,"ĠFraser":28059,"done":28060,"Ġmultinational":28061,"Ġfeder":28062,"Ġworthless":28063,"Ġpest":28064,"Yesterday":28065,"ankind":28066,"Ġgays":28067,"Ġborne":28068,"ĠPOS":28069,"Picture":28070,"Ġpercentages":28071,"251":28072,"rame":28073,"Ġpotions":28074,"AMD":28075,"ĠLebanese":28076,"Ġrang":28077,"ĠLSU":28078,"ongs":28079,"Ġpeninsula":28080,"ĠClause":28081,"ALK":28082,"oha":28083,"ĠMacBook":28084,"Ġunanimous":28085,"Ġlenders":28086,"Ġhangs":28087,"Ġfranchises":28088,"orers":28089,"ĠUpdates":28090,"Ġisolate":28091,"andro":28092,"Soon":28093,"Ġdisruptive":28094,"ĠSurve":28095,"Ġstitches":28096,"ĠScorp":28097,"ĠDominion":28098,"Ġsupplying":28099,"Arg":28100,"Ġturret":28101,"ĠLuk":28102,"Ġbrackets":28103,"*)":28104,"ĠRevolutionary":28105,"ĠHonest":28106,"Ġnoticing":28107,"ĠShannon":28108,"Ġafforded":28109,"Ġtha":28110,"ĠJanet":28111,"!--":28112,"ĠNarendra":28113,"ĠPlot":28114,"Hol":28115,"sever":28116,"eenth":28117,"Ġobstruction":28118,"Ġ1024":28119,"staff":28120,"jas":28121,"orget":28122,"scenes":28123,"laughs":28124,"ĠFargo":28125,"crime":28126,"Ġorchestr":28127,"Ġdelet":28128,"iliary":28129,"rieved":28130,"Ġmilitar":28131,"ĠGreene":28132,"âĹı":28133,"ãģ¦":28134,"ĠGuards":28135,"Ġunleashed":28136,"ĠWeber":28137,"Ġadjustable":28138,"Ġcaliber":28139,"Ġmotivations":28140,"ĠÃł":28141,"mAh":28142,"ĠLanka":28143,"handle":28144,"Ġpent":28145,"ĠRav":28146,"ĠAngular":28147,"ĠKau":28148,"umbing":28149,"Ġphilanthrop":28150,"Ġdehyd":28151,"Ġtoxicity":28152,"eer":28153,"ĠYORK":28154,"witz":28155,"å¼":28156,"ĠIE":28157,"community":28158,"ĠAH":28159,"Ġretali":28160,"Ġmassively":28161,"ĠDaniels":28162,"ĠDEL":28163,"Ġcarcin":28164,"Url":28165,"Ġrouting":28166,"ĠNPCs":28167,"ĠRAF":28168,"ryce":28169,"Ġwaived":28170,"ĠGuatem":28171,"Everybody":28172,"Ġcovenant":28173,"Ġ173":28174,"Ġrelaxing":28175,"Ġquart":28176,"almost":28177,"Ġguarded":28178,"ĠSoldiers":28179,"ĠPLAY":28180,"Ġoutgoing":28181,"LAND":28182,"Ġrewrite":28183,"ĠMOV":28184,"ĠImper":28185,"ĠSolution":28186,"Ġphenomenal":28187,"Ġlongevity":28188,"Ġimpat":28189,"ĠNissan":28190,"irie":28191,"Ġodor":28192,"ĠZar":28193,"oks":28194,"Ġmilitias":28195,"ĠSPEC":28196,"Ġtolerated":28197,"arser":28198,"ĠBradford":28199,"+,":28200,"Ġsurreal":28201,"sf":28202,"Canadian":28203,"Ġresemblance":28204,"Ġcarbohydrate":28205,"VIEW":28206,"Ġaccessory":28207,"meal":28208,"largest":28209,"iegel":28210,"Someone":28211,"Ġtoughest":28212,"oso":28213,"Ġfunnel":28214,"Ġcondemnation":28215,"luent":28216,"Ġwired":28217,"ĠSunset":28218,"Jesus":28219,"ĠPST":28220,"ĠPages":28221,"ĠTycoon":28222,"ĠPF":28223,"Ġselections":28224,"Ġà¤":28225,"partisan":28226,"Ġhighs":28227,"ĠRune":28228,"Ġcrafts":28229,"lead":28230,"ĠParents":28231,"Ġreclaim":28232,"eker":28233,"ĠAllied":28234,"aeper":28235,"Ġlooming":28236,"Ġbeneficiaries":28237,"ĠHull":28238,"Students":28239,"Jewish":28240,"dj":28241,"Ġpact":28242,"template":28243,"ĠOfficials":28244,"ĠBaylor":28245,"Ġhemp":28246,"Ġyouths":28247,"ĠLevels":28248,"ĠXiao":28249,"ĠChes":28250,"Ġendeavor":28251,"ĠRemoved":28252,"Ġhippocamp":28253,"Hell":28254,"ãĤĬ":28255,"805":28256,"Ġdinosaur":28257,"ĠWrath":28258,"ĠIndonesian":28259,"Ġcalculator":28260,"ĠDictionary":28261,"Ġ420":28262,"ĠMAG":28263,"(_":28264,"!,":28265,"tarians":28266,"Ġrestricting":28267,"racuse":28268,"Ġweekday":28269,"OUNT":28270,"Ġshrugged":28271,"leground":28272,"Ġbald":28273,"ĠDoctors":28274,"Ġtouted":28275,"ĠMaxwell":28276,"Ġ214":28277,"Ġdiplomat":28278,"Ġrepression":28279,"Ġconstituency":28280,"vice":28281,"ranked":28282,"ĠNapoleon":28283,"gang":28284,"ĠForever":28285,"tun":28286,"Ġbulb":28287,"ĠPDT":28288,"ĠCisco":28289,"VEN":28290,"Ġresumed":28291,"Steven":28292,"ĠManitoba":28293,"Ġfabulous":28294,"ĠAgents":28295,"1984":28296,"Ġamusing":28297,"ĠMysteries":28298,"Ġorthodox":28299,"floor":28300,"Ġquestionnaire":28301,"Ġpenetrate":28302,"Ġfilmmakers":28303,"ĠUnc":28304,"Ġstamped":28305,"Ġthirteen":28306,"Ġoutfield":28307,"Ġforwarded":28308,"Ġappra":28309,"Ġaided":28310,"try":28311,"Ġunfocused":28312,"ĠLiz":28313,"ĠWendy":28314,"ĠScene":28315,"Charg":28316,"Ġrejects":28317,"Ġleftist":28318,"ĠProvidence":28319,"ĠBrid":28320,"regn":28321,"Ġprophecy":28322,"ĠLIVE":28323,"499":28324,"Ġforge":28325,"ĠFML":28326,"Ġintrinsic":28327,"ĠFrog":28328,"Ġwont":28329,"ĠHolt":28330,"Ġfamed":28331,"CLUS":28332,"aepernick":28333,"ĠHate":28334,"ĠCay":28335,"Ġregistering":28336,"ortality":28337,"ropy":28338,"ocalyptic":28339,"aan":28340,"nav":28341,"Ġfascist":28342,"IFIED":28343,"Ġimplicated":28344,"ĠResort":28345,"ĠChandler":28346,"ĠBrick":28347,"Pin":28348,"ysc":28349,"Usage":28350,"ĠHelm":28351,"usra":28352,"âĺħâĺħ":28353,"ĠAbbas":28354,"Ġunanimously":28355,"Ġkeeper":28356,"Ġaddicted":28357,"???":28358,"Ġhelmets":28359,"Ġantioxid":28360,"apsed":28361,"808":28362,"giene":28363,"Ġwaits":28364,"Ġminion":28365,"raved":28366,"ĠPorsche":28367,"Ġdreaming":28368,"Ġ171":28369,"ĠCain":28370,"Ġunfor":28371,"asso":28372,"ĠConfiguration":28373,"kun":28374,"hardt":28375,"Ġnested":28376,"ĠLDS":28377,"LES":28378,"Ġtying":28379,"enos":28380,"Ġcue":28381,"ĠMarqu":28382,"skirts":28383,"Ġclicked":28384,"Ġexpiration":28385,"ĠAccordingly":28386,"ĠWC":28387,"Ġblessings":28388,"Ġaddictive":28389,"ĠNarr":28390,"yx":28391,"ĠJaguars":28392,"Ġrents":28393,"ĠSiber":28394,"Ġtipped":28395,"ousse":28396,"ĠFitzgerald":28397,"Ġhierarch":28398,"outine":28399,"Ġwavelength":28400,">.":28401,"chid":28402,"ĠProcessing":28403,"/+":28404,"ranking":28405,"Easy":28406,"ĠConstruct":28407,"Ġtet":28408,"insured":28409,"HUD":28410,"Ġquoting":28411,"Ġcommunicated":28412,"inx":28413,"Ġinmate":28414,"Ġerected":28415,"ĠAbsolutely":28416,"ĠSurely":28417,"Ġunim":28418,"ĠThrone":28419,"heid":28420,"Ġclaws":28421,"Ġsuperstar":28422,"ĠLenn":28423,"ĠWhis":28424,"Uk":28425,"abol":28426,"Ġsket":28427,"ĠNiet":28428,"Ġperks":28429,"Ġaffinity":28430,"Ġopenings":28431,"phasis":28432,"Ġdiscriminate":28433,"Tip":28434,"vc":28435,"Ġgrinding":28436,"ĠJenny":28437,"Ġasthma":28438,"holes":28439,"ĠHomer":28440,"Ġregisters":28441,"ĠGlad":28442,"Ġcreations":28443,"Ġlithium":28444,"Ġapplause":28445,"until":28446,"Justice":28447,"ĠTurks":28448,"Ġscandals":28449,"Ġbake":28450,"tank":28451,"Mech":28452,"ĠMeans":28453,"ĠMaid":28454,"Republicans":28455,"isal":28456,"windows":28457,"ĠSantos":28458,"Ġvegetation":28459,"338":28460,"tri":28461,"Ġflux":28462,"insert":28463,"Ġclarified":28464,"Ġmortg":28465,"ĠChim":28466,"ĠTort":28467,"Ġdisclaim":28468,"metal":28469,"ĠAside":28470,"Ġinduction":28471,"Ġinfl":28472,"Ġatheists":28473,"amph":28474,"Ġether":28475,"ĠVital":28476,"ĠBuilt":28477,"Mind":28478,"Ġweaponry":28479,"SET":28480,"Ġ186":28481,"admin":28482,"gam":28483,"contract":28484,"afa":28485,"Ġderivatives":28486,"Ġsnacks":28487,"Ġchurn":28488,"Econom":28489,"Ġcapped":28490,"ĠUnderstanding":28491,"ĠHers":28492,"ĠIz":28493,"Ġduct":28494,"IENT":28495,"aughty":28496,"ĠâľĶ":28497,"ĠNP":28498,"Ġsailing":28499,"Initialized":28500,"Ġted":28501,"Ġreactors":28502,"ĠLomb":28503,"Ġchoke":28504,"ĠWorm":28505,"Ġadmiration":28506,"Ġswung":28507,"ensibly":28508,"Ġrash":28509,"ĠGoals":28510,"ĠImportant":28511,"Shot":28512,"ĠRas":28513,"Ġtrainers":28514,"ĠBun":28515,"Working":28516,"Ġharmed":28517,"ĠPandora":28518,"ĠLTE":28519,"Ġmushroom":28520,"ĠCHAR":28521,"ĠFee":28522,"ĠMoy":28523,"Born":28524,"oliberal":28525,"ĠMartial":28526,"Ġgentlemen":28527,"Ġlingering":28528,"Official":28529,"Ġgraffiti":28530,"ĠNames":28531,"Der":28532,"Ġquint":28533,"istrate":28534,"azeera":28535,"ĠNOTICE":28536,"ĠFlorence":28537,"Ġpayable":28538,"Ġdepicts":28539,"ĠSpecies":28540,"Heart":28541,"âĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢâĶĢ":28542,"Ġenclosed":28543,"Increases":28544,"Daily":28545,"ĠLis":28546,"Ġenactment":28547,"ĠBacon":28548,"ĠSteele":28549,"demand":28550,"Ġ183":28551,"Ġmouths":28552,"Ġstranded":28553,"Ġenhancement":28554,"011":28555,"ĠWhats":28556,"Ġhealed":28557,"eny":28558,"ĠRab":28559,"Ġ340":28560,"ĠLabyrinth":28561,"roach":28562,"ĠYosh":28563,"ĠClippers":28564,"Ġconcerts":28565,"Internet":28566,"355":28567,"Ġstickers":28568,"Ġtermed":28569,"ĠAxe":28570,"Ġgrandparents":28571,"France":28572,"ĠClim":28573,"ĠUh":28574,"ulic":28575,"Ġthrill":28576,"centric":28577,"ĠOverview":28578,"ĠConduct":28579,"Ġsubstantive":28580,"Ġ182":28581,"mur":28582,"Ġstray":28583,"ĠCoff":28584,"Ġrepetitive":28585,"ĠForgotten":28586,"Ġqualification":28587,"ewitness":28588,"ĠZimbabwe":28589,"Ġsimulated":28590,"ĠJD":28591,"253":28592,"ĠWare":28593,"Ġunsc":28594,"Times":28595,"Ġsummons":28596,"Ġdisconnected":28597,"Ġ184":28598,"cius":28599,"ĠGujar":28600,"odka":28601,"Ġerase":28602,"ĠTobacco":28603,"elected":28604,"Ġuncont":28605,"ĠShepard":28606,"ĠLamp":28607,"Ġalerted":28608,"Ġoperative":28609,"arna":28610,"uint":28611,"Ġnegligence":28612,"acements":28613,"Ġsupra":28614,"Ġprevail":28615,"ĠShark":28616,"Ġbelts":28617,"ãģ«":28618,"Ġtighter":28619,"Engineers":28620,"Ġinactive":28621,"Ġexponent":28622,"ĠWillie":28623,"aples":28624,"Ġheir":28625,"ĠHits":28626,"iann":28627,"ĠSays":28628,"Ġcurrents":28629,"ĠBengal":28630,"Ġarist":28631,"Buffer":28632,"Ġbreeze":28633,"ĠWesley":28634,"Cola":28635,"Ġpronoun":28636,"Ġdeed":28637,"ĠKling":28638,"Ġoft":28639,"Ġinflict":28640,"Ġpunishing":28641,"Ġnm":28642,"iku":28643,"ODUCT":28644,"014":28645,"Ġsubsidy":28646,"ĠDEA":28647,"ĠHerbert":28648,"ĠJal":28649,"Bank":28650,"Ġdeferred":28651,"Ġshipment":28652,"Bott":28653,"Ġalle":28654,"bearing":28655,"HTML":28656,"Offline":28657,"Ġ213":28658,"Ġscrolling":28659,"Ġscanned":28660,"ĠLibyan":28661,"ĠTOP":28662,"chrom":28663,"dt":28664,"column":28665,"PsyNetMessage":28666,"Zero":28667,"Ġtorso":28668,"050":28669,"âķIJ":28670,"Ġimperson":28671,"ĠSchwartz":28672,"udic":28673,"Ġpissed":28674,"ĠSapp":28675,"257":28676,"ĠISPs":28677,"ogl":28678,"Ġsupervised":28679,"Ġadolescent":28680,"Ġattained":28681,"ĠDelivery":28682,"ĠBunny":28683,"Ġ1937":28684,"Ġminiature":28685,"Ġos":28686,"Ġ370":28687,"608":28688,"ĠMourinho":28689,"Ġinnate":28690,"Ġtempo":28691,"ĠNM":28692,"ĠFallen":28693,"009":28694,"Ġprovocative":28695,"Streamer":28696,"ĠBenedict":28697,"ĠBolshe":28698,"Ġturtle":28699,"ĠPCB":28700,"ĠEqual":28701,"Director":28702,"ĠRend":28703,"Ġfluids":28704,"Authorities":28705,"Ġcousins":28706,"requency":28707,"ĠNeighbor":28708,"sets":28709,"shared":28710,"Charles":28711,"password":28712,"Ġgears":28713,"Ġ211":28714,"ĠHardware":28715,"rika":28716,"Ġupstream":28717,"Hom":28718,"Ġdisproportionately":28719,"ivities":28720,"Ġundefined":28721,"Ġelectrons":28722,"Ġcommemor":28723,"Eventually":28724,"Ġ><":28725,"Ġirresponsible":28726,"218":28727,"ĠReleased":28728,"ĠOVER":28729,"ĠIGN":28730,"ĠBread":28731,"stellar":28732,"ĠSage":28733,"tted":28734,"damage":28735,"edition":28736,"ĠPrec":28737,"Ġlime":28738,"Ġconfinement":28739,"Ġcalorie":28740,"weapon":28741,"Ġdiffering":28742,"ĠSina":28743,"mys":28744,"amd":28745,"Ġintricate":28746,"kk":28747,"ĠPAT":28748,"ão":28749,"stones":28750,"links":28751,"Ġranch":28752,"Semitic":28753,"Ġdifferentiate":28754,"ĠSinger":28755,"occupied":28756,"Ġfortress":28757,"cmd":28758,"Ġinterception":28759,"ĠAnkara":28760,"Ġrept":28761,"ĠSolitaire":28762,"Ġremake":28763,"pred":28764,"Ġdared":28765,"autions":28766,"ĠBACK":28767,"Running":28768,"Ġdebugging":28769,"Ġgraphs":28770,"399":28771,"ĠNigel":28772,"Ġbun":28773,"Ġpillow":28774,"Ġprogressed":28775,"fashioned":28776,"Ġobedience":28777,"ERN":28778,"Ġrehears":28779,"Cell":28780,"tl":28781,"Sher":28782,"Ġherald":28783,"ĠPayment":28784,"ĠCory":28785,"ĠDept":28786,"Ġrepent":28787,"ĠWeak":28788,"uckland":28789,"Ġpleasing":28790,"Ġshortages":28791,"Ġjurors":28792,"ĠKab":28793,"qqa":28794,"Anti":28795,"Ġwow":28796,"ĠRCMP":28797,"Ġtsun":28798,"ĠSic":28799,"Ġcomprises":28800,"Ġspies":28801,"Ġprecinct":28802,"nu":28803,"Ġurges":28804,"Ġtimed":28805,"Ġstripes":28806,"ĠBoots":28807,"Ġyen":28808,"Advanced":28809,"Ġdiscrete":28810,"ĠArchangel":28811,"employment":28812,"Diff":28813,"Ġmonuments":28814,"Ġ209":28815,"worker":28816,"Ġ196":28817,"ĠIg":28818,"utterstock":28819,"TPS":28820,"Jac":28821,"Ġhomelessness":28822,"Ġcommentator":28823,"Ġracially":28824,"fing":28825,"seed":28826,"Ele":28827,"ellation":28828,"Ġethanol":28829,"Ġparish":28830,"ĠDong":28831,"ĠAwakening":28832,"Ġdeviation":28833,"ĠBearing":28834,"ĠTsuk":28835,"Ġrecess":28836,"Ġlymph":28837,"ĠCannabis":28838,"åľ":28839,"ĠNEWS":28840,"Ġdra":28841,"ĠStefan":28842,"ĠWrong":28843,"ĠSAM":28844,"Ġloosely":28845,"Ġinterpreter":28846,"ĠPlain":28847,"Government":28848,"Ġbigotry":28849,"Ġgrenades":28850,"avez":28851,"pictured":28852,"Ġmandated":28853,"ĠMonk":28854,"ĠPedro":28855,"Ġlava":28856,"274":28857,"Ġcynical":28858,"ĠScrolls":28859,"locks":28860,"Mp":28861,"Ġcongregation":28862,"ornings":28863,"phil":28864,"ĠIbid":28865,"Ġferv":28866,"Ġdisappearing":28867,"Ġarrogant":28868,"syn":28869,"ĠMaver":28870,"ĠSuit":28871,"241":28872,"Ġabbre":28873,"ackers":28874,"Pa":28875,"ĠYel":28876,"Whenever":28877,"Ġ235":28878,"ĠVine":28879,"ĠAnat":28880,"Ġextinct":28881,"LET":28882,"Ġexecutable":28883,"VERS":28884,"oxide":28885,"DNA":28886,"ĠPrel":28887,"Ġresentment":28888,"Ġcomprise":28889,"ĠAviv":28890,"Ġinterceptions":28891,"Ġprolific":28892,"INA":28893,"ĠErin":28894,"thought":28895,"219":28896,"ĠPsychiatry":28897,"unky":28898,"chemist":28899,"Ho":28900,"ĠMcCoy":28901,"Ġbricks":28902,"Los":28903,"rily":28904,"ĠUSSR":28905,"Ġrud":28906,"Ġlaud":28907,"ĠWise":28908,"ĠEmerald":28909,"Ġrevived":28910,"Ġdamned":28911,"ĠRepair":28912,"idem":28913,"ctica":28914,"Ġpatriarch":28915,"ĠNurs":28916,"meg":28917,"Ġcheapest":28918,"reements":28919,"empty":28920,"ĠCelebr":28921,"Ġdeprivation":28922,"chanted":28923,"ĠThumbnails":28924,"Energy":28925,"ĠEthan":28926,"ĠQing":28927,"Ġopposes":28928,"WIND":28929,"vik":28930,"ĠMau":28931,"ĠSUB":28932,"667":28933,"GRE":28934,"ĠVolunte":28935,"nton":28936,"Cook":28937,"åIJ":28938,"esque":28939,"Ġplummet":28940,"Ġsuing":28941,"Ġpronounce":28942,"Ġresisting":28943,"ĠFishing":28944,"ĠTrials":28945,"Ġyell":28946,"Ġ310":28947,"Ġinduct":28948,"Ġpersonalized":28949,"often":28950,"Reb":28951,"EMBER":28952,"Ġviewpoint":28953,"Ġexistential":28954,"())":28955,"remove":28956,"MENTS":28957,"lasses":28958,"Ġevapor":28959,"Ġaisle":28960,"meta":28961,"Ġreflective":28962,"Ġentitlement":28963,"Ġdevised":28964,"music":28965,"ascade":28966,"Ġwinding":28967,"offset":28968,"Ġaccessibility":28969,"kered":28970,"Better":28971,"ĠJohnston":28972,"thinking":28973,"Snow":28974,"ĠCroatia":28975,"ĠAtomic":28976,"271":28977,"348":28978,"Ġtextbook":28979,"ĠSixth":28980,"ĠاÙĦ":28981,"Ġslider":28982,"ĠBurger":28983,"bol":28984,"Sync":28985,"Ġgrandchildren":28986,"Ġcerv":28987,"+)":28988,"Ġeternity":28989,"Ġtweeting":28990,"Ġspeculative":28991,"Ġpivotal":28992,"ĠWP":28993,"ĠTER":28994,"ynamic":28995,"Ġupl":28996,"ĠCats":28997,"perhaps":28998,"Ġclassmates":28999,"Ġblatant":29000,"'-":29001,"Ġlakh":29002,"antine":29003,"ĠBorg":29004,"iom":29005,"/(":29006,"ĠAthletic":29007,"Ġsar":29008,"OTA":29009,"ĠHoffman":29010,"Nevertheless":29011,"Ġadorable":29012,"Ġspawned":29013,"Associated":29014,"ĠDomestic":29015,"Ġimplant":29016,"ĠLuxem":29017,"ĠKens":29018,"Ġpumps":29019,"ĠSAT":29020,"Attributes":29021,"509":29022,"avour":29023,"Ġcentralized":29024,"ĠTN":29025,"Ġfreshly":29026,"ĠAchieve":29027,"Ġoutsiders":29028,"herty":29029,"ĠRee":29030,"ĠTowers":29031,"ĠDart":29032,"akable":29033,"Ġmp":29034,"ĠHeavenly":29035,"Ġripe":29036,"ĠCaroline":29037,"ryan":29038,"Ġclassics":29039,"Ġretiring":29040,"Ġ228":29041,"Ġah":29042,"Ġdealings":29043,"Ġpunching":29044,"ĠChapman":29045,"Options":29046,"maxwell":29047,"volume":29048,"Ġstal":29049,"Ġexported":29050,"ĠQuite":29051,"Ġnumerical":29052,"Burn":29053,"Fact":29054,"ĠKeystone":29055,"Ġtrending":29056,"Ġaltering":29057,"ĠAfricans":29058,"478":29059,"ĠMN":29060,"ĠKnock":29061,"Ġtemptation":29062,"Ġprestige":29063,"Overview":29064,"ĠTraditional":29065,"ĠBahrain":29066,"Private":29067,"ĠHOU":29068,"Ġbarr":29069,"ĠTat":29070,"Cube":29071,"USD":29072,"ĠGrande":29073,"ĠGat":29074,"ĠFlo":29075,"Ġresides":29076,"Ġindec":29077,"volent":29078,"Ġperpetual":29079,"ubes":29080,"Ġworldview":29081,"ĠQuantum":29082,"Ġfiltered":29083,"Ġensu":29084,"orgetown":29085,"ERSON":29086,"ĠMild":29087,"379":29088,"OTT":29089,"Ã¥":29090,"Ġvitamins":29091,"Ġribbon":29092,"Ġsincerely":29093,"ĠHin":29094,"Ġeighteen":29095,"Ġcontradictory":29096,"Ġglaring":29097,"Ġexpectancy":29098,"Ġconspir":29099,"Ġmonstrous":29100,"Ġ380":29101,"reci":29102,"Ġhandic":29103,"Ġpumped":29104,"Ġindicative":29105,"Ġrapp":29106,"Ġavail":29107,"ĠLEGO":29108,"ĠMarijuana":29109,"1985":29110,"erton":29111,"Ġtwentieth":29112,"################################":29113,"ĠSwamp":29114,"Ġvaluation":29115,"Ġaffiliates":29116,"adjusted":29117,"ĠFacility":29118,"262":29119,"Ġenzymes":29120,"itudinal":29121,"Ġimprint":29122,"Site":29123,"Ġinstaller":29124,"ĠTRA":29125,"mology":29126,"linear":29127,"ĠCollective":29128,"igating":29129,"ĠToken":29130,"Ġspeculated":29131,"KN":29132,"ĠCly":29133,"ority":29134,"Ġdefer":29135,"Ġinspectors":29136,"approved":29137,"RM":29138,"ĠSuns":29139,"Ġinforming":29140,"ĠSyracuse":29141,"ibli":29142,"765":29143,"Ġglove":29144,"Ġauthorize":29145,"â̦â̦â̦â̦â̦â̦â̦â̦":29146,"ĠCruise":29147,"Ġcontracting":29148,"shell":29149,"IFE":29150,"ĠJewel":29151,"pract":29152,"ĠPhotoshop":29153,"ĠKnowing":29154,"harm":29155,"Ġattractions":29156,"adan":29157,"etus":29158,"018":29159,"wagen":29160,"Alt":29161,"Ġmultiply":29162,"Ġequilibrium":29163,":{":29164,"ĠFighters":29165,"ĠEdgar":29166,"Ġfourteen":29167,"Govern":29168,"Ġmisuse":29169,"Ġabusing":29170,"Ġancestry":29171,"ramer":29172,"644":29173,"Ġworms":29174,"Ġthicker":29175,"ĠCombine":29176,"Ġpeasants":29177,"Ġvind":29178,"Ġconquest":29179,"Ġmocked":29180,"Ġcinnamon":29181,"ĠCald":29182,"ĠGallup":29183,"Ġavoidance":29184,"Ġincarnation":29185,"ĠStrat":29186,"Ġtasted":29187,"enta":29188,"ĠNeal":29189,"pared":29190,"Ġterminology":29191,"jection":29192,"Scientists":29193,"ĠINS":29194,"ĠDee":29195,"Ġdirectories":29196,"Road":29197,"ĠShap":29198,"bright":29199,"ĠDirectors":29200,"ĠColumn":29201,"Ġbob":29202,"Ġpreferably":29203,"Ġglitch":29204,"furt":29205,"Ġeg":29206,"idis":29207,"CBC":29208,"Ġsurrendered":29209,"Ġtestament":29210,"336":29211,"uggest":29212,"ĠNil":29213,"another":29214,"Ġpathetic":29215,"ĠDonna":29216,"Ġ218":29217,"ĠAvery":29218,"Ġwhiskey":29219,"Ġfixture":29220,"ĠConquest":29221,"Ġbets":29222,"Occ":29223,"ĠLeicester":29224,"].\"":29225,"Ġ));":29226,"Ġflashes":29227,"456":29228,"Ġmasked":29229,"gebra":29230,"Ġcomputed":29231,"chel":29232,"auder":29233,"Ġdefeats":29234,"ĠLiberation":29235,"ĠOsama":29236,"ĠVive":29237,"Changes":29238,"Channel":29239,"Ġtariffs":29240,"Ġmage":29241,"ĠSax":29242,"Ġinadvertently":29243,"ĠCRE":29244,"ĠReaper":29245,"inky":29246,"grading":29247,"Ġstereotyp":29248,"Ġcurl":29249,"ĠFANT":29250,"Ġframeworks":29251,"Mom":29252,"ĠAnch":29253,"Ġflavour":29254,"carbon":29255,"Ġpermitting":29256,"letcher":29257,"ĠMozilla":29258,"ĠParking":29259,"ĠChamp":29260,"Scroll":29261,"Ġmurderer":29262,"Ġrested":29263,"Ġowes":29264,"ĠPoss":29265,"ADD":29266,"IFF":29267,"resolution":29268,"ĠMining":29269,"Ġcomparative":29270,"Dim":29271,"Ġneighbouring":29272,"ĠAST":29273,"ĠToxic":29274,"Ġbiases":29275,"Ġgunfire":29276,"urous":29277,"ĠMoment":29278,"1983":29279,"Ġpervasive":29280,"ttp":29281,"ĠNormally":29282,"rir":29283,"Sarah":29284,"ĠAlbany":29285,"Ġunsett":29286,"ĠSMS":29287,"ipers":29288,"layer":29289,"ĠWhites":29290,"uple":29291,"Ġturbo":29292,"ĠLeeds":29293,"Ġthats":29294,"ĠMiner":29295,"MER":29296,"ĠReign":29297,"Ġperme":29298,"ĠBlitz":29299,"Ġ1934":29300,"Ġintimidating":29301,"tube":29302,"Ġeccentric":29303,"abolic":29304,"boxes":29305,"ĠAssociates":29306,"votes":29307,"Ġsimulate":29308,"umbo":29309,"astery":29310,"Ġshipments":29311,"FFFF":29312,"anth":29313,"Ġseasoned":29314,"Ġexperimentation":29315,"âĸł":29316,"laws":29317,"Meet":29318,"iddles":29319,"antics":29320,"Rating":29321,"ISIS":29322,"hift":29323,"Ġfronts":29324,"buf":29325,"017":29326,"Ġunatt":29327,"ĠDil":29328,"leases":29329,"ĠGardens":29330,"777":29331,"touch":29332,"vell":29333,"458":29334,"Ġ=====":29335,"saving":29336,"Ġerosion":29337,"ĠQuin":29338,"Ġearns":29339,"Ġaccomplishment":29340,"ĠWei":29341,"Ġ<[":29342,"_____":29343,"Ġirrig":29344,"ĠTeddy":29345,"Ġconquered":29346,"ĠArmored":29347,"Ġasserts":29348,"Ġmanipulating":29349,"ré":29350,"Ġtranscripts":29351,"Gallery":29352,"Ġplotting":29353,"Neil":29354,"Ġbetrayal":29355,"loader":29356,"ĠSul":29357,"Ġdisplacement":29358,"Ġroyalty":29359,"ĠWI":29360,"heit":29361,"ĠDevices":29362,"allel":29363,"Ġmunicipalities":29364,"Ġcanal":29365,"Stars":29366,"ĠUAE":29367,"Ġ\"â̦":29368,"ĠCU":29369,"above":29370,"Ġresonance":29371,"ĠguiActiveUn":29372,"added":29373,"ĠBraves":29374,"ĠIbn":29375,"Ġhereby":29376,"ĠBRE":29377,"Ġshareholder":29378,"ĠHir":29379,"ĠJi":29380,"Ġstrangely":29381,"Ġadmired":29382,"Ġplight":29383,"Ġbachelor":29384,"ĠPole":29385,"ciplinary":29386,"Tony":29387,"ĠArmenian":29388,"Ġunman":29389,"ĠZionist":29390,"Stage":29391,"iscover":29392,"Ġautomotive":29393,"Ġsidelines":29394,"Ġslick":29395,"ĠRenaissance":29396,"ĠFUN":29397,"Images":29398,"ĠHaj":29399,"Ġping":29400,"Ġshortcut":29401,"ĠBlvd":29402,"ĠLooks":29403,"Ġbursts":29404,"Ġclamp":29405,"Ġmish":29406,"Ġsorting":29407,"Ġpatriot":29408,"Ġcorrectness":29409,"ĠScandinav":29410,"ĠCavaliers":29411,"python":29412,"azar":29413,"Ġ375":29414,"ĠJaune":29415,"409":29416,"Ġdetrimental":29417,"Ġstabbing":29418,"Ġpoisoned":29419,"Ġfountain":29420,"ocent":29421,"orst":29422,"ĠMari":29423,"Ġrains":29424,"ĠOvers":29425,"ĠInstitution":29426,"udget":29427,"AMY":29428,"tale":29429,"ĠKR":29430,"ĠPrices":29431,"Ġheadaches":29432,"Ġlandsl":29433,"ĠAura":29434,"Bonus":29435,"ĠZhao":29436,"ĠHip":29437,"Ġhops":29438,"ĠKurdistan":29439,"Ġexploiting":29440,"ryn":29441,"Ġhypocrisy":29442,"opening":29443,"Ġgunshot":29444,"Ġwed":29445,"interstitial":29446,"Interstitial":29447,"Ġamen":29448,"Breaking":29449,"Ġmarketed":29450,"Wire":29451,"ĠCrowd":29452,"Continue":29453,"ĠKnown":29454,"ĠEffective":29455,"orean":29456,"izons":29457,"Joseph":29458,"Ġescalation":29459,"username":29460,"Ġcurtain":29461,"ATES":29462,"ĠPAR":29463,"ĠMiy":29464,"Ġcounterfe":29465,"lene":29466,"Ġcontenders":29467,"daily":29468,"ĠAsc":29469,"ĠPhillip":29470,"mostly":29471,"Ġfilename":29472,"hene":29473,"Ġresembling":29474,"Ġstaging":29475,"ĠChloe":29476,"Ġwiring":29477,"Hon":29478,"ĠRenew":29479,"ottage":29480,"ĠHybrid":29481,"much":29482,"Ġstrokes":29483,"Ġpolicymakers":29484,"APTER":29485,"ĠArkham":29486,"plot":29487,"Ġassistants":29488,"Ġdeport":29489,"ĠSega":29490,"Ġinfluenza":29491,"ĠCursed":29492,"ĠKobe":29493,"Ġskinny":29494,"Provider":29495,"ĠRip":29496,"Ġincremental":29497,"products":29498,"BF":29499,"Ġdome":29500,"ĠCredits":29501,"Ġlosers":29502,"ints":29503,"ĠBetty":29504,"ĠTalent":29505,"ĠDAM":29506,"Lv":29507,"Ess":29508,"Ġdens":29509,"temp":29510,"Judge":29511,"odic":29512,"Ġ'(":29513,"URES":29514,"etsk":29515,"VO":29516,"Ġretrieved":29517,"Ġarchitects":29518,"Ùĩ":29519,"Ġethic":29520,"ĠSecondary":29521,"stocks":29522,"adia":29523,"Ġ325":29524,"ĠOpinion":29525,"Ġsimultaneous":29526,"Ġdizz":29527,"ulp":29528,"Ġsmuggling":29529,"ippery":29530,"Random":29531,"facing":29532,"ĠDas":29533,"Ġstockp":29534,"Ġdisclosures":29535,"pointer":29536,"Ġcoral":29537,"ĠSelection":29538,"ĠPike":29539,"ivalent":29540,"Ġruthless":29541,"ĠRim":29542,"Ġensuing":29543,"ĠExperiment":29544,"Ġcongressman":29545,"Ġbeliever":29546,"Ġunspecified":29547,"ĠMord":29548,"Ġknowledgeable":29549,"ĠVERY":29550,"TX":29551,"Ġstraps":29552,"Ġturf":29553,"apeshifter":29554,"Ġmarital":29555,"Ġflock":29556,"ãģĨ":29557,"263":29558,"AMES":29559,"ĠOpposition":29560,"Ġtreasures":29561,"ĠGOD":29562,"Ġmodeled":29563,"ĠWORLD":29564,"Ġ([":29565,"ĠUsage":29566,"HF":29567,"Ġ$(":29568,"ussed":29569,"Ġpioneer":29570,"Eight":29571,"parse":29572,"bread":29573,"ritz":29574,"ĠMiranda":29575,"ĠKant":29576,"++)":29577,"oren":29578,"Ġprovoked":29579,"Ġbreeds":29580,"ĠIncludes":29581,"ĠPastebin":29582,"ĠFlip":29583,"Java":29584,"Ġbrink":29585,"Ġrumored":29586,"Ġunseen":29587,"Ġgarnered":29588,"ĠDefin":29589,"alted":29590,"Ġtattoos":29591,"Ġhesitation":29592,"isitions":29593,"ĠWeaver":29594,"ĠReporting":29595,"Ġtherapies":29596,"Ġconsultants":29597,"Ġresidual":29598,"ĠMali":29599,"ĠRoma":29600,"iago":29601,"ĠResidents":29602,"ubi":29603,"Ġremedies":29604,"Ġadaptive":29605,"ĠAlive":29606,"ĠBarcl":29607,"Ġwallets":29608,"crypt":29609,"etermination":29610,"ĠPelosi":29611,"Ġslipping":29612,"otonin":29613,"Ġalliances":29614,"patrick":29615,"iris":29616,"Ġorth":29617,"ĠPerkins":29618,"ĠDeV":29619,"ĠGets":29620,"Ġdrying":29621,"gee":29622,"forest":29623,"ĠForget":29624,"orem":29625,"339":29626,"Ġvaguely":29627,"ĠDion":29628,"ĠPorn":29629,"ĠHOW":29630,"Ġpneum":29631,"Ġrubble":29632,"ĠTaste":29633,"encia":29634,"ĠGel":29635,"Ġdst":29636,"Ġ245":29637,"ĠMorocco":29638,"inflamm":29639,"ĠTwins":29640,"Ġbots":29641,"daughter":29642,"ĠBalk":29643,"Ġbrethren":29644,"Ġlogos":29645,"Ġgobl":29646,"fps":29647,"Ġsubdivision":29648,"Ġpawn":29649,"Ġsqueezed":29650,"Ġmorale":29651,"ĠDW":29652,"'\"":29653,"Ġknot":29654,"ooky":29655,"Ġdivisive":29656,"Ġboosted":29657,"chy":29658,"ãĥIJ":29659,"ifact":29660,"Ġnewcomers":29661,"ĠWrestling":29662,"Ġscouts":29663,"wolves":29664,"Rat":29665,"Ġnineteenth":29666,"ĠOsborne":29667,"Stats":29668,"Ġempowered":29669,"Ġpsychopath":29670,"ĠOEM":29671,"uggage":29672,"ĠPK":29673,"ĠMohammad":29674,"Pak":29675,"Ġanarchists":29676,"ĠExtract":29677,"esthes":29678,"ĠStockholm":29679,"loo":29680,"ĠGraph":29681,"Ġdeploying":29682,"ĠStranger":29683,"ĠMold":29684,"Ġstaffer":29685,"Ġdiscounted":29686,"uckle":29687,"please":29688,"ĠLanding":29689,"ÃŃa":29690,"Ġ193":29691,"Ġante":29692,"Ġrepetition":29693,"Ġ+/-":29694,"Ġparody":29695,"Ġlively":29696,"AAA":29697,"ĠHorus":29698,"Ġpits":29699,"inders":29700,"LOC":29701,"ĠVenice":29702,"406":29703,"ĠDiscover":29704,"âĨ":29705,"ellectual":29706,"Ġpens":29707,"Ġeyel":29708,"iguous":29709,"Impl":29710,"Ġjoking":29711,"Ġinval":29712,"ĠBelfast":29713,"Ġcreditors":29714,"ĠSkywalker":29715,"ovsky":29716,"Ġceasefire":29717,"Ġseals":29718,"isoft":29719,")).":29720,"ĠFelix":29721,"ITS":29722,"Ġtresp":29723,"ĠBlockchain":29724,"eware":29725,"ĠSchwar":29726,"enne":29727,"mounted":29728,"ĠBeacon":29729,"lesh":29730,"Ġimmensely":29731,"Ġcheering":29732,"Employ":29733,"scene":29734,"ishly":29735,"atchewan":29736,"ĠNicolas":29737,"Ġdrained":29738,"ĠExit":29739,"ĠAzerb":29740,"jun":29741,"Ġfloated":29742,"uania":29743,"Deep":29744,"Ġsuperv":29745,"Ġmystical":29746,"ĠDollar":29747,"ĠApostle":29748,"ĠREL":29749,"ĠProvided":29750,"ĠBucks":29751,"ãĥ´":29752,"cutting":29753,"Ġenhancements":29754,"ĠPenguins":29755,"ĠIsaiah":29756,"Ġjerk":29757,"ĠWyn":29758,"Ġstalled":29759,"Ġcryptocurrencies":29760,"ĠRoland":29761,"single":29762,"Ġlumin":29763,"ĠFellow":29764,"ĠCapacity":29765,"ĠKazakh":29766,"WN":29767,"Ġfinanced":29768,"389":29769,"Ġtid":29770,"Ġcollusion":29771,"ĠMyr":29772,"îĢ":29773,"Senator":29774,"Ġpediatric":29775,"Ġneatly":29776,"Ġsandwiches":29777,"ĠArchitecture":29778,"Ġtucked":29779,"Ġbalcony":29780,"Ġearthquakes":29781,"quire":29782,"Future":29783,"Ġhefty":29784,"éĹ":29785,"Ġspecializes":29786,"Ġstresses":29787,"Ġsender":29788,"Ġmisunderstanding":29789,"Ġepile":29790,"Ġprovoke":29791,"ĠColors":29792,"Ġdismay":29793,"uko":29794,"[_":29795,"586":29796,"neutral":29797,"Ġdonating":29798,"ĠRandall":29799,"Multi":29800,"Ġconveniently":29801,"ĠSung":29802,"ĠCoca":29803,"Ġtents":29804,"ĠAcceler":29805,"Ġpartnered":29806,"272":29807,"irming":29808,"ĠBAS":29809,"sometimes":29810,"Ġobjected":29811,"ubric":29812,"posed":29813,"LCS":29814,"grass":29815,"Ġattributable":29816,"VIS":29817,"Israeli":29818,"Ġrepeats":29819,"ĠRM":29820,"vag":29821,"uta":29822,"inous":29823,"Ġinert":29824,"ĠMiguel":29825,"æŃ":29826,"ĠHawaiian":29827,"Board":29828,"Ġartific":29829,"ĠAzerbai":29830,"asio":29831,"ĠRent":29832,"AIN":29833,"Ġappliances":29834,"Ġnationality":29835,"Ġasshole":29836,"ĠNeb":29837,"Ġnotch":29838,"hani":29839,"ĠBride":29840,"Availability":29841,"Ġintercepted":29842,"Ġcontinental":29843,"Ġswelling":29844,"ĠPerspect":29845,"bies":29846,".<":29847,"ithmetic":29848,"ĠLara":29849,"Ġtempting":29850,"addr":29851,"Ġoverseeing":29852,"clad":29853,"ĠDV":29854,"ĠGingrich":29855,"Ġmun":29856,"ĠAppropri":29857,"Ġalterations":29858,"ĠPatreon":29859,"Ġhavoc":29860,"Ġdisciplines":29861,"Ġnotoriously":29862,"akuya":29863,"ieri":29864,"?).":29865,"ĠWent":29866,"Ġsilicon":29867,"Ġtremb":29868,"Container":29869,"Known":29870,"Ġmortar":29871,"este":29872,"icka":29873,"Arthur":29874,"ĠPreviously":29875,"ĠMarty":29876,"Ġsparse":29877,"gins":29878,"Ġinward":29879,"ĠParticipant":29880,"Copy":29881,"ĠMisc":29882,"Ġantibiotic":29883,"ĠRetro":29884,"Ġelusive":29885,"Ġassail":29886,"ĠBattalion":29887,"ĠBought":29888,"Ġdiminish":29889,"ĠEuropa":29890,"session":29891,"ĠDangerous":29892,"iesel":29893,"Ġdisbelief":29894,"Ġblasts":29895,"extreme":29896,"ĠBoyd":29897,"ĠProjects":29898,"ĠGuys":29899,"Ġundergone":29900,"Ġgrill":29901,"ĠDwight":29902,"Ġ197":29903,"USER":29904,"Ġfilesystem":29905,"Ġclocks":29906,"Taylor":29907,"Ġwrapper":29908,"Ġfolding":29909,"ousand":29910,"ĠPhilippine":29911,"ATIONAL":29912,"ĠPerth":29913,"Ġashes":29914,"Ġaccumulate":29915,"ĠGateway":29916,"Shop":29917,"orkshire":29918,"Han":29919,"ĠBarrel":29920,"ĠLeh":29921,"ĠXV":29922,"Ġwhim":29923,"Ġrepo":29924,"ĠCG":29925,"ĠMam":29926,"Ġincorporating":29927,"Ġbailout":29928,"Ġlinguistic":29929,"Ġdisinteg":29930,"CLE":29931,"Ġcinematic":29932,"ĠFiber":29933,"Syn":29934,"ilion":29935,"ĠCompos":29936,"chens":29937,"Ġneoc":29938,"Ġboiled":29939,"FINE":29940,"ono":29941,"uncle":29942,"iken":29943,"ĠBM":29944,"ι":29945,"Ġreceipts":29946,"Ġdisposed":29947,"ĠThirty":29948,"ĠRough":29949,"ĠABS":29950,"Ġnotwithstanding":29951,"ollen":29952,"#$":29953,"Ġunreliable":29954,"Ġbloom":29955,"Ġmediocre":29956,"Ġtram":29957,"ĠTasman":29958,"Ġshakes":29959,"Ġmanifesto":29960,"ĠMW":29961,"Ġsatisfactory":29962,"Ġshores":29963,"Ġcomputation":29964,"Ġassertions":29965,"ormons":29966,"arag":29967,"abit":29968,"Democrats":29969,"ĠLoot":29970,"ĠVolks":29971,"haired":29972,"Ġgravitational":29973,"Sing":29974,"ĠMiz":29975,"Ġthrottle":29976,"Ġtyranny":29977,"ĠViews":29978,"Ġrobber":29979,"ĠMinority":29980,"Ġshrine":29981,"scope":29982,"purpose":29983,"Ġnucleus":29984,"ourcing":29985,"ĠUSDA":29986,"ĠDHS":29987,"wra":29988,"ĠBowie":29989,"Scale":29990,"ĠBEL":29991,"xi":29992,"Iter":29993,"Ġ(),":29994,"wright":29995,"Ġsailors":29996,"oused":29997,"NASA":29998,"ĠProof":29999,"ĠMineral":30000,"token":30001,"ĠFD":30002,"Rew":30003,"Ġell":30004,"630":30005,"Ġchancellor":30006,"ĠGos":30007,"Ġamounted":30008,"ĠRecre":30009,"omez":30010,"ĠOptim":30011,"ĠOlive":30012,"Ġtracker":30013,"owler":30014,"ĠUnique":30015,"Root":30016,"Ġmaritime":30017,"ĠQuran":30018,"ĠAdapt":30019,"Ġecosystems":30020,"ĠRepeat":30021,"ĠSoy":30022,"ĠIMP":30023,"Ġgraduating":30024,"andem":30025,"Pur":30026,"ĠReset":30027,"ĠTrick":30028,"ĠPhilly":30029,"ĠTue":30030,"ĠMalaysian":30031,"Ġclimax":30032,"Ġbury":30033,"Ġconspic":30034,"ĠSouthampton":30035,"ĠFlowers":30036,"Ġescorted":30037,"ĠEducational":30038,"ĠIRC":30039,"Ġbrutally":30040,"eating":30041,"Ġpillar":30042,"ĠSang":30043,"ĠJude":30044,"arling":30045,"ĠAmnesty":30046,"Ġreminding":30047,"ĠAdministrative":30048,"hesda":30049,"Ġflashed":30050,"ĠPBS":30051,"perate":30052,"feature":30053,"Ġswipe":30054,"Ġgraves":30055,"oultry":30056,"261":30057,"breaks":30058,"ĠGuer":30059,"Ġshrimp":30060,"ĠVoting":30061,"quist":30062,"Ġanalytical":30063,"Ġtablespoons":30064,"ĠSOU":30065,"Ġresearched":30066,"Ġdisrupted":30067,"Ġjour":30068,"Ġreplica":30069,"Ġcartoons":30070,"bians":30071,"})":30072,"copy":30073,"Got":30074,"ouched":30075,"PUT":30076,"Ġswarm":30077,"notations":30078,"said":30079,"Ġrebuilt":30080,"Ġcollaborate":30081,"Ġraging":30082,"Ġnar":30083,"Ġdemographics":30084,"ĠDDR":30085,"Ġdistrust":30086,"ossier":30087,"ĠKro":30088,"Ġpumpkin":30089,"Ġregrets":30090,"Ġfatalities":30091,"ĠLens":30092,"ĠOle":30093,"pd":30094,"Ġpuppet":30095,"ĠOutlook":30096,"ĠStam":30097,"Ol":30098,"Fair":30099,"UU":30100,"Ġrewritten":30101,"ı":30102,"Ġfascinated":30103,"Ġvectors":30104,"Ġtribunal":30105,"uay":30106,"ĠMats":30107,"ĠCoins":30108,"[[":30109,"Ġ181":30110,"Ġrenders":30111,"ĠKaepernick":30112,"Ġespionage":30113,"Ġsumm":30114,"Ġditch":30115,"Account":30116,"Ġspreadsheet":30117,"Ġmutant":30118,"past":30119,"407":30120,"Ġdye":30121,"Ġinitiation":30122,"Ġ4000":30123,"Ġpunishable":30124,"Ġthinner":30125,"ĠKhal":30126,"Ġintermedi":30127,"Dun":30128,"ĠGotham":30129,"Ġeagerly":30130,"Ġvaginal":30131,"powers":30132,"VW":30133,"ĠWATCHED":30134,"Ġpredator":30135,"amsung":30136,"Ġdisparity":30137,"Ġ[*":30138,"Ġamph":30139,"Ġoutskirts":30140,"ĠSpirits":30141,"Ġskeletal":30142,"л":30143,"ĠRear":30144,"Ġissuance":30145,"ĠLogic":30146,"released":30147,"ZZ":30148,"ĠBound":30149,"Entry":30150,"Ġexits":30151,"isol":30152,"ĠFounder":30153,"Ġwre":30154,"ĠGreenland":30155,"ĠMMO":30156,"taker":30157,"INC":30158,"ãģ¾":30159,"Ġhourly":30160,"henko":30161,"Ġfantasies":30162,"Ġdisob":30163,"Ġdemolition":30164,"ãĥĭ":30165,"Ġenlisted":30166,"ratulations":30167,"Ġmisguided":30168,"Ġensured":30169,"Ġdiscouraged":30170,"mort":30171,"Ġflank":30172,"Ġcess":30173,"Ġreacts":30174,"ĠSere":30175,"sensitive":30176,"ĠSerpent":30177,"assad":30178,"Ġ247":30179,"Ġcalmly":30180,"busters":30181,"Ġbleed":30182,"ĠStro":30183,"Ġamusement":30184,"ĠAntarctica":30185,"Ġscept":30186,"ĠGaw":30187,"aq":30188,"asonic":30189,"Ġsprawling":30190,"native":30191,"aturated":30192,"ĠBattlefield":30193,"IVERS":30194,"EB":30195,"ĠGems":30196,"ĠNorthwestern":30197,"ĠFilms":30198,"ĠAutomatic":30199,"Ġapprehend":30200,"ãģ¨":30201,"ĠguiName":30202,"Ġbackend":30203,"Ġevidenced":30204,"geant":30205,"012":30206,"ĠSiege":30207,"ĠexternalTo":30208,"ĠunfocusedRange":30209,"ĠguiActiveUnfocused":30210,"ĠguiIcon":30211,"ĠexternalToEVA":30212,"ĠexternalToEVAOnly":30213,"Fri":30214,"chard":30215,"enaries":30216,"Ġchiefs":30217,"Ġcf":30218,"ĠHUD":30219,"Ġcorrobor":30220,"ĠdB":30221,"ĠTaken":30222,"ĠPatricia":30223,"rail":30224,"ĠCharm":30225,"ĠLibertarian":30226,"rieve":30227,"Personal":30228,"ĠOUR":30229,"geries":30230,"Ġdumping":30231,"Ġneurological":30232,"itimate":30233,"ĠClintons":30234,"rafted":30235,"ĠMolly":30236,"Ġterminals":30237,"register":30238,"Ġflare":30239,"Ġencoded":30240,"Ġautopsy":30241,"pel":30242,"machine":30243,"Ġexemptions":30244,"ĠRoyals":30245,"distance":30246,"Ġdrafts":30247,"Ġlame":30248,"ĠCunning":30249,"Ġspouses":30250,"ĠMarkets":30251,"ĠCarrier":30252,"Ġimplying":30253,"ĠYak":30254,"sid":30255,"Ġloser":30256,"Ġvigilant":30257,"Ġimpeachment":30258,"Ġaugmented":30259,"ĠEmployees":30260,"Ġunintended":30261,"ternally":30262,"ĠWatt":30263,"Ġrecognizable":30264,"essim":30265,"æĿ":30266,"Ġcoated":30267,"rha":30268,"Ġlieutenant":30269,"ĠLegislation":30270,"published":30271,"444":30272,"013":30273,"Ġideally":30274,"ĠPassword":30275,"Ġsimplify":30276,"ĠMeta":30277,"ĠMRI":30278,"Ġpleading":30279,"organized":30280,"handler":30281,"Ġunravel":30282,"correct":30283,"Ġicy":30284,"Ġparanoid":30285,"Ġpasser":30286,"Ġinspections":30287,"ofer":30288,"ĠHealthcare":30289,"283":30290,"ĠBrut":30291,"iola":30292,"forge":30293,"ĠMedieval":30294,"MSN":30295,"ievers":30296,"ĠProgramming":30297,"åī":30298,"Ġ223":30299,"mu":30300,"ĠCLE":30301,"uga":30302,"Ġshoppers":30303,"Ġinformative":30304,"ĠPlans":30305,"Ġsupplementation":30306,"ĠTests":30307,"tyard":30308,"ocytes":30309,"ĠVega":30310,"ĠGujarat":30311,"ermanent":30312,"Except":30313,"ĠLOT":30314,"alla":30315,"ĠCumm":30316,"ĠOsw":30317,"Ġvenom":30318,"ĠDebt":30319,"ĠDOWN":30320,"Ġreunion":30321,"Ġmuc":30322,"ĠRelief":30323,"Ġgeop":30324,"ĠðŁĺ":30325,"alogue":30326,"Anth":30327,"echo":30328,"Ġcorros":30329,"Ġreplication":30330,"ĠBlazing":30331,"ĠDaughter":30332,"Ġinflic":30333,"ĠLindsey":30334,"ÙĪ":30335,"284":30336,"Exit":30337,"Ġgloom":30338,"TAIN":30339,"Ġundermining":30340,"Ġadvising":30341,"hidden":30342,"Ġoverflow":30343,"Ġgor":30344,"urdue":30345,"Ġechoes":30346,"enhagen":30347,"Ġimpuls":30348,"drug":30349,"cash":30350,"Ġasync":30351,"Ġmirac":30352,"atts":30353,"punk":30354,"Ġpivot":30355,"ĠLegislative":30356,"Ġbloggers":30357,"ĠClaw":30358,"sburg":30359,"dyl":30360,"ĠRecommend":30361,"Ġverte":30362,"Ġprohibiting":30363,"ĠPanther":30364,"Jonathan":30365,"Ġomin":30366,"Ġhateful":30367,"281":30368,"ĠOrche":30369,"ĠMurdoch":30370,"downs":30371,"Ġasymm":30372,"GER":30373,"Always":30374,"Ġinforms":30375,"ĠWM":30376,"ĠPony":30377,"ĠAppendix":30378,"ĠArlington":30379,"Jam":30380,"Ġmedicinal":30381,"ĠSlam":30382,"ITIES":30383,"Ġreaff":30384,"ĠRi":30385,"FG":30386,"Spring":30387,"bool":30388,"Ġthighs":30389,"Ġmarkings":30390,"ĠRaqqa":30391,"ĠLak":30392,"poll":30393,"tsky":30394,"ĠMorty":30395,"ĠDefinition":30396,"Ġdebunk":30397,"endered":30398,"ĠLeone":30399,"avers":30400,"Ġmortgages":30401,"Apparently":30402,"Nic":30403,"haus":30404,"ĠThousands":30405,"auld":30406,"Ġmash":30407,"shoot":30408,"Ġdiarr":30409,"Ġconsciously":30410,"Hero":30411,"eas":30412,"ĠNaturally":30413,"ĠDestroyer":30414,"Ġdashboard":30415,"services":30416,"Rog":30417,"Ġmillennials":30418,"Ġinvade":30419,"-(":30420,"Ġcommissions":30421,"ĠAuckland":30422,"Ġbroadcasts":30423,"Ġfrontal":30424,"Ġcrank":30425,"ĠHistoric":30426,"Ġrumours":30427,"CTV":30428,"Ġsteril":30429,"Ġbooster":30430,"rocket":30431,"ãĤ¼":30432,"utsche":30433,"ĠPI":30434,"Ġ233":30435,"ĠProducer":30436,"ĠAnalytics":30437,"Ġinvaluable":30438,"Ġunintention":30439,"ĠCY":30440,"Ġscrutin":30441,"Ġgigg":30442,"Ġengulf":30443,"Ġproletariat":30444,"Ġhacks":30445,"ĠHew":30446,"arak":30447,"ĠSlime":30448,"ielding":30449,"agher":30450,"ĠElliot":30451,"Ġtelecom":30452,"Ġ219":30453,"ultan":30454,"ĠArbor":30455,"ĠScouts":30456,"Ban":30457,"Ġlifespan":30458,"Ġblasp":30459,"388":30460,"Ġjudiciary":30461,"ĠContinental":30462,"asking":30463,"McC":30464,"LED":30465,"Ġbaggage":30466,"ĠSorcerer":30467,"Ġremnants":30468,"ĠGriffith":30469,"etsu":30470,"ĠSubaru":30471,"ĠPersonality":30472,"designed":30473,"ushima":30474,"agnar":30475,"Ġrecoil":30476,"Ġpassions":30477,"\\\":":30478,"Ġtee":30479,"Ġabolition":30480,"ĠCreating":30481,"jac":30482,"Ġ194":30483,"019":30484,"Ġpillars":30485,"riched":30486,"/\"":30487,"tk":30488,"Ġlivelihood":30489,"Ġroasted":30490,"ahon":30491,"ĠHutch":30492,"assert":30493,"Ġdividend":30494,"Ġknit":30495,"Ġdaunting":30496,"Ġdisturbance":30497,"Ġshale":30498,"Ġcultivated":30499,"Ġrefrigerator":30500,"LB":30501,"ĠNET":30502,"Ġcommercials":30503,"Ġthinkers":30504,"455":30505,"Ġchop":30506,"Broad":30507,"Ġsuspicions":30508,"Ġtagged":30509,"lifting":30510,"Ġstylish":30511,"ĠShields":30512,"Shortly":30513,"Ġtails":30514,"Auth":30515,"STE":30516,"ĠGAME":30517,"Ġseism":30518,"ĠKis":30519,"ologne":30520,"Ġcowork":30521,"Ġforcibly":30522,"Ġthyroid":30523,"ĠPB":30524,"ANE":30525,"married":30526,"horse":30527,"Ġpolymer":30528,"ĠChal":30529,"odor":30530,"DEBUG":30531,"ĠContext":30532,"Ġbliss":30533,"Ġpinpoint":30534,"ĠMathemat":30535,"legram":30536,"ĠWeekend":30537,"Ġlabelled":30538,"Ġbart":30539,"itles":30540,"Ġestrogen":30541,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":30542,"\"'":30543,"Ġvisibly":30544,"Ġoutsider":30545,"aida":30546,"Area":30547,"Ġdissemin":30548,"Ġdishonest":30549,"ĠClosed":30550,"ĠBulletin":30551,"ĠRamsey":30552,"sword":30553,"ĠXI":30554,"ourced":30555,"Same":30556,"346":30557,"ĠRepe":30558,"ĠKou":30559,"cake":30560,"emis":30561,"Cache":30562,"ĠMeaning":30563,"ĠEnlight":30564,"onomy":30565,"Ġmanifestation":30566,"sworth":30567,"Jay":30568,"Ġchore":30569,"ör":30570,"Dream":30571,"Ġsanctioned":30572,"Ġculturally":30573,"ĠAra":30574,"Nav":30575,"Ġtheological":30576,"Ġstrut":30577,"ĠVO":30578,"ĠHandbook":30579,"Ġconstructing":30580,"Ġ¶":30581,"ĠBenefits":30582,"ĠPsychological":30583,"sac":30584,"å¸":30585,"policy":30586,"ĠMatters":30587,"ĠReported":30588,"ĠByte":30589,"Ġvitro":30590,"ĠMaiden":30591,"Ġlam":30592,"ĠJennings":30593,"Ġgarment":30594,"ĠRutgers":30595,"ĠStafford":30596,"ĠWellington":30597,"Ġintermitt":30598,"Ġnpm":30599,"Ġordeal":30600,"Ġplugged":30601,"ooming":30602,"inished":30603,"framework":30604,"Ġtimber":30605,"Ġcass":30606,"Ġ850":30607,"iless":30608,"ĠRedux":30609,"768":30610,"Stre":30611,"Ġsurpassed":30612,"whel":30613,"Ġparallels":30614,"Ġveil":30615,"ĠGI":30616,"ĠREST":30617,"Ġreadiness":30618,"sort":30619,"Ġmodifying":30620,"ĠSlate":30621,"ruff":30622,"Ġmarble":30623,"Ġinfrared":30624,"Ġauditor":30625,"ĠFANTASY":30626,"ĠPoverty":30627,"ĠSPD":30628,"Ġ\"(":30629,"Ky":30630,"RAY":30631,"Ġexecutions":30632,"ĠBeverly":30633,"ĠMarxism":30634,"ĠBurst":30635,"ĠKali":30636,"estones":30637,"Clearly":30638,"Ell":30639,"ãģ§":30640,"ĠProceedings":30641,"Token":30642,"IFIC":30643,"ña":30644,"Central":30645,"ĠHaley":30646,"ĠDrama":30647,"Ġformations":30648,"ORN":30649,"Books":30650,"Ġdominating":30651,"ĠFlyers":30652,"ĠCompanion":30653,"Ġdisciplined":30654,"ĠYugoslav":30655,"ĠSpells":30656,"Ġvengeance":30657,"Ġlandlords":30658,"Len":30659,"ĠOgre":30660,"anoia":30661,"Ġpiercing":30662,"Ġcongreg":30663,"Ġscorer":30664,"obia":30665,"Ġnickel":30666,"ĠLearns":30667,"Ġrejo":30668,"Ġmasterpiece":30669,"Flash":30670,"Ġinhabited":30671,"ĠOpenGL":30672,"ĠDud":30673,"ĠICO":30674,"Ġarter":30675,"Ġplur":30676,"Ġmastery":30677,"Ġlongstanding":30678,"sted":30679,"Ġwines":30680,"Ġtelevised":30681,"ĠShrine":30682,"ĠBayern":30683,"Ġâĵĺ":30684,"Ġenclosure":30685,"john":30686,"Ġprophets":30687,"ĠResurrection":30688,"ĠOrders":30689,"Ġuneven":30690,"rals":30691,"Ġdwind":30692,"ĠLah":30693,"ĠSloven":30694,"378":30695,"Ġinsistence":30696,"affle":30697,"ĠClone":30698,"Ġhardship":30699,"ĠCongressman":30700,"Ġplead":30701,"Ġreviewers":30702,"Ġcured":30703,"Ġ1935":30704,"asley":30705,"fake":30706,"ĠThinking":30707,"ydia":30708,"PART":30709,"ĠDota":30710,"oit":30711,"Ġwhipped":30712,"Ġbouncing":30713,"ĠHispanics":30714,"comings":30715,"Ġcannabin":30716,"ĠChambers":30717,"ĠZack":30718,"Optional":30719,"Ġcoats":30720,"Ġprowess":30721,"ĠNorton":30722,"Ġplainly":30723,"Ġfreight":30724,"Ġinhibition":30725,"Ġclam":30726,"Ġ303":30727,"kef":30728,"aleigh":30729,"Luke":30730,"Ġpsycho":30731,"atorium":30732,"MED":30733,"Ġtreaties":30734,"Ġindisc":30735,"Ġdc":30736,"OPS":30737,"Ġresilient":30738,"ĠInterstate":30739,"Ġslack":30740,"Ġmundane":30741,"Ġestablishes":30742,"359":30743,"Ġstrained":30744,"Ġnond":30745,"Sus":30746,"Ġcaste":30747,"arate":30748,"ieving":30749,"Ġunfairly":30750,"Ġparser":30751,"onial":30752,"ursive":30753,"Via":30754,"ĠOtto":30755,"ĠAuthorities":30756,"stroke":30757,"KR":30758,"ĠMercy":30759,"Ġfurnished":30760,"Ġoutset":30761,"Ġmetic":30762,"1982":30763,"olithic":30764,"ĠTent":30765,"ogical":30766,"ĠAircraft":30767,"Ġhides":30768,"ĠBecame":30769,"Ġeducators":30770,"reaching":30771,"Ġvolatility":30772,"Ġtoddler":30773,"ĠNASCAR":30774,"ĠTwelve":30775,"ĠHighlights":30776,"Ġgrape":30777,"Ġsplits":30778,"Ġpeasant":30779,"Ġreneg":30780,"ĠMSI":30781,"Temp":30782,"stars":30783,"Ġtrek":30784,"ĠHyde":30785,"binding":30786,"Ġrealism":30787,"Ġoxide":30788,"ĠHos":30789,"Ġmounts":30790,"Ġbiting":30791,"Ġcollapsing":30792,"Ġpostal":30793,"Ġmuseums":30794,"Ġdetached":30795,"Ġrespecting":30796,"Ġmonopol":30797,"Ġworkflow":30798,"ĠCake":30799,"Template":30800,"ĠOrganisation":30801,"Ġpersistence":30802,"369":30803,"Coming":30804,"Brad":30805,"Ġredundant":30806,"ĠGTA":30807,"Ġbending":30808,"Ġrevoked":30809,"Ġoffending":30810,"Ġframing":30811,"Ġprintf":30812,"Commun":30813,"members":30814,"Outside":30815,"Ġconstrued":30816,"Ġcoded":30817,"FORE":30818,"Ġchast":30819,"Chat":30820,"Indian":30821,"ĠYard":30822,"?!\"":30823,"ĠPorts":30824,"ĠXavier":30825,"ĠRET":30826,"'.\"":30827,"ĠBoat":30828,"ivated":30829,"icht":30830,"umerable":30831,"Ds":30832,"ĠDunn":30833,"Ġcoffin":30834,"Ġsecurely":30835,"ĠRaptors":30836,"ĠBes":30837,"Installation":30838,"Ġinception":30839,"ĠHealthy":30840,"endants":30841,"Ġpsychologists":30842,"ĠSheikh":30843,"cultural":30844,"ĠBlackBerry":30845,"shift":30846,"Fred":30847,"oche":30848,"Ġcakes":30849,"ĠSEO":30850,"ĠGian":30851,"ĠAsians":30852,"ogging":30853,"element":30854,"Ġpundits":30855,"ĠVaugh":30856,"ĠGavin":30857,"Ġhitter":30858,"Ġdrowned":30859,"Ġchalk":30860,"ĠZika":30861,"Ġmeasles":30862,"802":30863,"â̦..":30864,"ĠAWS":30865,"]\"":30866,"Ġdistort":30867,"ĠMast":30868,"Ġantibodies":30869,"ĠMash":30870,"Memory":30871,"ĠUganda":30872,"ĠProb":30873,"Ġvomiting":30874,"ĠTurns":30875,"Ġoccupying":30876,"Ġevasion":30877,"ĠTherapy":30878,"Ġpromo":30879,"Ġelectr":30880,"Ġblueprint":30881,"ĠDre":30882,"priced":30883,"ĠDepot":30884,"Ġalleviate":30885,"ĠSomali":30886,"marg":30887,"nine":30888,"Ġnostalgia":30889,"ĠShepherd":30890,"Ġcavalry":30891,"Ġtorped":30892,"ĠBloody":30893,"xb":30894,"Ġsank":30895,"Ġgoalt":30896,"reportprint":30897,"embedreportprint":30898,"cloneembedreportprint":30899,"ĠInitially":30900,"ĠFischer":30901,"Ġnoteworthy":30902,"cern":30903,"Ġinefficient":30904,"rawdownload":30905,"rawdownloadcloneembedreportprint":30906,"cation":30907,"ĠDynasty":30908,"lag":30909,"DES":30910,"Ġdistinctly":30911,"ĠEstonia":30912,"Ġopenness":30913,"Ġgossip":30914,"ruck":30915,"Width":30916,"ĠIbrahim":30917,"Ġpetroleum":30918,"Ġavatar":30919,"ĠHed":30920,"atha":30921,"ĠHogwarts":30922,"Ġcaves":30923,"678":30924,"Ġsafeguard":30925,"ĠMog":30926,"isson":30927,"ĠDurham":30928,"slaught":30929,"ĠGraduate":30930,"Ġsubconscious":30931,"ĠExcellent":30932,"ĠDum":30933,"-----":30934,"Ġpiles":30935,"ĠWORK":30936,"ĠGarn":30937,"ĠFol":30938,"ĠATM":30939,"Ġavoids":30940,"ĠTul":30941,"Ġbleak":30942,"ELY":30943,"ivist":30944,"lightly":30945,"Pers":30946,"ĠDob":30947,"ĠLS":30948,"Ġinsanity":30949,"ε":30950,"atalie":30951,"Enlarge":30952,"Ġtwists":30953,"Ġfaulty":30954,"Ġpiracy":30955,"Ġimpover":30956,"Ġrugged":30957,"ĠFashion":30958,"Ġsands":30959,"'?":30960,"swick":30961,"Ġnatives":30962,"Ġhen":30963,"ĠNoise":30964,"ãĥĹ":30965,"Ġgreens":30966,"Ġfreezer":30967,"Ġdynasty":30968,"ĠFathers":30969,"ĠNewark":30970,"Ġarchaeological":30971,"Ġot":30972,"obar":30973,"Ġblockade":30974,"Ġallerg":30975,"LV":30976,"Ġdebit":30977,"ĠRFC":30978,"ĠMilton":30979,"ĠPressure":30980,"Ġwillingly":30981,"Ġdisproportionate":30982,"Ġoppressive":30983,"Ġdiamonds":30984,"Ġbelongings":30985,"1970":30986,"Ġbells":30987,"Ġimperialism":30988,"Ġ227":30989,"Ġexploding":30990,"ĠEclipse":30991,"Ġ1919":30992,"Ġrant":30993,"Ġnominations":30994,"347":30995,"Ġpeacefully":30996,"rica":30997,"ĠFUCK":30998,"Ġvibration":30999,"malink":31000,"Ġropes":31001,"ĠIvanka":31002,"ĠBrewery":31003,"ĠBooker":31004,"ĠOwens":31005,"goers":31006,"Services":31007,"ĠSnape":31008,"Ġ191":31009,"395":31010,"Ġ299":31011,"justice":31012,"Ġbri":31013,"Ġdiscs":31014,"Ġprominently":31015,"Ġvulgar":31016,"Ġskipping":31017,"lves":31018,"Ġtsunami":31019,"374":31020,"ĠUrug":31021,"ĠEid":31022,"recated":31023,"phen":31024,"Ġfaults":31025,"ĠStarted":31026,"950":31027,"Ġpi":31028,"Ġdetector":31029,"Ġbastard":31030,"Ġvalidated":31031,"SpaceEngineers":31032,"OURCE":31033,"Ġ(~":31034,"Ġunsur":31035,"Ġaffirmed":31036,"Ġfascism":31037,"Ġresolving":31038,"ĠChavez":31039,"ĠCyn":31040,"Ġdetract":31041,"Lost":31042,"Ġrigged":31043,"Ġhomage":31044,"ĠBruno":31045,"555":31046,"eca":31047,"Ġpresses":31048,"Ġhumour":31049,"Ġspacing":31050,"Ġ'/":31051,"olkien":31052,"Coun":31053,"OPER":31054,"Tre":31055,"Son":31056,"ĠCambodia":31057,"ierre":31058,"mong":31059,"ozy":31060,"Ġliquidity":31061,"ĠSoviets":31062,"ĠFernando":31063,"Ġ229":31064,"Ġslug":31065,"ĠCatalan":31066,"electric":31067,"Ġscenery":31068,"ĠHearth":31069,"Ġconstrained":31070,"Ġgoalie":31071,"ĠGuidelines":31072,"ĠAmmo":31073,"ĠPearson":31074,"Ġtaxed":31075,"Ġfetus":31076,"Response":31077,"ĠAlexis":31078,"thia":31079,"Guy":31080,"Ġreconstruct":31081,"Ġextremes":31082,"Ġconcluding":31083,"ĠPeg":31084,"ooks":31085,"Ġdeductions":31086,"Rose":31087,"Ġgroundbreaking":31088,"ĠTarg":31089,"ãĥģ":31090,"ĠReve":31091,"resource":31092,"Ġmoons":31093,"Ġelectromagnetic":31094,"Ġamidst":31095,"ĠViktor":31096,"NESS":31097,"BACK":31098,"Ġcommute":31099,"ĠAnaheim":31100,"Ġfluctuations":31101,"640":31102,"Ġnoodles":31103,"ĠCopenhagen":31104,"ĠTide":31105,"ĠGrizz":31106,"ĠSEE":31107,"Ġpipelines":31108,"Ġscars":31109,"endo":31110,"agus":31111,"ĠETF":31112,"/#":31113,"ĠBecome":31114,"448":31115,"Ġvisc":31116,"ĠRecommended":31117,"Ġjumper":31118,"Ġcognition":31119,"Ġassassin":31120,"Ġwitnessing":31121,"ĠSetup":31122,"Ġlac":31123,"vim":31124,"ISM":31125,"pages":31126,"SSL":31127,"358":31128,"Ġadject":31129,"industrial":31130,"lore":31131,"chery":31132,"Ġglitter":31133,"Ġcalf":31134,"Florida":31135,"Ġspoilers":31136,"Ġsucceeds":31137,"Ġchanting":31138,"Ġslogans":31139,"ĠTracy":31140,"Visit":31141,"rology":31142,"Ġmornings":31143,"Ġlineage":31144,"Ġsip":31145,"Ġintensely":31146,"Ġflourish":31147,"ĠSleeping":31148,"ĠFem":31149,"orpor":31150,"ĠKlan":31151,"ĠDarth":31152,"hack":31153,"ĠNielsen":31154,"Ġtumors":31155,"Ġprocurement":31156,"ĠYorkshire":31157,"Ġraided":31158,"KY":31159,"Anna":31160,"Ġ//[":31161,"ĠDisorder":31162,"ĠMustang":31163,"ĠWen":31164,"ĠTrying":31165,"sq":31166,"Ġdeliveries":31167,"Ġshutter":31168,"Ġcerebral":31169,"Ġbipolar":31170,"ĠCN":31171,"lass":31172,"jet":31173,"Ġdebating":31174,">:":31175,"Ġeagle":31176,"grades":31177,"ĠDixon":31178,"UGC":31179,"MAS":31180,"ĠDraco":31181,"ĠMachines":31182,"affer":31183,"Ġeman":31184,"²":31185,"pron":31186,"ĠGym":31187,"Ġcomparatively":31188,"ĠTribunal":31189,"PRO":31190,"Ġlex":31191,"Ġfertile":31192,"Ġdepressing":31193,"Ġsuperficial":31194,"essential":31195,"ĠHunters":31196,"gp":31197,"Ġprominence":31198,"Liber":31199,"ĠAncest":31200,"otechnology":31201,"Ġmocking":31202,"ĠTraff":31203,"ĸļ":31204,"Medium":31205,"Iraq":31206,"Ġpsychiatrist":31207,"Quantity":31208,"ĠLect":31209,"Ġnoisy":31210,"520":31211,"GY":31212,"Ġslapped":31213,"ĠMTV":31214,"Ġpara":31215,"pull":31216,"Multiple":31217,"asher":31218,"Ġnour":31219,"ĠSeg":31220,"Spell":31221,"vous":31222,"ordial":31223,"Senior":31224,"ĠGoldberg":31225,"ĠPlasma":31226,"need":31227,"Ġmessenger":31228,"eret":31229,"Ġteamed":31230,"Ġliteracy":31231,"ĠLeah":31232,"ĠDoyle":31233,"Ġemitted":31234,"UX":31235,"Ġevade":31236,"Ġmaze":31237,"Ġwrongly":31238,"ĠLars":31239,"Ġstereotype":31240,"Ġpledges":31241,"Ġaroma":31242,"ĠMET":31243,"Ġacre":31244,"ĠOD":31245,"Ġff":31246,"Ġbreweries":31247,"ĠHilton":31248,"undle":31249,"ĠKak":31250,"ĠThankfully":31251,"ĠCanucks":31252,"inctions":31253,"ĠAppears":31254,"Ġcoer":31255,"Ġundermined":31256,"rovers":31257,"Andre":31258,"Ġblaze":31259,"umers":31260,"Ġfamine":31261,"amphetamine":31262,"ulkan":31263,"Amount":31264,"Ġdesperation":31265,"wikipedia":31266,"development":31267,"ĠCorinth":31268,"ussia":31269,"Jackson":31270,"LI":31271,"Native":31272,"Rs":31273,"Ohio":31274,"ĠKathleen":31275,"Fortunately":31276,"Ġattendant":31277,"ĠPreferred":31278,"ĠDidn":31279,"ĠVs":31280,"Mis":31281,"Ġrespondent":31282,"Ġboun":31283,"stable":31284,"Ġpaved":31285,"Ġunexpl":31286,"ĠCheney":31287,"LM":31288,"ĠCull":31289,"blown":31290,"Ġconfronting":31291,"ocese":31292,"serving":31293,"Wi":31294,"ĠLithuania":31295,"anni":31296,"Ġstalk":31297,"hd":31298,"Ġvener":31299,"APH":31300,"ynchronous":31301,"URR":31302,"umably":31303,"historic":31304,"Half":31305,"Hay":31306,"Ġresilience":31307,"spection":31308,"Ġabandoning":31309,"Obs":31310,"ĠDebbie":31311,"Ġgradient":31312,"ĠPlaint":31313,"ĠCanal":31314,"ARCH":31315,"Ġexpansive":31316,"Ġfung":31317,"Ġbounced":31318,"Und":31319,"Ġprecautions":31320,"Ġclarification":31321,"Ġdagger":31322,"Ġgrips":31323,"Ġµ":31324,"ĠRivera":31325,"ĠUndead":31326,"isites":31327,"ĠFIRST":31328,"ño":31329,"audi":31330,"Ġhostages":31331,"Ġcompliant":31332,"Ġalumni":31333,"Seven":31334,"Ġcybersecurity":31335,"either":31336,"Collect":31337,"Ġinvariably":31338,"ĠSoci":31339,"Ġlawmaker":31340,"Ġale":31341,"ĠPersonally":31342,"Nazi":31343,"Ġcustomization":31344,"ĠProc":31345,"ĠSaskatchewan":31346,"eaturing":31347,"Ġspared":31348,"Ġdiscontinued":31349,"Ġcomputational":31350,"ĠMotorola":31351,"Ġsupremacist":31352,"governmental":31353,"Ġparadise":31354,"ĠDowning":31355,"ĠNikon":31356,"Ġcatalyst":31357,"berra":31358,"Toronto":31359,"875":31360,"beta":31361,"ĠMacron":31362,"Ġunrealistic":31363,"vector":31364,"ĠVehicles":31365,"itiveness":31366,"ĠRV":31367,"ĠColbert":31368,"sin":31369,"oji":31370,"entin":31371,"ĠKrish":31372,"hello":31373,"ffield":31374,"oky":31375,"ĠTate":31376,"Ġmaple":31377,"Ġaids":31378,"chemical":31379,"334":31380,"nuts":31381,"ĠWarp":31382,"Ġxx":31383,"ĠRobb":31384,"umerous":31385,"_-_":31386,"ftime":31387,"ĠVW":31388,"Ġwinger":31389,"ĠDome":31390,"tools":31391,"ĠPV":31392,"ĠGeorgetown":31393,"Ġgeared":31394,"Ġjihadists":31395,"Ġcp":31396,"Ġsteroids":31397,"Mother":31398,"clerosis":31399,"ĠDRM":31400,"nesia":31401,"Ġlinger":31402,"Ġimmersive":31403,"ĠCOUN":31404,"Ġoutweigh":31405,"ensual":31406,"Band":31407,"Ġtransforms":31408,"matched":31409,"psons":31410,"ĠJudicial":31411,"factor":31412,"Ġreferral":31413,"Ġoddly":31414,"ĠWenger":31415,"Bring":31416,"ĠBows":31417,"602":31418,"ICLE":31419,"Ġlions":31420,"ĠAcademic":31421,"ĠThorn":31422,"ĠRaider":31423,"kefeller":31424,"Storage":31425,"Lower":31426,"ĠOrt":31427,"ĠEquality":31428,"ALT":31429,"ĠSOC":31430,"Types":31431,"Ġlyn":31432,"ĠAsset":31433,"coat":31434,"TPP":31435,"CVE":31436,"ĠPioneer":31437,"application":31438,"Modern":31439,"ĠHK":31440,"Environment":31441,"Alright":31442,"Rain":31443,"IPP":31444,"ĠShiite":31445,"Ġmound":31446,"ĠAbilities":31447,"condition":31448,"Staff":31449,"Ġcompetence":31450,"ĠMoor":31451,"ĠDiablo":31452,"Ġwithheld":31453,"Ġostensibly":31454,"ĠBrom":31455,"Ġmsg":31456,"Ġdenomin":31457,"ĠReferences":31458,"ĠFP":31459,"Ġplunged":31460,"Ġpamph":31461,"moving":31462,"central":31463,"Ġdownright":31464,"Ġfading":31465,"Tal":31466,"Typ":31467,"ĠThy":31468,"ukes":31469,"ithe":31470,"Ġove":31471,"Ġbattled":31472,"Ġseafood":31473,"Ġfigur":31474,"ĠRD":31475,"crop":31476,"Ġsquads":31477,"{\\":31478,"à¹":31479,"ĠEh":31480,"Ġinterviewing":31481,"ĠQin":31482,"Ġaspiring":31483,"PLIC":31484,"Ġclauses":31485,"ĠGast":31486,"ĠNir":31487,"Ġluggage":31488,"Ġhose":31489,"Ġsystemd":31490,"Ġdescending":31491,"ĠRevised":31492,"ĠRails":31493,"align":31494,"709":31495,"337":31496,"Ġfug":31497,"charging":31498,"tags":31499,"Ġuter":31500,"kish":31501,"WARNING":31502,"490":31503,"profits":31504,"Ġvoyage":31505,"Ġace":31506,"ĠVanguard":31507,"ĠTanks":31508,"ĠMuk":31509,"Ġ226":31510,"Safe":31511,"Armor":31512,"Ġvolcanic":31513,"Ġwomb":31514,"ĠMIL":31515,"Ġbeginner":31516,"ĠRecogn":31517,"ĠAAP":31518,"PLAY":31519,")!":31520,"Ġdetecting":31521,"cn":31522,"Ġbreaches":31523,"Basically":31524,"ĠPag":31525,"ĠMunicipal":31526,"ĠIndie":31527,"ĠLaf":31528,"ĠDisable":31529,"ĠOlson":31530,"Ġrestrained":31531,"Ġrulings":31532,"Ġhumane":31533,"events":31534,"ĠCinema":31535,"displayText":31536,"ĠHatch":31537,"actionDate":31538,"onnaissance":31539,"Ġassaulting":31540,"ĠLug":31541,"CHAT":31542,"Ġvigorous":31543,"ĠPerse":31544,"Ġintolerance":31545,"ĠSnapchat":31546,"ĠSharks":31547,"Ġdummy":31548,"ĠDiagn":31549,"ĠGuitar":31550,"imeters":31551,"403":31552,"REG":31553,"Ax":31554,"Ġseparates":31555,"ĠMahm":31556,"Ġtv":31557,"jah":31558,"OOL":31559,"Circ":31560,"ĠWindsor":31561,"ussian":31562,"Ġintuition":31563,"Ġdisdain":31564,"ĠDonovan":31565,"Ġ221":31566,"Emb":31567,"Ġcondemning":31568,"Ġgenerosity":31569,"zzy":31570,"Ġpanties":31571,"ĠPrevent":31572,"ActionCode":31573,"ANA":31574,"342":31575,"externalActionCode":31576,"Ġspecifying":31577,"Ġcrystall":31578,"Jere":31579,"Ġrupt":31580,"ĠApprentice":31581,"Ġprofiling":31582,"к":31583,"Strike":31584,"Ġsideline":31585,"Ġobligated":31586,"Ġoccult":31587,"Ġbureaucratic":31588,"antically":31589,"rupted":31590,"negative":31591,"ĠEthiopia":31592,"ĠCivic":31593,"Ġinsiders":31594,"eligible":31595,"ĠTVs":31596,"ĠBAR":31597,"ĠTI":31598,"iologist":31599,"ĠAIR":31600,"Ġsubstituted":31601,"Arab":31602,"ĠSaul":31603,"ĠYog":31604,"prem":31605,"Ġbuilders":31606,"Ġstationary":31607,"Ġdoubtful":31608,"Ġvigorously":31609,"Ġthrilling":31610,"Physical":31611,"ĠCarey":31612,"ĠHydra":31613,"geoning":31614,"ĠSly":31615,"yton":31616,"Ġborrowers":31617,"ĠParkinson":31618,"Ġë":31619,"ĠJamaica":31620,"Ġsatir":31621,"Ġinsurgents":31622,"ĠFirm":31623,"Ġisot":31624,"ĠKarn":31625,"ourning":31626,"akens":31627,"docs":31628,"little":31629,"ĠMonaco":31630,"CLASS":31631,"Turkey":31632,"Ly":31633,"ĠConan":31634,"assic":31635,"Ġstarred":31636,"ĠPacers":31637,"eties":31638,"Ġtipping":31639,"Moon":31640,"ĠRw":31641,"same":31642,"Ġcavity":31643,"Ġgoof":31644,"ĠZo":31645,"Shock":31646,"ummer":31647,"Ġemphasizes":31648,"Ġregrett":31649,"Ġnovelty":31650,"Ġenvy":31651,"ĠPassive":31652,"rw":31653,"505":31654,"Ġindifferent":31655,"ĠRica":31656,"ĠHimself":31657,"ĠFreddie":31658,"Ġadip":31659,"ä¸Ģ":31660,"Ġbreakout":31661,"Ġhurried":31662,"ĠHuang":31663,"ĠDisk":31664,"Ġroaming":31665,"?????-?????-":31666,"UV":31667,"ĠRicky":31668,"ĠSigma":31669,"Ġmarginalized":31670,"Ġedits":31671,"Ġ304":31672,"memory":31673,"Ġspecimen":31674,"293":31675,"ãģ¯":31676,"Ġvertically":31677,"Ġaudition":31678,"ĠHeck":31679,"Ġcaster":31680,"ĠHoldings":31681,"adal":31682,"ĠCron":31683,"ĠLiam":31684,"Ġdeflect":31685,"Pick":31686,"ĠDebug":31687,"REF":31688,"Ġversatility":31689,"othes":31690,"classified":31691,"ĠMahar":31692,"ĠHort":31693,"Counter":31694,"stasy":31695,"noticed":31696,"331":31697,"ĠShim":31698,"fuck":31699,"ĠBie":31700,"Ġairing":31701,"ĠProtein":31702,"ĠHolding":31703,"Ġspectators":31704,"iliated":31705,"ĠThatcher":31706,"nosis":31707,"ãĥ¼ãĥ³":31708,"Tele":31709,"Boston":31710,"ĠTempl":31711,"stay":31712,"Ġdeclarations":31713,"479":31714,"Volume":31715,"ĠDesigner":31716,"ĠOverwatch":31717,"idae":31718,"Ġonwards":31719,"Ġnets":31720,"ĠManila":31721,"particularly":31722,"Ġpolitic":31723,"oother":31724,"Ġportraits":31725,"Ġpavement":31726,"cffff":31727,"Ġsaints":31728,"Ġbeginners":31729,"ESPN":31730,"Ġshortcomings":31731,"âķIJâķIJ":31732,"Ġcomet":31733,"ĠOrganic":31734,"quel":31735,"Ġhospitalized":31736,"Break":31737,"Ġpeel":31738,"dylib":31739,"aspx":31740,"urances":31741,"ĠTIM":31742,"Pg":31743,"Ġreadable":31744,"ĠMalik":31745,"Ġmuzzle":31746,"Ġbenchmarks":31747,"dal":31748,"ĠVacc":31749,"ĠHicks":31750,"609":31751,"ĠBiblical":31752,"heng":31753,"Ġoverload":31754,"ĠCivilization":31755,"Ġimmoral":31756,"Ġfries":31757,"ãĤĴ":31758,"Ġreproduced":31759,"Ġformulation":31760,"jug":31761,"irez":31762,"gear":31763,"Ġcoached":31764,"MpServer":31765,"ĠSJ":31766,"ĠKw":31767,"Init":31768,"deal":31769,"ĠOro":31770,"ĠLoki":31771,"ĠSongs":31772,"Ġ232":31773,"ĠLouise":31774,"asionally":31775,"Ġuncond":31776,"ollywood":31777,"Ġprogressives":31778,"ĠEnough":31779,"ĠDoe":31780,"Ġwreckage":31781,"Ġbrushed":31782,"ĠBaseType":31783,"Ġzoning":31784,"ishable":31785,"hetically":31786,"ĠCaucus":31787,"ĠHue":31788,"Ġkarma":31789,"ĠSporting":31790,"Ġtrader":31791,"Ġseeming":31792,"ĠCapture":31793,"430":31794,"bish":31795,"Ġtunes":31796,"Ġindoors":31797,"ĠSphere":31798,"ĠDancing":31799,"TERN":31800,"Ġnob":31801,"ĠGST":31802,"maps":31803,"Ġpeppers":31804,"Fit":31805,"Ġoversees":31806,"ĠRabbi":31807,"ĠRuler":31808,"vertising":31809,"office":31810,"xxx":31811,"Ġraft":31812,"Changed":31813,"Ġtextbooks":31814,"Links":31815,"ĠOmn":31816,"ãĢij":31817,"Ġinconvenience":31818,"ĠDonetsk":31819,"=~":31820,"Ġimplicitly":31821,"Ġboosts":31822,"ĠBones":31823,"ĠBoom":31824,"Courtesy":31825,"Ġsensational":31826,"ANY":31827,"Ġgreedy":31828,"eden":31829,"Ġinexper":31830,"ĠLer":31831,"ĠVale":31832,"Ġtighten":31833,"ĠEAR":31834,"ĠNum":31835,"Ġancestor":31836,"Sent":31837,"ĠHorde":31838,"urgical":31839,"allah":31840,"Ġsap":31841,"amba":31842,"ĠSpread":31843,"twitch":31844,"Ġgrandson":31845,"Ġfracture":31846,"Ġmoderator":31847,"ĠSeventh":31848,"ĠReverse":31849,"Ġestimation":31850,"Choose":31851,"Ġparach":31852,"Ġbarric":31853,"ãĢIJ":31854,"Ġcompass":31855,"Ġallergic":31856,"âĢķ":31857,"OTHER":31858,"errilla":31859,"Ġwagon":31860,"Ġzinc":31861,"Ġrubbed":31862,"ĠFuller":31863,"ĠLuxembourg":31864,"ĠHoover":31865,"Ġliar":31866,"ĠEvening":31867,"ĠCobb":31868,"esteem":31869,"Ġselector":31870,"ĠBrawl":31871,"isance":31872,"ĠEk":31873,"Ġtroop":31874,"Ġguts":31875,"ĠAppeal":31876,"ĠTibetan":31877,"Ġroutines":31878,"ĠMent":31879,"Ġsummarized":31880,"steamapps":31881,"Ġtranqu":31882,"Ġ1929":31883,"oran":31884,"ĠAuthent":31885,"Ġgmaxwell":31886,"Ġapprehens":31887,"Ġpoems":31888,"Ġsausage":31889,"ĠWebster":31890,"urus":31891,"Ġthemed":31892,"Ġlounge":31893,"Ġcharger":31894,"Spoiler":31895,"Ġspilled":31896,"hog":31897,"ĠSunder":31898,"ĠAin":31899,"ĠAngry":31900,"Ġdisqual":31901,"ĠFrequency":31902,"ĠEthernet":31903,"Ġhelper":31904,"Percent":31905,"Ġhorrifying":31906,"Ġail":31907,"ĠAllan":31908,"EEE":31909,"ĠCrossing":31910,"449":31911,"Ġholog":31912,"ĠPuzzles":31913,"ĠGoes":31914,"erenn":31915,"604":31916,"ãģı":31917,"ĠRafael":31918,"Ġatten":31919,"ĠEmanuel":31920,"Ġupro":31921,"ĠSusp":31922,"Psych":31923,"ĠTrainer":31924,"ĠNES":31925,"ĠHunts":31926,"becue":31927,"Ġcounselor":31928,"Rule":31929,"Ġtoxins":31930,"Ġbanners":31931,"rifice":31932,"Ġgreeting":31933,"Ġfrenzy":31934,"Ġallocate":31935,"Ġ*)":31936,"expr":31937,"503":31938,"ĠChick":31939,"ĠTorn":31940,"Ġconsolidation":31941,"ĠFletcher":31942,"switch":31943,"frac":31944,"clips":31945,"ĠMcKin":31946,"ĠLunar":31947,"Month":31948,"ITCH":31949,"Ġscholarly":31950,"raped":31951,"398":31952,"Ġ1910":31953,"Ġegreg":31954,"Ġinsecure":31955,"Ġvictorious":31956,"cffffcc":31957,"Ġsingled":31958,"Ġelves":31959,"ĠWond":31960,"burst":31961,"Ġcamoufl":31962,"ĠBLACK":31963,"Ġconditioned":31964,"çī":31965,"answered":31966,"Ġcompulsory":31967,"ascist":31968,"Ġpodcasts":31969,"ĠFrankfurt":31970,"bnb":31971,"Ġneoliberal":31972,"ĠKeyboard":31973,"ĠBelle":31974,"warm":31975,"Ġtrusts":31976,"Ġinsured":31977,"ĠBucc":31978,"usable":31979,"607":31980,"ĠPlains":31981,"Ġ1890":31982,"Ġsabotage":31983,"Ġlodged":31984,"felt":31985,"Ġga":31986,"ĠNarc":31987,"ĠSalem":31988,"Ġseventy":31989,"ĠBlank":31990,"pocket":31991,"Ġwhisper":31992,"Ġmating":31993,"omics":31994,"ĠSalman":31995,"ĠKad":31996,"Ġangered":31997,"Ġcollisions":31998,"Ġextraordinarily":31999,"Ġcoercion":32000,"Ghost":32001,"birds":32002,"èĢ":32003,"kok":32004,"Ġpermissible":32005,"avorable":32006,"Ġpointers":32007,"Ġdissip":32008,"aci":32009,"Ġtheatrical":32010,"ĠCosmic":32011,"Ġforgetting":32012,"Ġfinalized":32013,"大":32014,"yout":32015,"library":32016,"Ġbooming":32017,"ĠBelieve":32018,"ĠTeacher":32019,"ĠLiv":32020,"ĠGOODMAN":32021,"ĠDominican":32022,"ORED":32023,"ĠParties":32024,"Ġprecipitation":32025,"ĠSlot":32026,"Roy":32027,"ĠCombined":32028,"Ġintegrating":32029,"Ġchrome":32030,"Ġintestinal":32031,"ĠRebell":32032,"Ġmatchups":32033,"Ġblockbuster":32034,"ĠLoren":32035,"ĠLevy":32036,"Ġpreaching":32037,"ĠSending":32038,"ĠPurpose":32039,"rax":32040,"fif":32041,"Ġauthoritative":32042,"ĠPET":32043,"astical":32044,"Ġdishon":32045,"Ġchatting":32046,"Ġ\"$:/":32047,"Connection":32048,"Ġrecreate":32049,"Ġdelinqu":32050,"Ġbroth":32051,"ĠDirty":32052,"ĠAdmin":32053,"zman":32054,"Ġscholarships":32055,"Ġ253":32056,"contact":32057,"alsa":32058,"767":32059,"creen":32060,"abbage":32061,"Ġ1915":32062,"Ġblended":32063,"Ġalarmed":32064,"Language":32065,"356":32066,"Ġblends":32067,"ĠChanged":32068,"Wolf":32069,"Ġhepat":32070,"Creating":32071,"Ġpersecut":32072,"Ġsweetness":32073,"arte":32074,"Ġforfeiture":32075,"ĠRoberto":32076,"impro":32077,"NFL":32078,"ĠMagnet":32079,"Detailed":32080,"Ġinsignificant":32081,"ĠPOLIT":32082,"ĠBBQ":32083,"ĠCPS":32084,"Ġseaw":32085,"aminer":32086,"mL":32087,"endif":32088,"finals":32089,"Ġ265":32090,"uish":32091,"Ġ})":32092,"ĠProblems":32093,"Ġemblem":32094,"Ġseriousness":32095,"Ġparsing":32096,"Ġsubstitution":32097,"Ġpressured":32098,"Ġrecycled":32099,"aleb":32100,"Ruby":32101,"Ġproficiency":32102,"Driver":32103,"ĠWester":32104,":'":32105,"AFTA":32106,"Ġmantle":32107,"ĠClayton":32108,"flag":32109,"Ġpractitioner":32110,"covered":32111,"ĠStruct":32112,"addafi":32113,"425":32114,"ĠTownship":32115,"ĠHydro":32116,"Louis":32117,"343":32118,"Ġcondo":32119,"ĠTao":32120,"Ġutilization":32121,"Ġnausea":32122,"ĠDems":32123,"ridges":32124,"pause":32125,"Ġformulas":32126,"Ġchallenger":32127,"376":32128,"Ġdefective":32129,"ĠRailway":32130,"ĠPubMed":32131,"Ġyogurt":32132,"lbs":32133,"ĠNorfolk":32134,"OPE":32135,"ĠMoody":32136,"Ġdistributor":32137,"Ġscrolls":32138,"Ġextracts":32139,"Stan":32140,"Ġviability":32141,"Ġexposes":32142,"Ġstarvation":32143,"ĠSteps":32144,"ĠDodd":32145,"few":32146,"STD":32147,"332":32148,"Ġclosures":32149,"Ġcomplementary":32150,"ĠSasha":32151,"umpy":32152,"Ġmonet":32153,"Ġarticulate":32154,"ĠDoct":32155,"killer":32156,"Ġscrim":32157,"Ġ264":32158,"Ġprostitutes":32159,"Ġsevered":32160,"Ġattachments":32161,"Ġcooled":32162,"Lev":32163,"ĠFalk":32164,"fail":32165,"Ġpoliceman":32166,"ĠDag":32167,"Ġprayed":32168,"ĠKernel":32169,"Ġclut":32170,"Ġcath":32171,"Ġanomaly":32172,"Storm":32173,"emaker":32174,"ĠBreakfast":32175,"uli":32176,"oire":32177,"JJ":32178,"hz":32179,"Operation":32180,"ĠSick":32181,"354":32182,"ĠGuatemala":32183,"Rate":32184,"Ġexposures":32185,"faces":32186,"ĠArchae":32187,"raf":32188,"ĠMia":32189,"Ġ2025":32190,"Ġopaque":32191,"Ġdisguised":32192,"ĠHeadquarters":32193,"Sah":32194,"Ġpots":32195,"978":32196,"ĠMalf":32197,"Ġfrowned":32198,"Ġpoisonous":32199,"ĠConvers":32200,"eeks":32201,"Ġcrab":32202,".\"\"":32203,"Ġtreason":32204,"Ġranc":32205,"Ġescalating":32206,"Ġwarr":32207,"Ġmobs":32208,"Ġlamps":32209,"ĠSunshine":32210,"ĠBrunswick":32211,"Phones":32212,"Ġspelled":32213,"ĠSkip":32214,"Ġ2050":32215,"Ġ1911":32216,"ĠPluto":32217,"ĠAmend":32218,"Ġmeats":32219,"387":32220,"Ġstomp":32221,"ĠZhou":32222,"ĠLeviathan":32223,"ĠHazard":32224,"adv":32225,"ĠOrwell":32226,"Ġaloud":32227,"Ġbumper":32228,"ĠAnarch":32229,"ubuntu":32230,"ĠSerious":32231,"fitting":32232,"ĠOptional":32233,"ĠCecil":32234,"REAM":32235,"Ġserotonin":32236,"Ġcultivate":32237,"agogue":32238,"}\\":32239,"Ġmosques":32240,"ĠSunny":32241,"Ġreactive":32242,"revolution":32243,"ĠLup":32244,"ĠFedora":32245,"Ġdefenseman":32246,"ĠVID":32247,"istine":32248,"Ġdrowning":32249,"ĠBroadcasting":32250,"Ġthriller":32251,"ĠScy":32252,"Ġaccelerating":32253,"Ġdirects":32254,"odied":32255,"bike":32256,"duration":32257,"Ġpainfully":32258,"Redd":32259,"Ġproductions":32260,"Ġgag":32261,"Ġwhist":32262,"Ġsock":32263,"Ġinfinitely":32264,"ĠConcern":32265,"ĠCitadel":32266,"Ġlieu":32267,"Ġcandles":32268,"ogeneous":32269,"arger":32270,"Ġheavenly":32271,"inflammatory":32272,"Performance":32273,"Cs":32274,"ructose":32275,"azaki":32276,"Ġpessim":32277,"Ġinference":32278,"Ġpowd":32279,"ĠZoe":32280,"Ġpaints":32281,"Ġdazz":32282,"pta":32283,"-----------":32284,"Ġinspir":32285,"ĠExperimental":32286,"ĠKnife":32287,"regor":32288,"bors":32289,"Ġshowers":32290,"romeda":32291,"Ġsaint":32292,"Ġbenign":32293,"ĠJiang":32294,"Ġenvisioned":32295,"Ġshroud":32296,"IFT":32297,"HO":32298,"Ġshuff":32299,"ĠICC":32300,"Ġsegreg":32301,"Ġrevisit":32302,"ighthouse":32303,"Li":32304,"Ġsubstrate":32305,"ĠSeas":32306,"ĠReward":32307,"ĠHep":32308,"ĠBrass":32309,"sbm":32310,"Ġeliminates":32311,"Ġstamina":32312,"ĠVAT":32313,"ĠLoan":32314,"Ġconstraint":32315,"Ġappropriated":32316,"Ġpes":32317,"ĠALE":32318,"ranging":32319,"Ġ404":32320,"392":32321,"Ġintellectuals":32322,"achu":32323,"Ġrestructuring":32324,"ĠLevin":32325,"Ġrunes":32326,"Ġdelightful":32327,"Ġcarbohydrates":32328,"ĠModels":32329,"ĠExpo":32330,"Ġtransporting":32331,"alloc":32332,"Ġringing":32333,"Samsung":32334,"Ġscarcely":32335,"ĠURLs":32336,"ĠMAS":32337,"Ġprototypes":32338,"Ġnarrator":32339,"ĠCPUs":32340,"cdn":32341,"ĠBarton":32342,"Ġdecidedly":32343,"ĠShu":32344,"ixir":32345,"ocious":32346,"ĠMyst":32347,"Nintendo":32348,"Ġreuse":32349,"Ġforgiven":32350,"Few":32351,"inical":32352,"nat":32353,"Ġseamless":32354,"ĠEva":32355,"ĠEVE":32356,"ĠJO":32357,"landers":32358,"Ġsofter":32359,"negie":32360,"Ġtransient":32361,"Ġorbital":32362,"Ġfulfil":32363,"ĠKom":32364,"Hopefully":32365,"Ġdynamically":32366,"ĠHunger":32367,"åĽ":32368,"ĠArmenia":32369,"elman":32370,"berto":32371,"Ġpige":32372,"ĠIDs":32373,"limit":32374,"Ġveins":32375,"Ġsoaring":32376,"packs":32377,"Golden":32378,"ĠCrab":32379,"istor":32380,"ĠRPM":32381,"Ġ$$":32382,"gression":32383,"Ġjihadist":32384,"Ġgamble":32385,"Ġcareg":32386,"Ġinflated":32387,"Face":32388,"ĠFirearms":32389,"ĠEmmanuel":32390,"âĿ":32391,"Ġshocks":32392,"grab":32393,"Ġsplend":32394,"ĠHPV":32395,"abortion":32396,"Above":32397,"Entity":32398,"players":32399,"Ġcommenced":32400,"ulence":32401,"Ġfulfillment":32402,"Ġembodiments":32403,"ĠWelfare":32404,"Ġhail":32405,"Ġ<@":32406,"tten":32407,"Ġcatcher":32408,"ĠJazeera":32409,"Ġvolcano":32410,"Ġstabilize":32411,"ĠHandler":32412,"Ġintensified":32413,"ĠAbrams":32414,"Ġhumiliation":32415,"paced":32416,"605":32417,"ĠCentOS":32418,"Specific":32419,"Ġheed":32420,"ĠCAM":32421,"ĠGalile":32422,"Die":32423,"Ġabolished":32424,"ĠThomson":32425,"ĠTeachers":32426,"ĠWass":32427,"jong":32428,"ĠISBN":32429,"ĠAllies":32430,"shake":32431,"å·":32432,"vict":32433,"Howard":32434,"Ġdeem":32435,"Ġexceedingly":32436,"ĠSmartstocks":32437,"ibe":32438,"Ġdoorway":32439,"Ġcompeted":32440,"igmat":32441,"Ġnationalists":32442,"Ġgroom":32443,"ĠKeen":32444,"Ġdisposable":32445,"decl":32446,"ĠTolkien":32447,"ĠScheme":32448,"Ġbiod":32449,"Ġavid":32450,"ĠElon":32451,"agar":32452,"ĠTSA":32453,"Roman":32454,"Ġartificially":32455,"Ġadvisors":32456,"XL":32457,"ĠInferno":32458,"366":32459,"Ġtedious":32460,"ĠPhotography":32461,"ĠCarrie":32462,"Ġtrope":32463,"ĠSandra":32464,"Ġdecimal":32465,"Queen":32466,"ĠGundam":32467,"ĠOM":32468,"otech":32469,"NBA":32470,"Ġ1932":32471,"Ġentrenched":32472,"ĠMarion":32473,"Ġfraternity":32474,"Labour":32475,"Henry":32476,"Ġlatitude":32477,"Either":32478,"Ġenhances":32479,"ĠPotential":32480,"Ġshines":32481,"idad":32482,"Ġbreadth":32483,"Ġcapacities":32484,"ĠðŁĻĤ":32485,"ĠBronx":32486,"Ġsexes":32487,"Ġdifferentiation":32488,"Ġheavyweight":32489,"ĠTaj":32490,"dra":32491,"Ġmigrate":32492,"Ġexhaustion":32493,"ĠRUN":32494,"elsius":32495,"ĠCuomo":32496,"Ġguitars":32497,"Ġclones":32498,"ĠSomew":32499,"ĠPry":32500,"-------------":32501,"Ġwarranted":32502,"cycles":32503,"Ġsalvage":32504,"Ġdisks":32505,"RANT":32506,"ĠNGOs":32507,"ĠMartian":32508,"\":[{\"":32509,"Ġaddicts":32510,"ojure":32511,"illet":32512,"Ġamazingly":32513,"artments":32514,"pixel":32515,"ĠGPUs":32516,"Layout":32517,"è£":32518,"ĠTamil":32519,"ĠBasil":32520,"Ġimpartial":32521,"ĠStructure":32522,"fork":32523,"bryce":32524,"Ġridge":32525,"ĠHamburg":32526,"rious":32527,"Ġblitz":32528,"cigarettes":32529,"Ġcanned":32530,"402":32531,"Ġironically":32532,"Ġcompassionate":32533,"ĠHawkins":32534,".#":32535,"ĠCathedral":32536,"Ġrallied":32537,"internal":32538,"Ġquota":32539,"stakes":32540,"TEXT":32541,"mom":32542,"Ġcompletes":32543,"Ġ238":32544,"Ġshrug":32545,"ãĥij":32546,"ĠNinth":32547,"Ġrevise":32548,"ĠProvider":32549,"Ġtreacher":32550,"Ġquasi":32551,"ĠPRES":32552,"Ġdeposition":32553,"Ġconfidentiality":32554,"issors":32555,"Ġimbalance":32556,"Ġspanning":32557,"Ġangular":32558,"ĠCul":32559,"communication":32560,"ĠNora":32561,"ĠGenius":32562,"opter":32563,"Ġsacked":32564,"Spot":32565,"Ġfinely":32566,"ĠCHR":32567,"282":32568,"waves":32569,"Palest":32570,"ĠRohing":32571,"NL":32572,"è¿":32573,"Ġshitty":32574,"ĠScalia":32575,"475":32576,"Progress":32577,"Ġreferencing":32578,"Ġclassrooms":32579,"abee":32580,"Ġsod":32581,"hesion":32582,"708":32583,"ĠZuckerberg":32584,"ĠFinish":32585,"ĠScotia":32586,"ĠSavior":32587,"ĠInstallation":32588,"antha":32589,"(-":32590,"Ġ302":32591,"ĠPunk":32592,"Ġcrater":32593,"youtu":32594,"Ġroast":32595,"Ġinfluencing":32596,"Ġdup":32597,"ĠJR":32598,"ĠGrav":32599,"Ġstature":32600,"Ġbathrooms":32601,"Aside":32602,"Wiki":32603,"mean":32604,"ĠZak":32605,"ĠOnes":32606,"ĠNath":32607,"Ġhypert":32608,"Ġcommencement":32609,"Civil":32610,"Ġmoderately":32611,"Ġdistributors":32612,"Ġbreastfeeding":32613,"Ġ980":32614,"ĠSik":32615,"ĠCig":32616,"ĠAMER":32617,"RIP":32618,"ĠCareer":32619,"usting":32620,"Ġmessed":32621,"Ġeh":32622,"ĠJensen":32623,"/$":32624,"Ġblackmail":32625,"Ġconversions":32626,"Ġscientifically":32627,"Ġmantra":32628,"paying":32629,"Ġivory":32630,"ĠCourts":32631,"OUGH":32632,"auntlet":32633,"Serial":32634,"Brow":32635,"ĠHundreds":32636,"323":32637,"Ġpee":32638,"Ġlinux":32639,"Ġsubmer":32640,"ĠPrincipal":32641,"485":32642,"ĠDSL":32643,"ĠCousins":32644,"Ġdoctrines":32645,"ĠAthletics":32646,"Ġ315":32647,"ĠKarma":32648,"Ġattent":32649,"urger":32650,"Ġprescribe":32651,"Ġencaps":32652,"ĠCame":32653,"Ġsecretive":32654,"ĠCrimes":32655,"dn":32656,"Clean":32657,"ĠEgyptians":32658,"ĠCarpenter":32659,"Ġll":32660,"Hum":32661,"ĠMilo":32662,"Ġcapitalists":32663,"Ġbriefed":32664,"Twe":32665,"ĠBasin":32666,"elvet":32667,"Mos":32668,"Ġplunge":32669,"ĠKaiser":32670,"ĠFuj":32671,"illin":32672,"Ġsafeguards":32673,"Ġoste":32674,"ĠOpportunity":32675,"ĠMafia":32676,"ĠCalling":32677,"apa":32678,"urban":32679,"brush":32680,"illard":32681,"cé":32682,"intelligence":32683,"ĠLob":32684,"ĠDruid":32685,"Ġsmoother":32686,"Ġfooting":32687,"Ġmotorists":32688,"arcity":32689,"Ġmasculinity":32690,"Ġmism":32691,"Ġabdominal":32692,"ĠTavern":32693,"ĠRoh":32694,"Ġescapes":32695,"signed":32696,"Anthony":32697,"Ġsacrificing":32698,"Ġintimacy":32699,"Ġanterior":32700,"ĠKod":32701,"Ġmotif":32702,"Ġgraz":32703,"Ġvisualization":32704,"Ġguitarist":32705,"ĠTrotsky":32706,"magic":32707,"Dar":32708,"ĠMori":32709,"Ġwards":32710,"Ġtoilets":32711,"lest":32712,"Ġteleport":32713,"ĠSundays":32714,"ĠPlat":32715,"ETS":32716,"ĠeSports":32717,"Patrick":32718,"ĠKatherine":32719,"enko":32720,"Ġhassle":32721,"ĠMick":32722,"ggles":32723,"Ġhob":32724,"aintain":32725,"Ġairborne":32726,"Ġspans":32727,"Ġchili":32728,"Ġaperture":32729,"Ġvolunteered":32730,"ĠIncident":32731,"ĠFres":32732,"ĠVeteran":32733,"aughtered":32734,"ingo":32735,"Ġuninsured":32736,"CLOSE":32737,"Ġfuse":32738,"Ġerotic":32739,"Ġadvertise":32740,"raising":32741,"Texture":32742,"Ġattends":32743,"ĠREAL":32744,"uddled":32745,"Ġsmoot":32746,"Ġ305":32747,"ĠWillis":32748,"Ġblond":32749,"Analysis":32750,"ĠVT":32751,"onica":32752,"Ġstronghold":32753,"RF":32754,"NM":32755,".>>":32756,"Ġprosperous":32757,"Ġboasted":32758,"292":32759,"ĠManufacturing":32760,"PRESS":32761,"gren":32762,"Ġpharmacy":32763,"ĠRockefeller":32764,"kai":32765,"Ġthumbs":32766,"ĠHut":32767,"Ġmotherboard":32768,"Ġguardians":32769,"ĠAlter":32770,"llular":32771,"Ġshack":32772,"Ġwisely":32773,"Ġbackbone":32774,"erva":32775,"Ġsuicides":32776,"ĠMcGregor":32777,"ijah":32778,"Emer":32779,"ĠBrav":32780,"Ġdesignate":32781,"POST":32782,"produced":32783,"Ġcleansing":32784,"irlwind":32785,"existent":32786,"ĠHumph":32787,"ĠPayne":32788,"Ġvested":32789,"Å¡":32790,"Ġstringent":32791,"iona":32792,"Ġunsub":32793,"Ġsummed":32794,"ĠHercules":32795,"subject":32796,"ĠRagnar":32797,"ĠNos":32798,"Ġcharacterization":32799,"Ġsavvy":32800,"ĠDawson":32801,"ĠCasino":32802,"Ġfri":32803,"ĠBarrier":32804,"Ġmisinformation":32805,"Ġinsulation":32806,"Ġcorridors":32807,"Ġairplanes":32808,"ĠNoct":32809,"ahi":32810,"Ġ1916":32811,"kb":32812,"armac":32813,"Ġshun":32814,"Ġschema":32815,"Ġhorrified":32816,"Ġ239":32817,"aunders":32818,"NB":32819,"iates":32820,"erity":32821,"ĠShard":32822,"Ġrarity":32823,"Ġgrouped":32824,"ĠGhana":32825,"against":32826,"ĠBiological":32827,"ĠAware":32828,"owell":32829,"ÏĦ":32830,"ĠBeau":32831,"shaw":32832,"Hack":32833,"ĠJulius":32834,"USS":32835,"olson":32836,"auna":32837,"cru":32838,"ĠMaurice":32839,"ĠIk":32840,"Ġsequencing":32841,"Ġradicals":32842,"Ġ(?,":32843,"virtual":32844,"Ġanyways":32845,"Ġreperc":32846,"Ġhandlers":32847,"Ġhesitant":32848,"éĥ":32849,"ĠMF":32850,"plementation":32851,"associated":32852,"Ġcampaigned":32853,"ĠYue":32854,"utations":32855,"ĠYoga":32856,"Ġsimmer":32857,"Ġrods":32858,"Ġmelody":32859,"Ġconvoy":32860,"videos":32861,"Ġscreened":32862,"Neg":32863,"ochemical":32864,"Ġ())":32865,"Ġultras":32866,"Ġantip":32867,"ĠIslanders":32868,"704":32869,"Ġfetish":32870,"Ġridiculously":32871,"ĠKart":32872,"Ġmitochondrial":32873,"Ġinterfering":32874,"Builder":32875,"Ġoverfl":32876,"Ġacne":32877,"ĠMud":32878,"ĠKerr":32879,"flex":32880,"ĠPostal":32881,"ĠBaltic":32882,"477":32883,"ĠPersons":32884,"ourage":32885,"HB":32886,"ĠMuse":32887,"ĠImmortal":32888,"ĠDriving":32889,"Ġpetitions":32890,"Ġsubscript":32891,"Ġsorce":32892,"ĠProcessor":32893,"uton":32894,"Sony":32895,"Ġphon":32896,"Ġraced":32897,"ĠAnthrop":32898,"Ġdaytime":32899,"ĠExercise":32900,"Adding":32901,"Ġengages":32902,"ĠQualcomm":32903,"Ġmiracles":32904,"Ġmemes":32905,"ĠDrink":32906,"ĠOrioles":32907,"Ġhairs":32908,"ĠPolar":32909,"athom":32910,"Ġslippery":32911,"ĠRemy":32912,"Ġcaramel":32913,"ĠYEAR":32914,"Ġalk":32915,"Ign":32916,"aution":32917,"ĠMerlin":32918,"ĠCran":32919,"Ġapologies":32920,"Ġ410":32921,"Ġouting":32922,"ĠMemories":32923,"appointed":32924,"Ġcountered":32925,"uld":32926,"posing":32927,"Ġfirewall":32928,"ĠWast":32929,"ĠWet":32930,"worked":32931,"seller":32932,"Ġrepealed":32933,"ereo":32934,"assuming":32935,"BLIC":32936,"mite":32937,"ĠCEOs":32938,"ĠChapel":32939,"elligent":32940,"________________________":32941,"Dog":32942,"Ġwart":32943,"Ġsubscriber":32944,"sports":32945,"Ġbegged":32946,"ĠMV":32947,"Ġsemif":32948,"ethical":32949,"Ġpreach":32950,"Ġrevital":32951,"Ġpunitive":32952,"Ġshortcuts":32953,"Ġinstituted":32954,"ĠWarsaw":32955,"Ġabdomen":32956,"ĠKING":32957,"Ġsuperintendent":32958,"Ġfry":32959,"ĠGeo":32960,"TOR":32961,"Ġcontradictions":32962,"aptic":32963,"Ġlandscapes":32964,"bugs":32965,"Ġclust":32966,"Ġvolley":32967,"cribed":32968,"Ġtandem":32969,"Ġrobes":32970,"WHAT":32971,"Ġpromoter":32972,"Ġeloqu":32973,"reviewed":32974,"ĠDK":32975,"ĠPlato":32976,"Ġfps":32977,"Tank":32978,"ĠDerrick":32979,"Ġprioritize":32980,"asper":32981,"ĠHonduras":32982,"ĠCompleted":32983,"nec":32984,"Ġmog":32985,"nir":32986,"ĠMayo":32987,"DEF":32988,"stall":32989,"inness":32990,"ĠVolkswagen":32991,"Ġprecaution":32992,"ĠMell":32993,"iak":32994,"istries":32995,"Ġ248":32996,"Ġoverlapping":32997,"Senate":32998,"ĠEnhance":32999,"resy":33000,"racial":33001,"ORTS":33002,"ĠMormons":33003,"Strong":33004,"ĠCoch":33005,"Mexico":33006,"ĠMaduro":33007,"Ġjars":33008,"Ġcane":33009,"Wik":33010,"olla":33011,"ifference":33012,"Ġphysicist":33013,"ĠMaggie":33014,"Ġ285":33015,"Ġdepiction":33016,"ĠMcLaren":33017,"Ju":33018,"Ġslows":33019,"Ġcommissioners":33020,"ĠWillow":33021,"ĠExplos":33022,"hovah":33023,"Ġtechnician":33024,"Ġhomicides":33025,"ĠFlav":33026,"ĠTruman":33027,"Ġ10000":33028,"uctor":33029,"Ġshader":33030,"Newsletter":33031,"457":33032,"Ġrever":33033,"Ġhardened":33034,"Ġwhereabouts":33035,"Ġredevelop":33036,"Ġcarbs":33037,"Ġtravers":33038,"Ġsquirrel":33039,"Ġfollower":33040,"Ġsings":33041,"508":33042,"Ġrabbits":33043,"emonium":33044,"Ġdocumenting":33045,"Ġmisunderstood":33046,")'":33047,"Rick":33048,"ggies":33049,"Ġpremie":33050,"Ġskating":33051,"Ġpassports":33052,"Ġfists":33053,"ageddon":33054,"Haw":33055,"ACP":33056,"080":33057,"ĠThoughts":33058,"ĠCarlson":33059,"Ġpriesthood":33060,"hua":33061,"Ġdungeons":33062,"ĠLoans":33063,"Ġantis":33064,"Ġfamiliarity":33065,"ĠSabb":33066,"opal":33067,"ĠInk":33068,"strike":33069,"Ġcram":33070,"Ġlegalized":33071,"Ġcuisine":33072,"Ġfibre":33073,"Travel":33074,"ĠMonument":33075,"ODY":33076,"ethy":33077,"Ġinterstate":33078,"ĠPUR":33079,"emporary":33080,"ĠArabian":33081,"developed":33082,"Ġsaddle":33083,"Ġgithub":33084,"ĠOffer":33085,"ĠISP":33086,"rolet":33087,"ĠSUPER":33088,"ĠDenis":33089,"Ġmultiplier":33090,"Ġstirred":33091,"Interestingly":33092,"Ġcustomary":33093,"Ġbilled":33094,"hex":33095,"Ġmultiplied":33096,"Ġflipping":33097,"ĠCrosby":33098,"Ġfundamentals":33099,"iae":33100,"ĠPlayed":33101,"ĠAtom":33102,"amazon":33103,"ĠFlam":33104,"eez":33105,"activated":33106,"Ġtablespoon":33107,"Ġliberalism":33108,"ĠPalin":33109,"ĠPatel":33110,"Num":33111,"ĠTAM":33112,"Ġsurn":33113,"ĠReloaded":33114,"Ġcoined":33115,"\"],":33116,"ĠClash":33117,"ĠAgu":33118,"Ġpragmatic":33119,"ĠActivate":33120,"Ġ802":33121,"Ġtrailers":33122,"Ġsilhou":33123,"Ġprobes":33124,"Ġcircus":33125,"ĠBain":33126,"ĠLindsay":33127,"ĠAbbey":33128,"Delivery":33129,"Ġconcession":33130,"Ġgastro":33131,"ĠSprite":33132,"ÄŁ":33133,"andel":33134,"Ġgimm":33135,"Ġautobi":33136,"ĠTurtle":33137,"Ġwonderfully":33138,"ĠHaram":33139,"ĠWorldwide":33140,"ĠHandle":33141,"Ġtheorists":33142,"Ġsleek":33143,"ĠZhu":33144,"ographically":33145,"EGA":33146,"ĠOwners":33147,"aths":33148,"ĠAntarctic":33149,"natal":33150,"=\"\"":33151,"flags":33152,"````":33153,"Ġsul":33154,"Kh":33155,"Ġpotassium":33156,"Ġlineman":33157,"Ġcereal":33158,"ĠSeasons":33159,"Ġ2022":33160,"Ġmathematic":33161,"Ġastronomers":33162,"professional":33163,"Ġfares":33164,"cknowled":33165,"Ġchi":33166,"Ġyoungsters":33167,"Ġmistakenly":33168,"Ġhemisphere":33169,"ĠDivinity":33170,"rone":33171,"Ġ\",":33172,"rings":33173,"Ġattracts":33174,"vana":33175,"å¹":33176,"CAP":33177,"Ġplaylist":33178,"Ġporch":33179,"ãģ£":33180,"Ġincorporates":33181,"Ġsoak":33182,"Ġasserting":33183,"ĠTerrorism":33184,"ĠPablo":33185,"Ja":33186,"cester":33187,"Ġfearing":33188,"ĠPrayer":33189,"Ġescalated":33190,"GW":33191,"Ġrobe":33192,"ĠBrighton":33193,"acists":33194,"ĠSymphony":33195,"ĠDwarf":33196,"ĠParade":33197,"ĠLego":33198,"Ġinexpl":33199,"Ġlords":33200,"leaf":33201,"RAG":33202,"liber":33203,"Ġcigars":33204,"ĠJehovah":33205,"606":33206,"WINDOWS":33207,"ĠLiberia":33208,"ebus":33209,"Heavy":33210,"Ġlubric":33211,"ĠRW":33212,"anguages":33213,"Ġnarrowed":33214,"computer":33215,"ĠEmber":33216,"Ġmurdering":33217,"Ġdownstream":33218,"ĠTuls":33219,"ĠTables":33220,"Topic":33221,"ĠAccuracy":33222,"=/":33223,"lost":33224,"ĠRei":33225,"Ġprogresses":33226,"bear":33227,"Ġestablishments":33228,"Justin":33229,"ĠPeach":33230,"ĠGomez":33231,"å¿":33232,"ĠTriangle":33233,"Ident":33234,"ĠHive":33235,"Resources":33236,"Ġmixes":33237,"ĠAssuming":33238,"Mu":33239,"Ġhypoc":33240,"Ġsane":33241,"ĠWan":33242,"idious":33243,"Success":33244,"Ġio":33245,"Angel":33246,"Ġdangerously":33247,"ĠCreature":33248,"WORK":33249,":[":33250,"ĠKatrina":33251,"Listener":33252,"Miller":33253,"ĠIdlib":33254,"hang":33255,"Ġcircumvent":33256,"href":33257,"Ġcelestial":33258,"ĠWeeks":33259,"ĠPug":33260,"ĠDalton":33261,"Ġsubpoena":33262,"uku":33263,"Ġpersisted":33264,"pei":33265,"olding":33266,"ĠDocuments":33267,"ĠHast":33268,"ĠCENT":33269,"Ġprimer":33270,"Ġsynonymous":33271,"Ġnib":33272,"ombs":33273,"Ġnotation":33274,"ĠDish":33275,"ĠAtmosp":33276,"Ġforbid":33277,"ĠANG":33278,"pattern":33279,"los":33280,"Ġprojectiles":33281,"brown":33282,".\",":33283,"ĠVenom":33284,"Ġfiercely":33285,"ublished":33286,"ĠUran":33287,"ĠNicarag":33288,"410":33289,"ĠCAL":33290,"OTOS":33291,"ĠMiracle":33292,"ĠEnchant":33293,"Ġguarding":33294,"append":33295,"Attach":33296,"Ġleveled":33297,"Ġcondoms":33298,"ihilation":33299,"649":33300,"Ġnightmares":33301,"ĠTHEY":33302,"ĠSTART":33303,"ĠKinn":33304,"Ġroommate":33305,"Ġhygiene":33306,"opping":33307,"Job":33308,"Ġlvl":33309,"ĠVER":33310,"ĠKeeping":33311,"abetic":33312,"Ġformatting":33313,"erala":33314,"Ġrevisions":33315,"Ġresurg":33316,"Tel":33317,"ĠGoodman":33318,"353":33319,"pod":33320,"Ġindisp":33321,"ĠTranslation":33322,"Ġgown":33323,"ĠMund":33324,"Ġcis":33325,"Ġbystand":33326,"collect":33327,"ĠPunjab":33328,"actively":33329,"ĠGamb":33330,"tell":33331,"Ġimporting":33332,"gencies":33333,"Ġlocom":33334,"ĠBrill":33335,"Holy":33336,"ĠBerger":33337,"Ġshowdown":33338,"Ġresponders":33339,"ILY":33340,"Ġtakedown":33341,"leted":33342,"Ġmattered":33343,"Ġpredictive":33344,"Ġoverlay":33345,"GPU":33346,"ĠVick":33347,"Ġconveyed":33348,"Tab":33349,"peer":33350,"Scan":33351,"Ġdefensively":33352,"vae":33353,"Ġapproving":33354,"Ġtiers":33355,"ĠVia":33356,"querade":33357,"ĠSaudis":33358,"Ġdemolished":33359,"ĠProphe":33360,"Ġmono":33361,"Ġhospitality":33362,"HAM":33363,"ĠAriel":33364,"MOD":33365,"ĠTorah":33366,"Ġblah":33367,"ĠBelarus":33368,"erential":33369,"ĠTuc":33370,"Ġbanker":33371,"397":33372,"Ġmosquit":33373,"ĠScientist":33374,"ĠMusical":33375,"Ġhust":33376,"Shift":33377,"Ġtorment":33378,"Ġstandoff":33379,"Educ":33380,"ĠFog":33381,"Ġamplifier":33382,"Shape":33383,"Instance":33384,"ĠCritics":33385,"Ġdaemon":33386,"Houston":33387,"Ġmattress":33388,"ĠIDF":33389,"Ġobscene":33390,"ĠAmer":33391,"hetti":33392,"Ġcompiling":33393,"352":33394,"verett":33395,"ĠReduction":33396,"istration":33397,"ĠBlessed":33398,"ĠBachelor":33399,"316":33400,"Ġprank":33401,"ĠVulcan":33402,"dding":33403,"Ġmourning":33404,"ĠQuint":33405,"ĠBlaster":33406,"testing":33407,"Ġsediment":33408,">>>":33409,"ĠEternity":33410,"ĠWHERE":33411,"ĠMaze":33412,"Ġreacting":33413,"ĠAlv":33414,"omsday":33415,"ĠCRA":33416,"Ġtranslator":33417,"Ġbogus":33418,"atu":33419,"Website":33420,"olls":33421,"Ġbaptism":33422,"Ġsibling":33423,"ĠAutumn":33424,"vez":33425,"ãģ®é":33426,"guards":33427,"Georg":33428,"assadors":33429,"ĠFreud":33430,"Ġcontinents":33431,"ĠRegistry":33432,"Bernie":33433,"ĸļ士":33434,"Ġtolerant":33435,"ĠUW":33436,"Ġhorribly":33437,"995":33438,"ĠMIDI":33439,"Ġimpatient":33440,"ocado":33441,"eri":33442,"ĠWorst":33443,"ĠNorris":33444,"ĠTalking":33445,"Ġdefends":33446,"ensable":33447,"Ġ2021":33448,"Ġanatomy":33449,"Lew":33450,"Ġdrawer":33451,"ĠCanberra":33452,"Ġpatriotic":33453,"é¾įåĸļ士":33454,"ĠAvg":33455,"ARM":33456,"Ġundisclosed":33457,"Ġfarewell":33458,"459":33459,"bable":33460,"ĠAllison":33461,"OLOG":33462,"Ġconco":33463,"tight":33464,"ĠACPI":33465,"ĠMines":33466,"lich":33467,"ĠâĶľ":33468,"represented":33469,"200000":33470,"Ġenthusiast":33471,"OTS":33472,"bil":33473,"ĠIngredients":33474,"Ġinventor":33475,"ĠMySQL":33476,"³³³":33477,"ĠABOUT":33478,"within":33479,"Ġmk":33480,"Bul":33481,"ĠFake":33482,"Ġdraconian":33483,"Wa":33484,"helm":33485,"ĠTerran":33486,"erville":33487,"Ġcommonplace":33488,"SIZE":33489,"Ġ\"<":33490,"replace":33491,"ographs":33492,"ĠSELECT":33493,"incible":33494,"ĠMostly":33495,"ĠSheffield":33496,"ĠIDE":33497,"uggle":33498,"Ġcitations":33499,"hurst":33500,"ĠUnix":33501,"Ġunleash":33502,"ĠPiper":33503,"ĠNano":33504,"Ġsuccumb":33505,"Ġreluctance":33506,"Ġ2500":33507,"ĠMerchant":33508,"Ġwiret":33509,"Ġcombos":33510,"ĠBirthday":33511,"Ġcharcoal":33512,"ĠUPS":33513,"ĠFairfax":33514,"Ġdriveway":33515,"ĠTek":33516,"ĠPitch":33517,"overe":33518,"Ġtechnicians":33519,"ĠActual":33520,"flation":33521,"ĠFiscal":33522,"ĠEmpty":33523,"anamo":33524,"Ġmagnesium":33525,"Ġslut":33526,"Ġgrowers":33527,"Investigators":33528,"():":33529,"ĠSatellite":33530,"ĠKeynes":33531,"missive":33532,"lane":33533,"Ġborough":33534,"344":33535,"ĠTEAM":33536,"ĠBethesda":33537,"CV":33538,"hower":33539,"ĠRAD":33540,"Ġchant":33541,"ĠRiy":33542,"Ġcompositions":33543,"Ġmildly":33544,"Ġmeddling":33545,"Ġagility":33546,"aneers":33547,"501":33548,"Ġsynth":33549,"linger":33550,"291":33551,"Ġexclaimed":33552,"Party":33553,"Ġcontamin":33554,"ĠManor":33555,"ĠRespond":33556,"Ġpraising":33557,"Ġmanners":33558,"fleet":33559,"Summer":33560,"ĠLynd":33561,"ĠDefinitely":33562,"grim":33563,"Ġbowling":33564,"stri":33565,"çĽ":33566,"ynt":33567,"Ġmandates":33568,"DIV":33569,"Ġreconcile":33570,"views":33571,"ĠDamon":33572,"vette":33573,"Flo":33574,"ĠGreatest":33575,"ilon":33576,"icia":33577,"Ġportrayal":33578,"Ġcushion":33579,"504":33580,"1979":33581,"ossal":33582,"Applic":33583,"scription":33584,"Ġmitigation":33585,"ATS":33586,"pac":33587,"Ġerased":33588,"Ġdeficiencies":33589,"ĠHollande":33590,"ĠXu":33591,"Ġbred":33592,"Ġpregnancies":33593,"femin":33594,"Ġemph":33595,"Ġplanners":33596,"Ġoutper":33597,"uttering":33598,"Ġperpetrator":33599,"Ġmotto":33600,"ĠEllison":33601,"ĠNEVER":33602,"Ġadmittedly":33603,"ARI":33604,"ĠAzerbaijan":33605,"Ġmillisec":33606,"Ġcombustion":33607,"ĠBottle":33608,"ĠLund":33609,"ĠPs":33610,"ĠDress":33611,"Ġfabricated":33612,"Ġbattered":33613,"Ġsidel":33614,"ĠNotting":33615,"Foreign":33616,"ĠJerome":33617,"020":33618,"ĠArbit":33619,"Ġknots":33620,"ĠRIGHT":33621,"Moving":33622,"ãģĻ":33623,"Ġsurgeries":33624,"Ġcourthouse":33625,"Ġmastered":33626,"Ġhovering":33627,"ĠBran":33628,"ĠAlison":33629,"Ġsafest":33630,"military":33631,"Ġbullied":33632,"Ġbarrage":33633,"Reader":33634,"ESE":33635,"ĠGeographic":33636,"Tools":33637,"314":33638,"ĠGeek":33639,"roth":33640,"glers":33641,"ĠFIN":33642,"Ïģ":33643,"ĠAston":33644,"altern":33645,"488":33646,"Ġveterin":33647,"Gamer":33648,"Ġintel":33649,"renches":33650,"Shield":33651,"Ġamnesty":33652,"ĠBhar":33653,"Ġpiled":33654,"Ġhonorable":33655,"ĠInstitutes":33656,"Ġsoaked":33657,"Ġcoma":33658,"ĠEFF":33659,"341":33660,"bytes":33661,"ĠGmail":33662,"lein":33663,"ĠCanadiens":33664,"material":33665,"Il":33666,"Ġinstructors":33667,"ĠKY":33668,"Ġconceive":33669,"ubb":33670,"ĠPossible":33671,"Ġeasing":33672,"ĠChristina":33673,"Ġcaric":33674,"ĠHDR":33675,"ROM":33676,"Ġshovel":33677,"delete":33678,"Ġpuff":33679,"ĠChanging":33680,"Ġseamlessly":33681,"Attribute":33682,"Ġacquisitions":33683,"akery":33684,"ĠEF":33685,"Ġautistic":33686,"ĠTakes":33687,"ĠPowder":33688,"ĠStir":33689,"510":33690,"ĠBubble":33691,"settings":33692,"ĠFowler":33693,"Ġmustard":33694,"Ġmoreover":33695,"Ġcopyrighted":33696,"ĠLEDs":33697,"1500":33698,"æī":33699,"ĠHIS":33700,"enf":33701,"Ġcustod":33702,"ĠHuck":33703,"Gi":33704,"Ġimg":33705,"Answer":33706,"Ct":33707,"jay":33708,"ĠInfrastructure":33709,"Ġfederally":33710,"Loc":33711,"Ġmicrobes":33712,"Ġoverrun":33713,"dds":33714,"otent":33715,"adiator":33716,">>>>>>>>":33717,"Ġtornado":33718,"Ġadjud":33719,"Ġintrigued":33720,"Ġsi":33721,"ĠRevelation":33722,"progress":33723,"Ġburglary":33724,"ĠSaiyan":33725,"ĠKathy":33726,"Ġserpent":33727,"ĠAndreas":33728,"Ġcompel":33729,"essler":33730,"ĠPlastic":33731,"ĠAdvent":33732,"ĠPositive":33733,"ĠQt":33734,"ĠHindus":33735,"registered":33736,"ularity":33737,"Ġrighteousness":33738,"Ġdemonic":33739,"uitive":33740,"ĠBDS":33741,"ĠGregg":33742,"cia":33743,"ĠCrusade":33744,"ĠSinai":33745,"WARE":33746,"+(":33747,"Ġmell":33748,"Ġderail":33749,"yards":33750,"Ast":33751,"Ġnoticeably":33752,"ĠOber":33753,"Ram":33754,"Ġunnoticed":33755,"Ġseq":33756,"avage":33757,"Ts":33758,"Ġ640":33759,"Ġconcede":33760,"Ġ])":33761,"Fill":33762,"Ġcaptivity":33763,"ĠImprovement":33764,"ĠCrusader":33765,"araoh":33766,"MAP":33767,"æĹ":33768,"Ġstride":33769,"always":33770,"Fly":33771,"Nit":33772,"Ġalgae":33773,"ĠCooking":33774,"ĠDoors":33775,"Malley":33776,"Ġpolicemen":33777,"ãģį":33778,"Ġastronaut":33779,"accessible":33780,"495":33781,"ĠRAW":33782,"cliffe":33783,"udicrous":33784,"Ġdepended":33785,"alach":33786,"Ġventures":33787,"rake":33788,"Ġtits":33789,"ĠHou":33790,"Ġcondom":33791,"ormonal":33792,"Ġindent":33793,"Ġuploading":33794,"Footnote":33795,"Important":33796,"Ġ271":33797,"Ġmindful":33798,"Ġcontends":33799,"Cra":33800,"Ġcalibr":33801,"ĠOECD":33802,"plugin":33803,"Fat":33804,"ĠISS":33805,"ĠDynamics":33806,"ansen":33807,"686":33808,"'),":33809,"Ġsprite":33810,"Ġhandheld":33811,"ĠHipp":33812,"=~=~":33813,"Trust":33814,"Ġsemantics":33815,"ĠBundes":33816,"ĠReno":33817,"ĠLiterature":33818,"sense":33819,"Gary":33820,"ĠAeg":33821,"ĠTrin":33822,"EEK":33823,"Ġcleric":33824,"ĠSSH":33825,"Ġchrist":33826,"Ġinvading":33827,"ibu":33828,"Ġenum":33829,"aura":33830,"Ġallege":33831,"ĠIncredible":33832,"BBC":33833,"Ġthru":33834,"Ġsailed":33835,"Ġemulate":33836,"Ġinsecurity":33837,"Ġcrou":33838,"Ġaccommodations":33839,"Ġincompetent":33840,"Ġslips":33841,"ĠEarthqu":33842,"sama":33843,"ILLE":33844,"ĠiPhones":33845,"asaki":33846,"Ġbye":33847,"Ġard":33848,"Ġextras":33849,"Ġslaughtered":33850,"Ġcrowdfunding":33851,"resso":33852,"Ġfilib":33853,"ĠERROR":33854,"ĠTLS":33855,"egg":33856,"ĠItal":33857,"Ġenlist":33858,"ĠCatalonia":33859,"ĠScots":33860,"Ġsergeant":33861,"Ġdissolve":33862,"NH":33863,"Ġstandings":33864,"rique":33865,"IQ":33866,"Ġbeneficiary":33867,"Ġaquarium":33868,"YouTube":33869,"ĠPowerShell":33870,"Ġbrightest":33871,"ĠWarrant":33872,"Sold":33873,"Writing":33874,"Ġbeginnings":33875,"ĠReserved":33876,"ĠLatinos":33877,"heading":33878,"Ġ440":33879,"Ġrooftop":33880,"ATING":33881,"Ġ390":33882,"VPN":33883,"Gs":33884,"kernel":33885,"turned":33886,"Ġpreferable":33887,"Ġturnovers":33888,"ĠHels":33889,"Sa":33890,"ĠShinji":33891,"veh":33892,"ĠMODULE":33893,"Viol":33894,"Ġexiting":33895,"Ġjab":33896,"ĠVanilla":33897,"Ġacron":33898,"ĠGap":33899,"bern":33900,"Ak":33901,"ĠMcGu":33902,"Ġendlessly":33903,"ĠFarage":33904,"ĠNoel":33905,"Va":33906,"MK":33907,"Ġbrute":33908,"ĠKru":33909,"ĠESV":33910,"ĠOlivia":33911,"âĢł":33912,"ĠKaf":33913,"Ġtrusting":33914,"Ġhots":33915,"324":33916,"Ġmalaria":33917,"Ġjson":33918,"Ġpounding":33919,"ortment":33920,"Country":33921,"Ġpostponed":33922,"Ġunequiv":33923,"?),":33924,"ĠRooney":33925,"udding":33926,"ĠLeap":33927,"urrence":33928,"shapeshifter":33929,"ĠHAS":33930,"osate":33931,"Ġcavern":33932,"Ġconservatism":33933,"ĠBAD":33934,"Ġmileage":33935,"Ġarresting":33936,"Vaults":33937,"Ġmixer":33938,"Democratic":33939,"ĠBenson":33940,"Ġauthored":33941,"8000":33942,"Ġproactive":33943,"ĠSpiritual":33944,"tre":33945,"Ġincarcerated":33946,"ĠSort":33947,"Ġpeaked":33948,"Ġwielding":33949,"reciation":33950,"×Ļ×":33951,"Patch":33952,"ĠEmmy":33953,"Ġexqu":33954,"tto":33955,"ĠRatio":33956,"ĠPicks":33957,"ĠGry":33958,"phant":33959,"Ġfret":33960,"Ġethn":33961,"Ġarchived":33962,"%-":33963,"cases":33964,"ĠBlaze":33965,"Ġimb":33966,"cv":33967,"yss":33968,"imony":33969,"Ġcountdown":33970,"Ġawakening":33971,"ĠTunisia":33972,"ĠRefer":33973,"ĠMJ":33974,"Ġunnatural":33975,"ĠCarnegie":33976,"izen":33977,"ĠNuggets":33978,"hess":33979,"Ġevils":33980,"647":33981,"Ġintroductory":33982,"loving":33983,"ĠMcMahon":33984,"Ġambiguity":33985,"Label":33986,"ĠAlmighty":33987,"Ġcoloring":33988,"ĠClaus":33989,"setting":33990,"NULL":33991,"ĠFavorite":33992,"ĠSIG":33993,">(":33994,"ĠShiva":33995,"ĠMayer":33996,"Ġstormed":33997,"ĠCoverage":33998,"weapons":33999,"igham":34000,"Ġunanswered":34001,"Ġleve":34002,"Ġcoy":34003,"cas":34004,"bags":34005,"asured":34006,"Seattle":34007,"ĠSantorum":34008,"serious":34009,"Ġcourageous":34010,"ĠSoup":34011,"Ġconfiscated":34012,"Ġ///":34013,"Ġunconventional":34014,"Ġmoms":34015,"ĠRohingya":34016,"ĠOrchestra":34017,"ĠPotion":34018,"Ġdiscredit":34019,"ĠFIL":34020,"fixed":34021,"ĠDeer":34022,"doi":34023,"ĠDimension":34024,"Ġbureaucrats":34025,"eteen":34026,"ĠactionGroup":34027,"ohm":34028,"Ġbumps":34029,"ĠUtility":34030,"Ġsubmarines":34031,"renheit":34032,"research":34033,"ĠShapiro":34034,"Ġsketches":34035,"Ġdeceptive":34036,"ĠVil":34037,"esame":34038,"ĠEssentially":34039,"Ġrampage":34040,"isky":34041,"Ġmuttered":34042,"thritis":34043,"Ġ236":34044,"fet":34045,"bars":34046,"Ġpupil":34047,"ĠThou":34048,"oS":34049,"song":34050,"Ġfractured":34051,"Ġrevert":34052,"picture":34053,"Ġcriterion":34054,"usher":34055,"Ġrepercussions":34056,"ĠVintage":34057,"ĠSuperintendent":34058,"Officers":34059,"Ġflagged":34060,"Ġblames":34061,"Ġinverse":34062,"ographers":34063,"Ġmakeshift":34064,"Ġdevoid":34065,"Ġfossils":34066,"ĠAristotle":34067,"ĠFunds":34068,"Ġdepleted":34069,"ĠFlu":34070,"ĠYuan":34071,"Ġwoes":34072,"Ġlipid":34073,"Ġsitu":34074,"requisites":34075,"Ġfurnish":34076,"ĠSamar":34077,"Ġshameful":34078,"Ġadversely":34079,"Ġadept":34080,"Ġremorse":34081,"Ġmurderous":34082,"uckles":34083,"ĠESL":34084,"Ġ314":34085,"sent":34086,"Ġredef":34087,"ĠCache":34088,"ĠPurs":34089,"igans":34090,"Ġ460":34091,"Ġprescriptions":34092,"Ġfres":34093,"Fuck":34094,"ocrates":34095,"Twenty":34096,"ĠWeird":34097,"ĠToggle":34098,"ĠCalled":34099,"itizens":34100,"Ġpoultry":34101,"Ġharvesting":34102,"ãĤ¦ãĤ¹":34103,"Bottom":34104,"Ġcautioned":34105,"tn":34106,"396":34107,"ĠNikki":34108,"Ġevaluations":34109,"Ġharassing":34110,"Ġbindings":34111,"ĠMonetary":34112,"Ġhitters":34113,"Ġadversary":34114,"unts":34115,"Ġsetback":34116,"Ġencrypt":34117,"ĠCait":34118,"Ġlows":34119,"enges":34120,"ĠNorn":34121,"Ġbulbs":34122,"Ġbottled":34123,"ĠVoyager":34124,"317":34125,"Ġspheres":34126,"politics":34127,"Ġsubtract":34128,"Ġsensations":34129,"Ġappalling":34130,"Ġ316":34131,"Ġenvironmentally":34132,"ĠSTEM":34133,"Ġpublishes":34134,"560":34135,"Ġdiligence":34136,"484":34137,"Ġadvises":34138,"Ġpetrol":34139,"Ġimagining":34140,"Ġpatrols":34141,"ĠInteger":34142,"ĠAshes":34143,"actus":34144,"ĠRadiant":34145,"ĠLT":34146,"itability":34147,"htaking":34148,"Setting":34149,"Ġnuanced":34150,"ĠReef":34151,"ĠDevelopers":34152,"Ni":34153,"pieces":34154,"990":34155,"License":34156,"Ġlowers":34157,"ĠOttoman":34158,"327":34159,"ooo":34160,"Ġquitting":34161,"markets":34162,"Behind":34163,"Ġbasin":34164,"Ġdocs":34165,"anie":34166,"flash":34167,"ctl":34168,"Ġcivilized":34169,"ĠFukushima":34170,"\"],\"":34171,"ĠKS":34172,"ĠHonestly":34173,"arat":34174,"Ġconstructs":34175,"ĠLans":34176,"ĠDire":34177,"ĠLIKE":34178,"ĠTrouble":34179,"Ġwithholding":34180,"ĠOblivion":34181,"Ġsanity":34182,"anya":34183,"Const":34184,"Ġgrocer":34185,"ĠCelsius":34186,"Ġrecounted":34187,"ĠWife":34188,"Border":34189,"atered":34190,"happy":34191,"Ġspoiler":34192,"Ġlogically":34193,"Hall":34194,"Ġsucceeding":34195,"Ġpolymorph":34196,"Ġaxes":34197,"ĠShotgun":34198,"ĠSlim":34199,"ĠPrinciples":34200,"ĠLeth":34201,"arta":34202,"Ġscor":34203,"Screenshot":34204,"Ġrelaxation":34205,"#$#$":34206,"Ġdeterrent":34207,"iddy":34208,"Ġpowerless":34209,"Ġlesbians":34210,"Ġchords":34211,"ĠEdited":34212,"selected":34213,"Ġseparatists":34214,"0002":34215,"Ġairspace":34216,"Ġturnaround":34217,"Ġcunning":34218,"PATH":34219,"Poly":34220,"Ġbombed":34221,"Ġtion":34222,"xs":34223,"Ġwithhold":34224,"Ġwaged":34225,"ĠLiberties":34226,"Flag":34227,"Ġcomforting":34228,"454":34229,"ĠIris":34230,"arers":34231,"Ġrag":34232,"Ġrelocated":34233,"ĠGuarant":34234,"Ġstrategically":34235,"Ġgamma":34236,"uberty":34237,"ĠLockheed":34238,"gres":34239,"Ġgrilled":34240,"ĠLowe":34241,"stats":34242,"ĠRocks":34243,"Ġsensing":34244,"Ġrenting":34245,"ĠGeological":34246,"اØ":34247,"otrop":34248,"Ġsew":34249,"Ġimproperly":34250,"486":34251,"Ġâĸł":34252,"Ġstarving":34253,"ĠBj":34254,"Discussion":34255,"328":34256,"ĠCombo":34257,"ĠFixes":34258,"NAT":34259,"Ġstriving":34260,"thora":34261,"Ġharvested":34262,"ĠPing":34263,"Ġplayful":34264,"Ġavenues":34265,"Ġoccupational":34266,"Ġwakes":34267,"ĠCourier":34268,"Ġdrummer":34269,"ĠBrowser":34270,"ĠHouth":34271,"itu":34272,"Ġapparel":34273,"paste":34274,"Ġhunted":34275,"ĠSecondly":34276,"lain":34277,"XY":34278,"ĠPIN":34279,"icons":34280,"Ġcocktails":34281,"Ġsizable":34282,"Ġhurdles":34283,"estinal":34284,"ĠRecreation":34285,"Ġeco":34286,"648":34287,"ĠDied":34288,"mint":34289,"Ġfingerprints":34290,"Ġdispose":34291,"ĠBosnia":34292,"tsy":34293,"2200":34294,"Ġinspected":34295,"ĠFou":34296,"Ġfuss":34297,"Ġambush":34298,"ĠRak":34299,"Ġmanifested":34300,"Prosecut":34301,"Ġsuffice":34302,"rences":34303,"Ġcompensated":34304,"ĠCyrus":34305,"Ġgenus":34306,"ĠWolverine":34307,"ĠTrends":34308,"Ġhikes":34309,"ĠSeen":34310,"Ġenrol":34311,"Cold":34312,"Ġpolitely":34313,"ĠSlav":34314,"ĠRupert":34315,"Ġeyewitness":34316,"ĠAlto":34317,"Ġuncomp":34318,"Ġposterior":34319,"Must":34320,"ĠHerz":34321,"Ġprogressively":34322,"Ġ234":34323,"Ġindifference":34324,"ĠCunningham":34325,"Ġacademia":34326,"Ġsewer":34327,"Ġastounding":34328,"ĠAES":34329,"rather":34330,"Ġeldest":34331,"Ġclimbs":34332,"ĠAdds":34333,"Ġoutcry":34334,"Ġcontag":34335,"ĠHouses":34336,"Ġpept":34337,"ĠMelania":34338,"interested":34339,"ĠUCH":34340,"ĠRoots":34341,"ĠHubbard":34342,"ĠTBD":34343,"ĠRomanian":34344,"filename":34345,"Stone":34346,"ĠImpl":34347,"Ġchromosome":34348,"Cle":34349,"dx":34350,"Ġscrambled":34351,"ĠPt":34352,"Ġ242":34353,"OPLE":34354,"Ġtremendously":34355,"Street":34356,"Ġcraving":34357,"Ġbundled":34358,"ĠRG":34359,"pipe":34360,"Ġinjuring":34361,"Ġarcane":34362,"Particip":34363,"ĠHeroic":34364,"sty":34365,"Ġtopping":34366,"ĠTempest":34367,"rentices":34368,"bh":34369,"Ġparanoia":34370,"ĠUnicode":34371,"Ġegregious":34372,"Ġ\\'":34373,"ĠOswald":34374,"Ġgravel":34375,"ĠSimpsons":34376,"Ġbland":34377,"ĠGuantanamo":34378,"Writer":34379,"liners":34380,"ĠDice":34381,"JC":34382,"Ġparity":34383,"Ġsided":34384,"Ġ237":34385,"ĠPyrrha":34386,"atters":34387,"dk":34388,"Fine":34389,"compan":34390,"Ġformulated":34391,"ĠIdol":34392,"ilers":34393,"hemoth":34394,"ĠFav":34395,"Ġintrusion":34396,"Ġcarrots":34397,"ĠLayer":34398,"ĠHacker":34399,"Ġ----------------":34400,"Ġmoderation":34401,"éģ":34402,"ococ":34403,"Ġcharacterize":34404,"ĠTeresa":34405,"Ġsocioeconomic":34406,"Ġperk":34407,"ĠParticipation":34408,"training":34409,"ĠPaulo":34410,"phys":34411,"Ġtrustworthy":34412,"Ġembodied":34413,"ĠMerch":34414,"currency":34415,"ĠPriority":34416,"Ġteasing":34417,"Ġabsorbing":34418,"Ġunfinished":34419,"ĠComparison":34420,"Ġdisple":34421,"writers":34422,"Ġprofessions":34423,"ĠPenguin":34424,"Ġangrily":34425,"ĠLINK":34426,"688":34427,"ĠCorrespond":34428,"Ġprevailed":34429,"Ġcartel":34430,"lp":34431,"asms":34432,"ĠRedemption":34433,"ĠIslamists":34434,"effects":34435,"dose":34436,"ĠLatter":34437,"ĠHalifax":34438,"Ġvas":34439,"ĠTopics":34440,"ĠNamed":34441,"advertising":34442,"zza":34443,"ICES":34444,"Ġretarded":34445,"achable":34446,"ĠPuppet":34447,"ĠItemLevel":34448,"Ġretract":34449,"Ġidentifiable":34450,"Aaron":34451,"ĠBuster":34452,"sol":34453,"helle":34454,"assemb":34455,"Hope":34456,"ranged":34457,"Ba":34458,"ĠPurch":34459,"éĢ":34460,"ĠSiri":34461,"Ġarrivals":34462,"Ġ1912":34463,"Ġshortened":34464,"Ġ312":34465,"Ġdiscrepancy":34466,"ĠTemperature":34467,"ĠWalton":34468,"Ġkinderg":34469,"polit":34470,"Ġremix":34471,"Ġconnectors":34472,"ãĥĺãĥ©":34473,"ĠKazakhstan":34474,"dominated":34475,"Ġsugars":34476,"imble":34477,"ĠPanic":34478,"ĠDemand":34479,"ĠColony":34480,"onen":34481,"ĠMER":34482,"775":34483,"uria":34484,"azaar":34485,"ĠDegree":34486,"Pri":34487,"Ġsunshine":34488,"Ġ251":34489,"Ġpsychedelic":34490,"Ġdigitally":34491,"ĠBraun":34492,"Ġshimmer":34493,"Ġshave":34494,"ĠTelesc":34495,"ĠAstral":34496,"ĠVenezuelan":34497,"ĠOG":34498,"Ġcrawling":34499,"Integ":34500,"ĠFeather":34501,"Ġunfolding":34502,"Ġappropriation":34503,"Ġè£ıè":34504,"ĠMobility":34505,"ĠNey":34506,"-.":34507,"bilt":34508,"LIN":34509,"ĠTube":34510,"ĠConversely":34511,"Ġkeyboards":34512,"ĠCao":34513,"Ġoverth":34514,"Ġlaure":34515,">>\\":34516,"ĠViper":34517,"acha":34518,"Offset":34519,"ĠRaleigh":34520,"ĠJae":34521,"Jordan":34522,"jp":34523,"Ġtotalitarian":34524,"Connector":34525,"Ġobserves":34526,"ĠSpartan":34527,"ĠImmediately":34528,"ĠScal":34529,"Cool":34530,"Ġtaps":34531,"Ġroar":34532,"Past":34533,"Ġchars":34534,"ĠBender":34535,"ĠSheldon":34536,"Ġpainter":34537,"Ġbeacon":34538,"ĠCreatures":34539,"Ġdownturn":34540,"Ġhinder":34541,"ĠAndromeda":34542,"ÃĽ":34543,"ccoli":34544,"ĠFitness":34545,"etrical":34546,"Ġutilizes":34547,"Ġsenate":34548,"Ġensemble":34549,"Ġcheers":34550,"TW":34551,"Ġaffluent":34552,"kil":34553,"rylic":34554,"ordering":34555,"Computer":34556,"Ġgruesome":34557,"ostics":34558,"ĠUbisoft":34559,"ĠKelley":34560,"Ġwrench":34561,"Ġbourgeoisie":34562,"IBLE":34563,"ĠPreston":34564,"worn":34565,"arist":34566,"reating":34567,"Ġstained":34568,"arine":34569,"Ġslime":34570,"ENN":34571,"Ġchests":34572,"Ġgroundwater":34573,"annot":34574,"ĠTray":34575,"ĠLocke":34576,"ĠCTR":34577,"Ġdudes":34578,"ĠExternal":34579,"ĠDecoder":34580,"Ġparamed":34581,"ĠMedline":34582,"809":34583,"ĠDinner":34584,"rupal":34585,"gz":34586,"ĠGum":34587,"ĠDemo":34588,"jee":34589,"Ġdh":34590,"berman":34591,"archs":34592,"Ġenqu":34593,"ĠEpstein":34594,"Ġdevastation":34595,"Ġfriendships":34596,"ĠArd":34597,"Ġ231":34598,"ĠRubin":34599,"ĠDistance":34600,"Ġspurred":34601,"Ġdossier":34602,"Ġoverlooking":34603,"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\":34604,"Forest":34605,"ĠComes":34606,"\\\",":34607,"ĠIranians":34608,"Ġfixtures":34609,"Laughs":34610,"Ġcurry":34611,"ĠKingston":34612,"Ġsquash":34613,"Ġcatalogue":34614,"Ġabnormalities":34615,"Ġdigestive":34616,".........":34617,"Ġsubordinate":34618,"ogly":34619,"Ġ249":34620,"Middle":34621,"Ġmassac":34622,"Ġburgers":34623,"Ġdownstairs":34624,"Ġ1931":34625,"394":34626,"ĠVG":34627,"Ġlasers":34628,"ĠSikh":34629,"ĠAlexa":34630,"derived":34631,"Ġcyclist":34632,"ãģ®éŃĶ":34633,"oneliness":34634,"!!!!!!!!":34635,"Ġbuffs":34636,"legate":34637,"Ġraping":34638,"Ġrecommending":34639,"rored":34640,"Ġmulticultural":34641,"unique":34642,"Ġbusinessmen":34643,"Ġuneasy":34644,"ĠMAP":34645,"Ġdispersed":34646,"cipline":34647,"Jess":34648,"ĠKerala":34649,"å§":34650,"Ġabstraction":34651,"Surv":34652,"Uh":34653,"Ġprinters":34654,"ija":34655,"owder":34656,"Ġanalogous":34657,"ĠASP":34658,"afer":34659,"Ġunfolded":34660,"Ġleveling":34661,"Ġbreached":34662,"ĠHearing":34663,"Ġnat":34664,"Ġtranslating":34665,"critical":34666,"Ġantagonist":34667,"ĠYesterday":34668,"Ġfuzzy":34669,"wash":34670,"mere":34671,"Ġbewild":34672,"ĠMae":34673,"Virgin":34674,"phrase":34675,"Ġsignaled":34676,"ĠHIGH":34677,"Ġprotester":34678,"Ġgarner":34679,"unknown":34680,"Ġkay":34681,"Ġabducted":34682,"Ġstalking":34683,"amn":34684,"Ġdeserving":34685,"ĠRiv":34686,"ĠJorge":34687,"Ġscratching":34688,"ĠSaving":34689,"iping":34690,"Ġtease":34691,"Ġmissionary":34692,"ĠMorrow":34693,"TIME":34694,"Present":34695,"Ġchemotherapy":34696,"terness":34697,"ĠHomes":34698,"ĠPurdue":34699,"Ġstaunch":34700,"ĠWhitney":34701,"ĠTHERE":34702,"μ":34703,"iatus":34704,"ĠErnest":34705,"ĠDeploy":34706,"Ġcoveted":34707,"FML":34708,"ĠDialogue":34709,"Ġexited":34710,"fruit":34711,"Ġnerd":34712,"\":\"\",\"":34713,"Ġvivo":34714,"ruly":34715,"460":34716,"ĠAmen":34717,"rehensible":34718,"Ġâĺ":34719,"DIR":34720,"Ġadherence":34721,"Ġchew":34722,"ĠCoke":34723,"ĠSergei":34724,"digital":34725,"ĠNeck":34726,"gently":34727,"enthal":34728,"/)":34729,"Ġweary":34730,"Ġguise":34731,"ĠConcord":34732,"ĠOnion":34733,"atcher":34734,"Ġbinge":34735,"ĠDirective":34736,"Ġmanned":34737,"ansk":34738,"Ġillusions":34739,"Ġbillionaires":34740,"383":34741,"olyn":34742,"odynamic":34743,"ĠWheat":34744,"ĠAlic":34745,"Ġcoloured":34746,"ĠNAFTA":34747,"abo":34748,"Ġmacros":34749,"independent":34750,"sweet":34751,"Ġspac":34752,"ĠKabul":34753,"ĠÄ":34754,"eme":34755,"Ġdictated":34756,"Ġshouts":34757,"={":34758,"Ġripping":34759,"ĠShay":34760,"ĠCricket":34761,"directed":34762,"Ġanalysed":34763,"ĠWARRANT":34764,"agons":34765,"ĠBlazers":34766,"Ġcheered":34767,"Ġarithmetic":34768,"ĠTanz":34769,"373":34770,"ĠFlags":34771,"Ġ295":34772,"Ġwitches":34773,"ĠIncluded":34774,"ĠGained":34775,"ĠBlades":34776,"Gam":34777,"ĠSamantha":34778,"ĠAtlantis":34779,"ĠPratt":34780,"Ġspoiled":34781,"ĠIB":34782,"ĠRamirez":34783,"Probably":34784,"rero":34785,"ĠNg":34786,"ĠWarlock":34787,"tp":34788,"Ġoverhe":34789,"Ġadministrations":34790,"Ġtint":34791,"Ġregiment":34792,"Ġpistols":34793,"Ġblankets":34794,"Ġepist":34795,"Ġbowls":34796,"Ġhydraulic":34797,"Ġdean":34798,"Ġjung":34799,"Ġascend":34800,"705":34801,"ĠSantiago":34802,"î":34803,"Ġunavoid":34804,"ĠShaman":34805,"reb":34806,"Ġstemming":34807,"998":34808,"ĠMG":34809,"sticks":34810,"esthesia":34811,"ERO":34812,"Ġmorbid":34813,"ĠGrill":34814,"ĠPoe":34815,"anyl":34816,"Ġdeleting":34817,"ĠSurveillance":34818,"Ġdirectives":34819,"Ġiterations":34820,"ĠRox":34821,"ĠMilky":34822,"Father":34823,"Ġpatented":34824,"447":34825,"Ġprecursor":34826,"Ġmaiden":34827,"ĠPhen":34828,"ĠVegan":34829,"ĠPatent":34830,"Kelly":34831,"Redditor":34832,"Ġnods":34833,"Ġventilation":34834,"ĠSchwarz":34835,"Ġwizards":34836,"Ġominous":34837,"ĠHeads":34838,"ĠBG":34839,"Ġlumber":34840,"ĠSpiel":34841,"ĠisEnabled":34842,"Ġancestral":34843,"ĠShips":34844,"Ġwrestler":34845,"phi":34846,"Ġyuan":34847,"ĠRebellion":34848,"Ġiceberg":34849,"Ġmagically":34850,"Ġdiversion":34851,"arro":34852,"ythm":34853,"ĠRiders":34854,"ĠRobbie":34855,"ĠKara":34856,"ĠMaintenance":34857,"ĠHerb":34858,"Ġharms":34859,"packed":34860,"ĠFeinstein":34861,"Ġmarrying":34862,"Ġblending":34863,"ĠRates":34864,"Ġ1880":34865,"Ġwrink":34866,"ĠUnch":34867,"ĠTorch":34868,"described":34869,"Ġhumanoid":34870,"ilitating":34871,"ĠConv":34872,"ĠFeld":34873,"IGHTS":34874,"Ġwhistleblower":34875,"ortmund":34876,"etsy":34877,"arrett":34878,"ĠMono":34879,"ĠIke":34880,"ĠCNBC":34881,"ĠWAY":34882,"ĠMDMA":34883,"ĠIndividuals":34884,"Ġsupplemental":34885,"Ġpowerhouse":34886,"ĠStru":34887,"Focus":34888,"aphael":34889,"ĠColleg":34890,"atti":34891,"ZA":34892,"Ġperenn":34893,"ĠSignature":34894,"ĠRodney":34895,"Ġcubes":34896,"iddled":34897,"ĠDante":34898,"ĠINV":34899,"ilingual":34900,"ĠCth":34901,"Ġsofa":34902,"Ġintimidate":34903,"ĠRoe":34904,"ĠDiplom":34905,"ĠCountries":34906,"ayson":34907,"Ġextradition":34908,"Ġdisabling":34909,"ĠCardiff":34910,"Ġmemorandum":34911,"ĠTrace":34912,"Ġ???":34913,"sector":34914,"ĠRouhani":34915,"ĠYates":34916,"ĠFreeze":34917,"Ġbladder":34918,"Motor":34919,"ĠPromise":34920,"antasy":34921,"Ġforeseeable":34922,"ĠCologne":34923,"container":34924,"ĠTrees":34925,"ĠGors":34926,"ĠSinclair":34927,"Ġbarring":34928,"keye":34929,"Ġslashed":34930,"ĠStatistical":34931,"éĩ":34932,"Ġâĸº":34933,"Allows":34934,"Ġhumility":34935,"Ġdrilled":34936,"ĠFurn":34937,"443":34938,"Ġsewage":34939,"Ġhomepage":34940,"Ġcourtyard":34941,"Ġvile":34942,"Ġsubsidiaries":34943,"ajo":34944,"directory":34945,"Ġammon":34946,"Vers":34947,"charges":34948,"Ġ}}":34949,"ĠChains":34950,"Ġ246":34951,"nob":34952,"Ġpercept":34953,"Ġgrit":34954,"Ġfishermen":34955,"ĠIraqis":34956,"ĠDISTR":34957,"ĠFULL":34958,"ĠEvaluation":34959,"graph":34960,"atial":34961,"Ġcooperating":34962,"Ġmelan":34963,"Ġenlightened":34964,"Ġali":34965,"tailed":34966,"Ġsalute":34967,"Ġweakest":34968,"ĠBulldogs":34969,"UA":34970,"ĠAlloy":34971,"Ġsemen":34972,"ocene":34973,"ĠWilliamson":34974,"spr":34975,",âĢĶ":34976,"ĠGF":34977,"ittens":34978,"Beat":34979,"ĠJunk":34980,"iphate":34981,"ĠFarmers":34982,"ĠBitcoins":34983,"igers":34984,"dh":34985,"ĠLoyal":34986,"payer":34987,"Ġentertained":34988,"Ġpenned":34989,"Ġcoupon":34990,"Queue":34991,"Ġweakening":34992,"carry":34993,"Ġunderestimate":34994,"Ġshootout":34995,"Ġcharismatic":34996,"ĠProcedure":34997,"Ġprudent":34998,"inances":34999,"Ġriches":35000,"Ġcortical":35001,"Ġstrides":35002,"Ġdrib":35003,"ĠOilers":35004,"540":35005,"ĠPerform":35006,"ĠBangkok":35007,"Ġeuth":35008,"SER":35009,"Ġsimplistic":35010,"tops":35011,"campaign":35012,"Quality":35013,"Ġimpoverished":35014,"ĠEisenhower":35015,"Ġaugment":35016,"ĠHarden":35017,"Ġintervened":35018,"Ġlistens":35019,"ĠKok":35020,"Ġsage":35021,"Ġrubbish":35022,"ĠDed":35023,"Ġmull":35024,"pelling":35025,"Ġvideot":35026,"Production":35027,"DJ":35028,"miah":35029,"Ġadaptations":35030,"Ġmedically":35031,"Ġboarded":35032,"Ġarrogance":35033,"Ġscrapped":35034,"Ġoppress":35035,"FORMATION":35036,"Ġjunction":35037,"415":35038,"EEEE":35039,"Skill":35040,"Ġsubdu":35041,"ĠSuggest":35042,"ĠPett":35043,"Ġlett":35044,"ĠManip":35045,"ĠCaf":35046,"ĠCooperation":35047,"Ther":35048,"Ġregained":35049,"¶æ":35050,"reflect":35051,"Ġthugs":35052,"ĠShelby":35053,"Ġdictates":35054,"ĠWeiner":35055,"ĠHale":35056,"Ġbattleground":35057,"schild":35058,"Ġcondol":35059,"hunt":35060,"ositories":35061,"Ġaccuses":35062,"Filename":35063,"Ġshri":35064,"Ġmotivate":35065,"Ġreflections":35066,"Null":35067,"ĠLobby":35068,"¥µ":35069,"ĠSATA":35070,"ĠBackup":35071,"Ñĥ":35072,"nin":35073,"ĠCorrection":35074,"Ġjuicy":35075,"utra":35076,"ĠPric":35077,"Ġrestraining":35078,"ĠAirbnb":35079,"ĠArrest":35080,"Ġappropriations":35081,"Ġslopes":35082,"Ġmanslaughter":35083,"Ġworkings":35084,"ĠHuss":35085,"ĠFrey":35086,"Leave":35087,"ĠHarmony":35088,"ĠFeder":35089,"Ġ430":35090,"Ġtrench":35091,"Ġgladly":35092,"Ġbullpen":35093,"ĠGau":35094,"bones":35095,"Ġgroove":35096,"Ġpretext":35097,"ãħĭ":35098,"Ġtransmitter":35099,"ĠComponent":35100,"Ġunderage":35101,"ĠEmpires":35102,"Tile":35103,"Ġoy":35104,"ĠMarvin":35105,"ĠCAS":35106,"Ġbloss":35107,"Ġreplicated":35108,"ĠMariners":35109,"Marcus":35110,"ĠBlocks":35111,"Ġliberated":35112,"Ġbutterfly":35113,"Feel":35114,"Ġfermentation":35115,"Ġyoutube":35116,"Ġoffend":35117,"ĠTerm":35118,"resist":35119,"Ġcessation":35120,"Ġinsurgency":35121,"Ġbir":35122,"ĠRaise":35123,"595":35124,"Ġhypotheses":35125,"502":35126,"Ġplaque":35127,"ocrat":35128,"Ġjackets":35129,"ĠHuffPost":35130,"among":35131,"Ġconfer":35132,"487":35133,"ĠLilly":35134,"Ġadapting":35135,"ĠFay":35136,"Ġshoved":35137,"vec":35138,"Ġrefine":35139,"Ġgon":35140,"Ġgunmen":35141,"zai":35142,"ĠShuttle":35143,"ĠIzan":35144,"Ġ1913":35145,"Ġplethora":35146,"··":35147,"Ġ510":35148,"Ġpuberty":35149,"Ġ241":35150,"ĠWealth":35151,"ĠAlma":35152,"ĠMEM":35153,"ĠAdults":35154,"Cas":35155,"prison":35156,"Race":35157,"Ġwaterproof":35158,"Ġathleticism":35159,"Ġcapitalize":35160,"ĠJuice":35161,"Ġilluminated":35162,"ĠPascal":35163,"Ġirritation":35164,"ĠWitnesses":35165,"adle":35166,"ĠAstro":35167,"Ġfax":35168,"ĠElvis":35169,"Primary":35170,"ĠLich":35171,"ĠElves":35172,"Ġresiding":35173,"Ġstumble":35174,"319":35175,"ĠPKK":35176,"Ġadversaries":35177,"DOS":35178,"ĠRitual":35179,"Ġsmear":35180,"Ġarson":35181,"idental":35182,"Ġscant":35183,"Ġmonarchy":35184,"Ġhalftime":35185,"Ġresidue":35186,"Ġindign":35187,"ĠShaun":35188,"ĠElm":35189,"auri":35190,"Aff":35191,"WATCH":35192,"ĠLyon":35193,"helps":35194,"361":35195,"Ġlobbyist":35196,"Ġdiminishing":35197,"Ġoutbreaks":35198,"Ġgoats":35199,"favorite":35200,"ĠNah":35201,"sonian":35202,"ĠBooster":35203,"Ġsandbox":35204,"ĠFare":35205,"ĠMalta":35206,"ĠattRot":35207,"ĠMOR":35208,"lde":35209,"Ġnavigating":35210,"Touch":35211,"Ġuntrue":35212,"ĠDisaster":35213,"Ġludicrous":35214,"Password":35215,"ĠJFK":35216,"blogspot":35217,"416":35218,"ĠUNDER":35219,"ernal":35220,"Ġdelaying":35221,"TOP":35222,"Ġimplants":35223,"ĠAVG":35224,"ĠHuge":35225,"attr":35226,"Ġjournalistic":35227,"ĠPeyton":35228,"ĠIA":35229,"Rap":35230,"goal":35231,"ĠProgramme":35232,"Ġsmashing":35233,"wives":35234,"println":35235,"ĠPlague":35236,"inus":35237,"EEP":35238,"Ġcruiser":35239,"ĠParish":35240,"uminium":35241,"Ġoccupants":35242,"ĠJihad":35243,"mop":35244,"Ġpint":35245,"Ġhect":35246,"ĠMecca":35247,"director":35248,"ĠFunding":35249,"ĠMixed":35250,"Ġstag":35251,"Tier":35252,"Ġgust":35253,"Ġbrightly":35254,"orsi":35255,"Ġuphill":35256,"RD":35257,"Ġlesions":35258,"ĠBundy":35259,"livious":35260,"Ġbiologist":35261,"ĠFaculty":35262,"ĠAuthorization":35263,"Ġ244":35264,"Allow":35265,"ï¸":35266,"ĠGiul":35267,"Ġpertinent":35268,"otaur":35269,"esse":35270,"ĠRoof":35271,"Ġunmanned":35272,"351":35273,"ĠShak":35274,"ĠOrient":35275,"Ġendanger":35276,"Dir":35277,"Ġreplen":35278,"edient":35279,"Ġtailor":35280,"Ġgadgets":35281,"Ġaudible":35282,"âĺĨ":35283,"Nice":35284,"Ġbombard":35285,"ĠRape":35286,"Ġdefiance":35287,"ĠTWO":35288,"ĠFilipino":35289,"Ġunaffected":35290,"ervatives":35291,"Ġsoared":35292,"ĠBolton":35293,"Ġcompromising":35294,"ĠBrewers":35295,"RAL":35296,"ĠAHL":35297,"icycle":35298,"Ġvampires":35299,"Ġdipped":35300,"oyer":35301,"ĠXIII":35302,"Ġsideways":35303,"ĠWaste":35304,"ĠDiss":35305,"ĠâĶľâĶĢâĶĢ":35306,"$.":35307,"Ġhabitats":35308,"ĠBeef":35309,"truth":35310,"trained":35311,"split":35312,"Rus":35313,"Andy":35314,"ĠBram":35315,"REP":35316,"pid":35317,"è£ħ":35318,"ĠMutant":35319,"Anim":35320,"ĠMarina":35321,"Ġfutile":35322,"highest":35323,"frequency":35324,"Ġepilepsy":35325,"Ġcoping":35326,"Ġconcise":35327,"Ġtracing":35328,"ĠSUN":35329,"panel":35330,"ĠSophie":35331,"ĠCrowley":35332,"ĠAdolf":35333,"ĠShooter":35334,"Ġshaky":35335,"ĠIG":35336,"ĠLies":35337,"ĠBarber":35338,"pkg":35339,"Ġuptake":35340,"Ġpredatory":35341,"ULTS":35342,"/**":35343,"Ġintoxicated":35344,"ĠWestbrook":35345,"odder":35346,"hement":35347,"Ġbaseman":35348,"APD":35349,"storage":35350,"ĠFifty":35351,"editor":35352,"GEN":35353,"UTION":35354,"irting":35355,"Ġsewing":35356,"rift":35357,"Ġagony":35358,"ĠSands":35359,"Ġ254":35360,"Cash":35361,"Ġlodge":35362,"Ġpunt":35363,"Natural":35364,"ĠIdeas":35365,"Ġerroneous":35366,"ĠSensor":35367,"ĠHannity":35368,"Ġ1921":35369,"Ġmould":35370,"ĠGon":35371,"kaya":35372,"Ġanonymously":35373,"ĠKEY":35374,"Ġsimulator":35375,"Winter":35376,"Ġstreamed":35377,"507":35378,"?\",":35379,"Ġteased":35380,"Ġcoefficient":35381,"Ġwartime":35382,"ĠTHR":35383,"''.":35384,"ĠBanking":35385,"mpire":35386,"Ġfandom":35387,"Ġlia":35388,"Ga":35389,"Ġdownhill":35390,"Ġinterpreting":35391,"Individual":35392,"Norm":35393,"Ġjealousy":35394,"bitcoin":35395,"Ġpleasures":35396,"ĠToys":35397,"ĠChevrolet":35398,"ĠAdvisor":35399,"IZE":35400,"Ġreceptions":35401,"706":35402,"Cro":35403,"Ġ262":35404,"Ġcitrus":35405,"iru":35406,"Reviewer":35407,"jected":35408,"UES":35409,"anz":35410,"1981":35411,"ĠWorker":35412,"Ġcomplied":35413,"orescent":35414,"continental":35415,"Ton":35416,"ĠPrism":35417,"ĠSheep":35418,"Ġ288":35419,"nox":35420,"ĠVog":35421,"Ord":35422,"Ġrealms":35423,"tek":35424,"Ġirrigation":35425,"Ġbicycles":35426,"Ġelectronically":35427,"poly":35428,"tall":35429,"());":35430,"Ġaesthetics":35431,"ĠIntegrated":35432,"Explore":35433,"Ġdunk":35434,"476":35435,"pain":35436,"ĠJacques":35437,"ĠDmit":35438,"Frames":35439,"Ġreunited":35440,"Ġhumid":35441,"Dro":35442,"Political":35443,"Ġyouthful":35444,"Ġentails":35445,"Ġmosquito":35446,"363":35447,"species":35448,"Ġcoordinating":35449,"ĠMayhem":35450,"ĠMagnus":35451,"Mount":35452,"Improved":35453,"ĠSTATE":35454,"ATTLE":35455,"Ġflowed":35456,"Ġtackled":35457,"Ġfashioned":35458,"Ġreorgan":35459,"ivari":35460,"finger":35461,"Ġreluctantly":35462,"etting":35463,"ĠVand":35464,"young":35465,"ĠGarland":35466,"Ġpresumption":35467,"Ġamenities":35468,"ĠPleasant":35469,"onential":35470,"ĠOxy":35471,"Ġmorals":35472,"ĠYah":35473,"Ready":35474,"Simon":35475,"Enh":35476,"Demon":35477,"Ġclich":35478,"Monitor":35479,"ĠDU":35480,"Ġwelcomes":35481,"Ġstandout":35482,"Ġdreadful":35483,"Ġbananas":35484,"Ġballoons":35485,"hooting":35486,"basic":35487,"Ġsuffix":35488,"Ġduly":35489,"cano":35490,"Chain":35491,"atos":35492,"Ġgeopolitical":35493,"Ġ(&":35494,"ĠGemini":35495,"ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ":35496,"Ġacquitted":35497,"Luck":35498,"protect":35499,"1024":35500,"Ġscarcity":35501,"Ġmindfulness":35502,"ecided":35503,"DN":35504,"prime":35505,"ĠPresidents":35506,"ĠVIDEO":35507,"Ġ(âĪĴ":35508,"addock":35509,"NOR":35510,"ĠPru":35511,"pun":35512,"ĠLOL":35513,"))))":35514,"ĠLiqu":35515,"ĠSAS":35516,"Ġstyling":35517,"Ġpunishments":35518,"Ġnumb":35519,"Ġascertain":35520,"ĠRockies":35521,"flu":35522,"Thumbnail":35523,"Ġperpetrated":35524,"ĠSemi":35525,"Ġdisarm":35526,"ĠOlder":35527,"ĠException":35528,"Ġexponentially":35529,"ĠCommunities":35530,"Ġabolish":35531,"ĠPartner":35532,"ptoms":35533,"Ġ777":35534,"ĠFoley":35535,"ĠCases":35536,"Ġgrease":35537,"ĠRebirth":35538,"Ground":35539,"Ġ;)":35540,"ĠDoctrine":35541,"ikini":35542,"Ye":35543,"ĠBlossom":35544,"Ġpersists":35545,"bill":35546,"Ġinfusion":35547,"Ġbuddies":35548,"911":35549,"ĠPatient":35550,"Ġdemos":35551,"Ġacquaintance":35552,"ĠPaw":35553,"atari":35554,"Ġxml":35555,"Ġfascination":35556,"ĠServe":35557,"ÏĤ":35558,"branded":35559,"Ġaz":35560,"Returns":35561,"Ġovershadow":35562,"Ġroam":35563,"Ġspeedy":35564,"numbered":35565,"helial":35566,"Ġdisciple":35567,"Ġassurances":35568,"given":35569,"pecting":35570,"ĠNatalie":35571,"çͰ":35572,"Ġmosquitoes":35573,"rotein":35574,"Ġnumeric":35575,"Ġindependents":35576,"Ġtransitional":35577,"Ġreactionary":35578,"ĠMechdragon":35579,"doctor":35580,"Ġshortest":35581,"Ġsequential":35582,"ĠBac":35583,"ĠAccounts":35584,"ãģĮ":35585,"achy":35586,"ractive":35587,"ĠRegiment":35588,"Ġbreathtaking":35589,"fficiency":35590,"ĠBates":35591,"Ġ311":35592,"Ġwardrobe":35593,"fts":35594,"ĠBerk":35595,"Simply":35596,"ĠRiverside":35597,"ivering":35598,"idential":35599,"lucent":35600,"Ġenriched":35601,"ĠConver":35602,"ĠGiving":35603,"ãĥĻ":35604,"Ġlegalize":35605,"ĠFTC":35606,"Ġfreaking":35607,"Mix":35608,"Ġterrestrial":35609,"esian":35610,"cients":35611,"Wing":35612,"LOAD":35613,"Ġledge":35614,"ĠViolent":35615,"ĠMetall":35616,"Ġ308":35617,"Ġsoutheastern":35618,"hetto":35619,"Meat":35620,"Ġslowdown":35621,"Ġretreated":35622,"Jeremy":35623,"endas":35624,"*****":35625,"eric":35626,"Ġreins":35627,"oppable":35628,"ĠHumanity":35629,"earances":35630,"rigan":35631,"Camera":35632,"Ġwaivers":35633,"soc":35634,"Ġalteration":35635,"transform":35636,"ĠCemetery":35637,"506":35638,"Ġindefinite":35639,"Ġstimulating":35640,"yg":35641,"603":35642,"ĠSop":35643,"Ġdescriptive":35644,"Phase":35645,"ĠEdmund":35646,"Ġpneumonia":35647,"ventus":35648,"Amb":35649,"Ġlaboratories":35650,"ĠExclusive":35651,"ugar":35652,"Were":35653,"Ġmalfunction":35654,"Ġhomosexuals":35655,"Ġ-------":35656,"uni":35657,"Ġturbines":35658,"ĠEquity":35659,"Du":35660,"Ġminded":35661,"ĠRH":35662,"ĠBlackhawks":35663,"Ġfeats":35664,"Ġ1700":35665,"repl":35666,"362":35667,"laden":35668,"Ġindispensable":35669,"lyss":35670,"tti":35671,"Ġreel":35672,"Ġdiverted":35673,"Ġlikeness":35674,"Ġsubscriptions":35675,"Ġfingert":35676,"Ġfilthy":35677,"destruct":35678,"draft":35679,"ĠBernardino":35680,"launch":35681,"Ġperplex":35682,"ĠSUM":35683,"carb":35684,"Ġsweater":35685,"ĠVenture":35686,"ĠJag":35687,"ĠCeleb":35688,"ĠVoters":35689,"Ġsteadfast":35690,"Ġathletics":35691,"ĠHanson":35692,"ĠDrac":35693,"Tracker":35694,"Ġcommend":35695,"ĠPresidency":35696,"ĠDID":35697,"informed":35698,"Ġwebpage":35699,"Pretty":35700,"Ġforcefully":35701,"ãĥĥãĤ¯":35702,"Ġrelocation":35703,"Ġsatire":35704,"âī":35705,"ĠSunderland":35706,"æĦ":35707,"Voice":35708,"????????":35709,"Ġinformant":35710,"Ġbowel":35711,"ĠUniform":35712,"Ġ...\"":35713,"Ġpurge":35714,"Ġpicnic":35715,"ĠUmb":35716,"ĠUPDATE":35717,"ĠSapphire":35718,"ĠStall":35719,"learn":35720,"Ġobjectively":35721,"Ġobliter":35722,"Ġloophole":35723,"Ġjourneys":35724,"Ġomission":35725,"Pros":35726,"ĠSidney":35727,"ploma":35728,"Ġsprayed":35729,"Ġguru":35730,"Ġtraitor":35731,"Ġtimet":35732,"Ġsnapping":35733,"ĠSevent":35734,"urnal":35735,"ĠUkip":35736,"Ġbowed":35737,"poral":35738,"liberal":35739,"Ros":35740,"Questions":35741,"iOS":35742,"Ġsummarize":35743,"STAT":35744,"Ġ1850":35745,"apest":35746,"Ġlender":35747,"ĠVariable":35748,"bringing":35749,"ĠLORD":35750,",)":35751,"Ġcollapses":35752,"xiety":35753,"ĠNed":35754,"YD":35755,"ĠScha":35756,"Ġantibody":35757,"Ġdisband":35758,"yre":35759,"illusion":35760,"Ġrover":35761,"shed":35762,"ĠHirosh":35763,"cci":35764,"Ġcalam":35765,"ĠMorton":35766,"Pinterest":35767,"Ġ1928":35768,"ĠEuras":35769,"ordes":35770,"Ġfences":35771,"ĠInventory":35772,"ĠValencia":35773,"ĠUd":35774,"ĠTiff":35775,"Ġsque":35776,"Ġquotation":35777,"Ġtroublesome":35778,"erker":35779,"QUEST":35780,"ĠKingdoms":35781,"south":35782,"Ġlevy":35783,"Prince":35784,"ĠSting":35785,"Ġnicknamed":35786,"Ġappe":35787,"Ġphotographic":35788,"Ġcorpus":35789,"reference":35790,"ĠTrog":35791,"Unt":35792,")=(":35793,"ĠLatvia":35794,"Ġactivating":35795,"Ġlicensee":35796,"Ġdisparities":35797,"ĠNewsletter":35798,"ãĥĥãĥĪ":35799,"Ġfreeing":35800,"ĠJeep":35801,"ĠPerception":35802,"insk":35803,"Ġsilicone":35804,"ĠHayden":35805,"Lean":35806,"ĠSuzuki":35807,"ibrarian":35808,"668":35809,"Ġspor":35810,"Ġcorrelations":35811,"aghetti":35812,"Ġtuber":35813,"ĠIPCC":35814,"ilus":35815,"ĠVu":35816,"Ġwealthiest":35817,"ĠCarbuncle":35818,"anza":35819,"Ġfooled":35820,"ĠZur":35821,"Ġdaddy":35822,"rano":35823,"ilian":35824,"Ġknockout":35825,"fman":35826,"required":35827,"ĠWikileaks":35828,"ĠDuffy":35829,"ONT":35830,"Ġinsol":35831,"ĠObjects":35832,"Ġbou":35833,"ĠNordic":35834,"ĠInsert":35835,"scan":35836,"Ġdancers":35837,"Ġidiots":35838,"majority":35839,"ĠNeville":35840,"ĠFreeBSD":35841,"Ġtart":35842,"panic":35843,"690":35844,"Ġcocoa":35845,"Ġsampled":35846,"Ġlookup":35847,"Indust":35848,"Ġinjections":35849,"genre":35850,"Ġau":35851,"Ġroadway":35852,"Ġgenitals":35853,"Kind":35854,"ĠExaminer":35855,"ĠYaz":35856,"Fresh":35857,"Ġparalysis":35858,"ĠAluminum":35859,"Ġreap":35860,"oké":35861,"Ġsloppy":35862,"ĠTunnel":35863,"posium":35864,"nery":35865,"enic":35866,"Ġherbal":35867,"ĠOuter":35868,"ĠBuilder":35869,"Ġincur":35870,"Ġideologies":35871,"Ġbackups":35872,"consuming":35873,"ĠDetect":35874,"deck":35875,"ĠKNOW":35876,"ĠGret":35877,"ĠMIC":35878,"Ġtoughness":35879,"ĠExhibit":35880,"Ġhive":35881,"Les":35882,"ĠSCHOOL":35883,"ĠAtari":35884,"alde":35885,"ĠNull":35886,"andestine":35887,"mouse":35888,"Ġbrigade":35889,"489":35890,"Ġrevol":35891,"ĠLawson":35892,"ĠWah":35893,"opoly":35894,"ebted":35895,"ĠSaunders":35896,"Ġ313":35897,"ĠWinc":35898,"Ġtaboo":35899,"ĠHelmet":35900,"Ġwedge":35901,"chip":35902,"ĠTina":35903,"bg":35904,"Ġinfuri":35905,"rn":35906,"Ġanomalies":35907,"ĠSync":35908,"ĠExam":35909,"ĠCommit":35910,"ĠDiary":35911,"ĠALSO":35912,"ĠDebor":35913,"omedical":35914,"Ġcomprehension":35915,"655":35916,"Ġempowering":35917,"Ġire":35918,"Ġjuices":35919,"ĠETH":35920,"ĠBoxing":35921,"=\"/":35922,"Ġfacilitated":35923,"poke":35924,"ĠParsons":35925,"ĠModer":35926,"travel":35927,"Ġcivilizations":35928,"Ġlibertarians":35929,"Ġrune":35930,"ĠClarks":35931,"athed":35932,"Ġcampaigners":35933,"ĠDispatch":35934,"ĠFahrenheit":35935,"ĠCapcom":35936,"----------":35937,"Ġlace":35938,"Ġdraining":35939,"Ġliner":35940,"ĠArtificial":35941,"én":35942,"task":35943,"]).":35944,"ĠGMO":35945,"ĠOperator":35946,"ordinary":35947,"ĠInfluence":35948,"ĠUps":35949,"Ġpotency":35950,"ussen":35951,"ospons":35952,"ĠSwim":35953,"ĠDeadline":35954,"Unity":35955,"Ġculinary":35956,"Ġenlightenment":35957,"Ġwearer":35958,"Ġmined":35959,"Ġply":35960,"Ġincest":35961,"ĠDVDs":35962,"Walk":35963,"BTC":35964,"Trade":35965,"Ġdeval":35966,"iband":35967,"ĠOversight":35968,"Palestinian":35969,"Ġdart":35970,"Ġmul":35971,"LR":35972,"Ġremovable":35973,"ĠRealms":35974,"ìĿ":35975,"Ġmiscar":35976,"ĠVulkan":35977,"685":35978,"ère":35979,"ĠSap":35980,"Ġmerging":35981,"ĠCarly":35982,"chester":35983,"Ġbrisk":35984,"Ġluxurious":35985,"ĠGenerator":35986,"Ġbitterness":35987,"Ġedible":35988,"Ġ243":35989,"TG":35990,"Ġrectangle":35991,"WithNo":35992,"below":35993,"Jenn":35994,"Ġdarkest":35995,"Ġhitch":35996,"Ġdosage":35997,"Ġscaven":35998,"ĠKeller":35999,"ĠIllustrated":36000,"Certainly":36001,"ĠMavericks":36002,"Marginal":36003,"Ġdiarrhea":36004,"Ġenormously":36005,"Ġ999":36006,"shr":36007,"quart":36008,"Ġadamant":36009,"ĠMew":36010,"Ġrenovation":36011,"Ġcervical":36012,"ĠPercentage":36013,"eners":36014,"ĠKimber":36015,"Ġfloats":36016,"Ġdex":36017,"ĠWitcher":36018,"ĠSwansea":36019,"dm":36020,"Ġsalty":36021,"yellow":36022,"Ġcape":36023,"ĠDrain":36024,"ĠPaula":36025,"ĠToledo":36026,"lesi":36027,"Magazine":36028,"ĠWick":36029,"ĠMn":36030,"ĠAck":36031,"ĠRiding":36032,"ASON":36033,"Ġhomophobic":36034,"ARP":36035,"Ġwandered":36036,"CPU":36037,"oodoo":36038,"ĠPipe":36039,"Ġtightening":36040,"ĠButt":36041,"318":36042,"Ġdeserted":36043,"Session":36044,"Ġfacilitating":36045,"Jump":36046,"Ġemergencies":36047,"OWER":36048,"Ġexhaustive":36049,"ĠAFTER":36050,"Ġheartbeat":36051,"ĠLabel":36052,"acky":36053,"ĠCertified":36054,"iltration":36055,"Ze":36056,"ĠUtt":36057,"Ġ1300":36058,"Ġpresume":36059,"ĠDisp":36060,"Ġsurged":36061,"Ġdolls":36062,"Columb":36063,"Ġchimpan":36064,"ĠRazor":36065,"Ġticks":36066,"Ġcouncillor":36067,"Ġpilgrimage":36068,"ĠRebels":36069,"ĠQC":36070,"ĠAuction":36071,"xia":36072,"ikk":36073,"bred":36074,"Ġinsertion":36075,"Ġcoarse":36076,"dB":36077,"SEE":36078,"ĠZap":36079,"ĠFoo":36080,"Ġcontempor":36081,"ĠQuarterly":36082,"otions":36083,"ĠAlchemist":36084,"ĠTrey":36085,"ĠDuo":36086,"Sweet":36087,"804":36088,"ĠGiov":36089,"Ġfunn":36090,"Nin":36091,"hoff":36092,"Ġramifications":36093,"Ġ1922":36094,"ĠExperts":36095,"azes":36096,"Ġgarments":36097,"arial":36098,"ĠNab":36099,"Ġ257":36100,"ĠVed":36101,"Ġhumorous":36102,"ĠPompe":36103,"Ġnylon":36104,"Ġlurking":36105,"ĠSergey":36106,"ĠMattis":36107,"Ġmisogyny":36108,"ĠComponents":36109,"ĠWatching":36110,"ĠFolk":36111,"ractical":36112,"Bush":36113,"Ġtaped":36114,"Ġgrouping":36115,"Ġbeads":36116,"Ġ2048":36117,"Ġcondu":36118,"querque":36119,"Reading":36120,"Ġgrievances":36121,"Ultra":36122,"Ġendpoint":36123,"Hig":36124,"ĠStatic":36125,"ĠScarborough":36126,"Lua":36127,"ĠMessi":36128,"aqu":36129,"ĠPsyNet":36130,"ĠRudd":36131,"Ġavenue":36132,"vp":36133,"Jer":36134,"Ġshady":36135,"ĠResist":36136,"ĠArtemis":36137,"Ġcareless":36138,"Ġbrokers":36139,"Ġtemperament":36140,"Ġ520":36141,"Tags":36142,"ĠTurning":36143,"Ġuttered":36144,"Ġpedd":36145,"Ġimprovised":36146,"Ġ:(":36147,"Ġtabl":36148,"Ġplains":36149,"1600":36150,"pressure":36151,"ĠEssence":36152,"margin":36153,"friends":36154,"ĠRestoration":36155,"Ġpollut":36156,"ĠPoker":36157,"ĠAugustine":36158,"ĠCIS":36159,"ĠSEAL":36160,"orama":36161,"Ġthwart":36162,"seek":36163,"Ġpagan":36164,"º":36165,"cpu":36166,"Ġgarn":36167,"Ġassortment":36168,"ĠILCS":36169,"tower":36170,"Recommended":36171,"Ġunborn":36172,"ĠRandomRedditor":36173,"ĠRandomRedditorWithNo":36174,"Ġparalyzed":36175,"Ġeruption":36176,"Ġintersect":36177,"ĠStoke":36178,"ĠSco":36179,"Bind":36180,"å¾":36181,"ĠPNG":36182,"ĠNegative":36183,"ĠNOAA":36184,"Leon":36185,"Ġalloy":36186,"ĠLama":36187,"ĠDiversity":36188,"575":36189,"Ġunderestimated":36190,"ĠScor":36191,"Ġmural":36192,"Ġbusted":36193,"soon":36194,"lif":36195,"Ġnonex":36196,"Ġallergy":36197,"ĠUnderworld":36198,"ĠRays":36199,"ĠBlasio":36200,"Ġhrs":36201,"ĠDir":36202,"Ġ327":36203,"byter":36204,"Ġreplacements":36205,"Ġactivates":36206,"rived":36207,"MH":36208,"Ġpans":36209,"ĠHI":36210,"Ġlongitudinal":36211,"Ġnuisance":36212,"aler":36213,"Ġswell":36214,"ĠSigned":36215,"sci":36216,"ĠIsles":36217,"ĠAGA":36218,"Ġdefiant":36219,"Ġsonic":36220,"ocon":36221,"KC":36222,"ĠAim":36223,"tie":36224,"ahah":36225,"ĠmL":36226,"DX":36227,"Ġbisc":36228,"ĠBillboard":36229,"ĠSYSTEM":36230,"NEY":36231,"gaard":36232,"Ġdistressed":36233,"formerly":36234,"Alan":36235,"Ġchefs":36236,"Ġoptics":36237,"ĠComet":36238,"ĠAMC":36239,"Ġredesigned":36240,"irmation":36241,"Ġsightings":36242,"382":36243,"311":36244,"ĠWB":36245,"Ġcontraction":36246,"ĠTOTAL":36247,"Dual":36248,"Ġstartled":36249,"Ġunderstandably":36250,"Ġsunglasses":36251,"ETHOD":36252,"Ġdocker":36253,"Ġsurfing":36254,"ĠHEL":36255,"ĠSlack":36256,"tones":36257,"Ġshalt":36258,"Visual":36259,"498":36260,"Department":36261,"cussion":36262,"Ġunrestricted":36263,"Ġtad":36264,"Ġrename":36265,"employed":36266,"Ġeducating":36267,"Ġgrinned":36268,"bedroom":36269,"ĠActivities":36270,"ĠVelvet":36271,"ĠSWAT":36272,"Ġshuffle":36273,"igor":36274,"Ġsaturation":36275,"Finding":36276,"cream":36277,"icter":36278,"Ġvodka":36279,"tracking":36280,"tec":36281,"Ġforeground":36282,"iesta":36283,"Ġvehement":36284,"ĠECB":36285,"ĠTie":36286,"Ey":36287,"Ġturtles":36288,"ĠRailroad":36289,"ĠKatz":36290,"ĠFrames":36291,"Ġmenace":36292,"ĠFellowship":36293,"ĠEssential":36294,"uggish":36295,"Ġdrip":36296,"chwitz":36297,"ĠKyoto":36298,"sb":36299,"ĠNina":36300,"Parameter":36301,"Ġalarms":36302,"ĠClaud":36303,"Ġpioneering":36304,"Ġchiefly":36305,"ĠScream":36306,"Collection":36307,"Ġthankfully":36308,"ĠRonaldo":36309,"åŃIJ":36310,"strip":36311,"ĠDisneyland":36312,"commercial":36313,"Seeing":36314,"Soul":36315,"Ġevacuate":36316,"Ġciv":36317,"ĠAshe":36318,"Ġdivides":36319,"ĠDagger":36320,"rehensive":36321,"Ġberries":36322,"ĠDF":36323,"Ġsushi":36324,"Ġplurality":36325,"WI":36326,"Ġdisadvantaged":36327,"Ġbattalion":36328,"obiles":36329,"451":36330,"Ġcling":36331,"Ġundeniable":36332,"ĠLounge":36333,"Ġhaunt":36334,"phe":36335,"Ġquantify":36336,"Ġdiffered":36337,"Ġ[*]":36338,"ĠViz":36339,"cum":36340,"slave":36341,"Ġvideog":36342,"Ġquar":36343,"Ġbundles":36344,"ĠAlonso":36345,"tackle":36346,"Ġneuronal":36347,"Ġlandslide":36348,"confirmed":36349,"ĠDepth":36350,"Ġrenewables":36351,"Bear":36352,"ĠMacedonia":36353,"Ġjerseys":36354,"Ġbunk":36355,"ĠSpawn":36356,"ĠControls":36357,"ĠBuchanan":36358,"Ġrobotics":36359,"Ġemphasizing":36360,"ĠTutorial":36361,"hyp":36362,"iston":36363,"Ġmonumental":36364,"æ°":36365,"ĠCarry":36366,"Ġtbsp":36367,"enance":36368,"Hill":36369,"arthed":36370,"Ġrotten":36371,"Dean":36372,"Ġtwisting":36373,"Ġgoodwill":36374,"Ġimmersion":36375,"Living":36376,"Ġbrushes":36377,"ĠCGI":36378,"ĠAtk":36379,"traditional":36380,"Ġphantom":36381,"ĠStamina":36382,"Ġexpansions":36383,"ĠMarin":36384,"Ġembarked":36385,"ĠEg":36386,"intestinal":36387,"ĠPEOPLE":36388,"ĠBooth":36389,"ĠAppalach":36390,"Ġrelegated":36391,"VT":36392,"MIT":36393,"Ġmuster":36394,"Ġwithdrawing":36395,"Ġmicroscope":36396,"ĠGathering":36397,"ĠCrescent":36398,"ĠArgentine":36399,"ĠDecre":36400,"ĠDominic":36401,"Ġbuds":36402,"antage":36403,"ĠIon":36404,"Ġwidened":36405,"ONSORED":36406,"ĠGloves":36407,"iannopoulos":36408,"razen":36409,"feel":36410,"Ġrepayment":36411,"Ġhindsight":36412,"ĠREALLY":36413,"ĠPistol":36414,"ĠBrah":36415,"Ġwatts":36416,"Ġsurvives":36417,"Ġflurry":36418,"issy":36419,"Alert":36420,"ĠUruguay":36421,"Phoenix":36422,"Slow":36423,"ĠGrave":36424,"ĠFir":36425,"Ġmanageable":36426,"Ġtariff":36427,"ĠUDP":36428,"ĠPistons":36429,"ĠNigerian":36430,"Ġstrikeouts":36431,"Ġcosmetics":36432,"whelming":36433,"fab":36434,"cape":36435,"proxy":36436,"Ġrethink":36437,"Ġovercoming":36438,"simple":36439,"Ġwoo":36440,"Ġdistracting":36441,"ĠStanton":36442,"ĠTulsa":36443,"ĠDock":36444,"659":36445,"Ġdiscord":36446,"ĠEmacs":36447,"ĠVes":36448,"ĠROB":36449,"Ġreassuring":36450,"Ġconsortium":36451,"Muslims":36452,"321":36453,"Ġprompts":36454,"sei":36455,"ĠHitch":36456,"imposed":36457,"ĠFool":36458,"Ġindiscrim":36459,"wrong":36460,"buquerque":36461,"Davis":36462,"!]":36463,"Ġtimeless":36464,"ĠNEED":36465,"Ġpesticide":36466,"Ġrallying":36467,"ĠCalder":36468,"Ġå¤":36469,"Ġxp":36470,"ĠUnle":36471,"ĠExport":36472,"luaj":36473,"Buff":36474,")[":36937,"Ġsqor":36938,"Saudi":36939,"Ġistg":36940,"Ġindulge":36941,"proc":36942,"Ġdisgusted":36943,"Ġcompounded":36944,"Ġnem":36945,"Ġschooling":36946,"ĠCure":36947,"processing":36948,"Sol":36949,"Ġproverb":36950,"itized":36951,"ĠAlvarez":36952,"Ġscarf":36953,"Ġrectangular":36954,"reve":36955,"Ġhormonal":36956,"ĠStress":36957,"itizen":36958,"Ġ425":36959,"girls":36960,"ĠNoir":36961,"ĠRapp":36962,"Ġmarches":36963,"church":36964,"ĠUses":36965,"Ġ405":36966,"ĠBerm":36967,"Ġordinances":36968,"ĠJudgment":36969,"Charges":36970,"ĠZin":36971,"Ġdusty":36972,"Ġstrawberries":36973,"Ġperce":36974,"ĠThur":36975,"ĠDeborah":36976,"netflix":36977,"ĠLambert":36978,"Ġamused":36979,"ĠGuang":36980,"YOU":36981,"RGB":36982,"ĠCCTV":36983,"Ġfiat":36984,"rang":36985,"Ġfederation":36986,"ĠMant":36987,"ĠBust":36988,"ĠMare":36989,"respective":36990,"ĠMigration":36991,"ĠBIT":36992,"590":36993,"Ġpatriotism":36994,"Ġoutlining":36995,"region":36996,"ĠJosé":36997,"Ġblasting":36998,"ĠEzra":36999,"Bs":37000,"Ġundermines":37001,"ĠSmooth":37002,"Ġclashed":37003,"radio":37004,"Ġtransitioning":37005,"ĠBuccaneers":37006,"ĠOwl":37007,"Ġplugs":37008,"Ġhiatus":37009,"ĠPinball":37010,"Ġmig":37011,"ĠNutr":37012,"ĠWolfe":37013,"Ġintegers":37014,"Ġorbits":37015,"ĠEdwin":37016,"ĠDirectX":37017,"bite":37018,"Ġblazing":37019,"vr":37020,"Edge":37021,"ĠPID":37022,"exit":37023,"ĠComed":37024,"ĠPathfinder":37025,"ĠGuid":37026,"ĠSigns":37027,"ĠZer":37028,"ĠAgenda":37029,"Ġreimbursement":37030,"Mesh":37031,"iPhone":37032,"ĠMarcos":37033,"ĠSites":37034,"hate":37035,"enburg":37036,"Ġsockets":37037,"pend":37038,"Batman":37039,"vir":37040,"ĠSHOW":37041,"Ġprovisional":37042,"conn":37043,"ĠDeaths":37044,"ATIVE":37045,"Profile":37046,"sym":37047,"JA":37048,"Ġninja":37049,"installed":37050,"idates":37051,"ebra":37052,"ĠOmaha":37053,"Ġseizing":37054,"ĠBeasts":37055,"Ġsalts":37056,"Mission":37057,"Generally":37058,"ĠTrilogy":37059,"heon":37060,"legates":37061,"Ġdime":37062,"Ġfaire":37063,"parable":37064,"Graph":37065,"Ġtotaling":37066,"Ġdiagrams":37067,"ĠYanuk":37068,"plet":37069,"ĠMeh":37070,"Ġmythical":37071,"ĠStephens":37072,"autical":37073,"ochemistry":37074,"Ġkilograms":37075,"Ġelbows":37076,"ancock":37077,"ĠBCE":37078,"ĠPrague":37079,"Ġimprov":37080,"ĠDevin":37081,"Ġ\"\\":37082,"paralle":37083,"Ġsupremacists":37084,"ĠBillion":37085,"Ġregimen":37086,"innacle":37087,"Ġrequisite":37088,"angan":37089,"ĠBurlington":37090,"ainment":37091,"ĠObjective":37092,"omsky":37093,"GV":37094,"Ġunilateral":37095,"Ġtc":37096,"Ġhires":37097,"mental":37098,"Ġinvoluntary":37099,"Ġtranspl":37100,"ĠASCII":37101,"¨":37102,"Events":37103,"Ġdoubted":37104,"ĠKaplan":37105,"ĠCourage":37106,"igon":37107,"ĠManaging":37108,"ĠTart":37109,"Ġfalsehood":37110,"ĠViolet":37111,"Ġairs":37112,"Ġfertilizer":37113,"Britain":37114,"Ġaquatic":37115,"ouf":37116,"Words":37117,"ĠHartford":37118,"Ġevenings":37119,"ĠVengeance":37120,"quite":37121,"Gall":37122,"ĠPret":37123,"Ġpdf":37124,"ĠLM":37125,"ĠSochi":37126,"ĠIntercept":37127,"920":37128,"Ġprofitability":37129,"ĠIdle":37130,"ĠMacDonald":37131,"ĠEstablishment":37132,"umsy":37133,"Ġgatherings":37134,"ĠNaj":37135,"Charlie":37136,"Ġascent":37137,"ĠProtector":37138,"Ġalgebra":37139,"Ġbios":37140,"forums":37141,"ELS":37142,"Introduced":37143,"Ġ335":37144,"Ġastronomy":37145,"Contribut":37146,"ĠPolic":37147,"Platform":37148,"Ġcontainment":37149,"wrap":37150,"Ġcoronary":37151,"ĠJelly":37152,"manager":37153,"Ġheartbreaking":37154,"cair":37155,"ĠChero":37156,"cgi":37157,"Medical":37158,"ĠAccountability":37159,"!!\"":37160,"ophile":37161,"Ġpsychotic":37162,"ĠRestrict":37163,"Ġequitable":37164,"issues":37165,"Ġ1905":37166,"ĠNek":37167,"cised":37168,"ĠTracking":37169,"Ġozone":37170,"Ġcooker":37171,"rosis":37172,"Ġreopen":37173,"Ġinfinity":37174,"ĠPharmaceutical":37175,"ensional":37176,"Attempt":37177,"ĠRory":37178,"Marco":37179,"Ġawaits":37180,"HOW":37181,"treated":37182,"Ġbolst":37183,"Ġrevered":37184,"Ġpods":37185,"oppers":37186,"0010":37187,"Ġamplitude":37188,"rican":37189,"SPONSORED":37190,"Ġtrousers":37191,"Ġhalves":37192,"ĠKaine":37193,"ĠCutler":37194,"ĠAUTH":37195,"Ġsplendid":37196,"Ġpreventive":37197,"ĠDudley":37198,"ifacts":37199,"uminati":37200,"ĠYin":37201,"Ġadmon":37202,"ĠVag":37203,"Ġinverted":37204,"Ġhastily":37205,"ĠHague":37206,"Lyn":37207,"Ġledger":37208,"Ġastronomical":37209,"getting":37210,"Ġcirca":37211,"ĠCic":37212,"ĠTennis":37213,"Limited":37214,"Ġdru":37215,"ĠBYU":37216,"Ġtravellers":37217,"Ġpane":37218,"ĠIntro":37219,"Ġpatiently":37220,"Ġaiding":37221,"Ġloos":37222,"ĠTough":37223,"Ġ293":37224,"Ġconsumes":37225,"SourceFile":37226,"Ġ\"\"\"":37227,"Ġbonding":37228,"Ġtilted":37229,"Ġmenstrual":37230,"ĠCelestial":37231,"ULAR":37232,"Plugin":37233,"Ġrisking":37234,"Naz":37235,"ĠRiyadh":37236,"Ġaccredited":37237,"Ġskirm":37238,"éĽ":37239,"Ġexaminer":37240,"Ġmessing":37241,"Ġnearing":37242,"ĠChern":37243,"ĠBeckham":37244,"Ġswapped":37245,"Ġgoose":37246,"Kay":37247,"Ġlofty":37248,"ĠWallet":37249,"Ġ['":37250,"Ġapocalypse":37251,"Ġbamboo":37252,"ĠSPACE":37253,"ĠElena":37254,"Ġ306":37255,"acons":37256,"Ġtightened":37257,"Ġadolescence":37258,"Ġrainy":37259,"Ġvandalism":37260,"ĠNewtown":37261,"Ġconject":37262,"cakes":37263,"Ġcheated":37264,"Ġmoderators":37265,"params":37266,"EFF":37267,"Ġdeceit":37268,"ĠSTL":37269,"ĠTanzania":37270,"ĠRI":37271,"Ġ1923":37272,"ĠExile":37273,"thel":37274,"Ġtheolog":37275,"Ġquirky":37276,"ĠIrvine":37277,"Ġneedy":37278,"oris":37279,"Um":37280,"Ka":37281,"Ġmailbox":37282,"322":37283,"Ġbos":37284,"ĠPetra":37285,"KING":37286,"Ġenlarged":37287,"Often":37288,"Ġbadass":37289,"Ġ343":37290,"ĠPlaces":37291,"ĠCAD":37292,"Ġpristine":37293,"Ġintervening":37294,"direction":37295,"Ġlaz":37296,"ĠDSM":37297,"Ġprojecting":37298,"ĠFunk":37299,"agog":37300,"payment":37301,"nov":37302,"Ġchatter":37303,"ARB":37304,"Ġexaminations":37305,"ĠHousehold":37306,"ĠGus":37307,"Ford":37308,"414":37309,"Boss":37310,"Ġmystic":37311,"Ġleaps":37312,"ĠBav":37313,"ulz":37314,"budget":37315,"Football":37316,"Ġsubsidized":37317,"Ġfirsthand":37318,"Ġcoincide":37319,"ocular":37320,"Conn":37321,"ĠCollabor":37322,"Ġfools":37323,"amura":37324,"ahar":37325,"rists":37326,"Ġswollen":37327,"Ġexpended":37328,"ĠPau":37329,"sup":37330,"Ġspar":37331,"Ġkeynote":37332,"suff":37333,"Ġunequal":37334,"Ġprogressing":37335,"strings":37336,"ĠGamergate":37337,"Disney":37338,"ĠEleven":37339,"omnia":37340,"Ġscripted":37341,"Ġearners":37342,"brother":37343,"ĠEnabled":37344,"æ³":37345,"Ġlarvae":37346,"ĠLOC":37347,"mess":37348,"Wilson":37349,"ĠTemplate":37350,"successfully":37351,"Ġparamount":37352,"Ġcamouflage":37353,"Ġbinds":37354,"ĠQuiet":37355,"ĠShutterstock":37356,"rush":37357,"Ġmascot":37358,"fortune":37359,"ĠColt":37360,"ĠBeyon":37361,"habi":37362,"Ġhairc":37363,"Ġ267":37364,"ĠDeus":37365,"Ġtwitch":37366,"Ġconcentrating":37367,"Ġnipples":37368,"cible":37369,"Ġgir":37370,"NZ":37371,"Math":37372,"nih":37373,"Required":37374,"Ġponder":37375,"ĠSAN":37376,"Ġweddings":37377,"Ġloneliness":37378,"NES":37379,"ĠMahjong":37380,"695":37381,"addle":37382,"ĠGarner":37383,"ĠCOUR":37384,"Bridge":37385,"Ġspree":37386,"ĠCaldwell":37387,"Ġbribery":37388,"Ġ��������":37389,"plugins":37390,"Ġracket":37391,"Ġchampagne":37392,"versible":37393,"Vote":37394,"Ġmodifiers":37395,"Mayor":37396,"680":37397,"Ġassemblies":37398,"ĠSultan":37399,"ĠNing":37400,"ĠLadies":37401,"Ġsulfur":37402,"Ġorbs":37403,"Ġ-----":37404,"_______":37405,"ĠJournalism":37406,"Ġesports":37407,"Ġlush":37408,"Ġhue":37409,"Ġspectral":37410,"Honest":37411,"ãĥı":37412,"Ġbushes":37413,"Ġreinforcement":37414,"Ġreopened":37415,"ĠWheels":37416,"ĠMorg":37417,"rieving":37418,"Ġauxiliary":37419,"ĠjQuery":37420,"ĠBAT":37421,"tesque":37422,"Ġvertex":37423,"pure":37424,"frey":37425,"ãĤº":37426,"dos":37427,"Ġtyph":37428,"Ġcull":37429,"Ġeq":37430,"Ġdecon":37431,"Ġtossing":37432,"Ġdisparate":37433,"ĠBrigham":37434,"printf":37435,"ledged":37436,"Ġsund":37437,"Ġcozy":37438,"Ġhepatitis":37439,"performing":37440,"Ġaval":37441,"ĠGG":37442,"future":37443,"Ġpetertodd":37444,"ĠKosovo":37445,"Ġmagnets":37446,"Already":37447,"ĠEdison":37448,"ĠCeres":37449,"ĠRAID":37450,"Ġbrilliance":37451,"576":37452,"Ġderives":37453,"Ġhypertension":37454,"ĠÎĶ":37455,"Ġlambda":37456,"Ġflair":37457,"Ġmissionaries":37458,"Ġrapes":37459,"ĠStarter":37460,"ĠMonths":37461,"Ġdefy":37462,"Ġseismic":37463,"ĠRaphael":37464,"Ġeurozone":37465,"656":37466,"zsche":37467,"Ġscratched":37468,"Ġbows":37469,"ĠLennon":37470,"ĠGaia":37471,"Ġdripping":37472,"facts":37473,"Ale":37474,"Ġfrogs":37475,"ĠBreast":37476,"ogeneity":37477,"ĠProsecutor":37478,"Ġamplified":37479,"ĠHodg":37480,"ĠFn":37481,"Thousands":37482,"ĠNIH":37483,"ĠMonitoring":37484,"FTWARE":37485,"ĠPriebus":37486,"ĠGrowing":37487,"hunter":37488,"Ġdiagnose":37489,"ĠMald":37490,"ĠLR":37491,"Ġcrowned":37492,"Ġbursting":37493,"Ġdissolution":37494,"javascript":37495,"Ġusefulness":37496,"ĠExecution":37497,":(":37498,"ĠIvory":37499,"aah":37500,"Ġpersecuted":37501,"violence":37502,"istas":37503,"ĠCrate":37504,"Ġimpulses":37505,"ĠSpani":37506,"edes":37507,"Handle":37508,"ĠZerg":37509,"thinkable":37510,"Lastly":37511,"Ġspontaneously":37512,"Ġinconvenient":37513,"Ġdismissing":37514,"Ġplotted":37515,"Ġeighty":37516,"Ġ737":37517,"rish":37518,"ĠThornton":37519,"atham":37520,"Ġsitcom":37521,"Ven":37522,"Recipe":37523,"tel":37524,"lund":37525,"Ġclears":37526,"ĠSasuke":37527,"Ġ258":37528,"Ġopting":37529,"Ġenraged":37530,"esthetic":37531,"ĠAe":37532,"uchs":37533,"Prep":37534,"Flow":37535,"Ġrunoff":37536,"ĠEating":37537,"ĠGiles":37538,"ĠActing":37539,"resources":37540,"ibaba":37541,"Ġrpm":37542,"Ġskewed":37543,"ĠBlanc":37544,"ĠSakuya":37545,"Ġhotter":37546,"Ġ1924":37547,"opian":37548,"cko":37549,"Ġcrumbling":37550,"Ġcaptains":37551,"ĠAppropriations":37552,"leaders":37553,"dropping":37554,"anuts":37555,"Ġreversing":37556,"ĠPose":37557,"ĠSek":37558,"Scot":37559,"ĠIdea":37560,"cise":37561,"ĠSlovenia":37562,"Ġ317":37563,"Doctor":37564,"Ġcrocod":37565,"aldi":37566,"Sea":37567,"ĠFarrell":37568,"Ġmercenaries":37569,"ĠRNC":37570,"ĠGuess":37571,"Ġpacing":37572,"Machine":37573,"StreamerBot":37574,"ĠCharity":37575,"Ġ298":37576,"Ġcannons":37577,"ĠToby":37578,"TPPStreamerBot":37579,"ĠPassion":37580,"cfg":37581,"Thom":37582,"Ġbadges":37583,"ĠBernstein":37584,".âĢĵ":37585,"ĠPOP":37586,"ĠConj":37587,"Ġinitialization":37588,"Ġbiodiversity":37589,"Dub":37590,"Ġfeudal":37591,"Ġdisclaimer":37592,"Ġcrow":37593,"Ġignition":37594,"arf":37595,"SHA":37596,"ĠkHz":37597,"hazard":37598,"ĠArtists":37599,"oeuv":37600,"679":37601,"ĠRudy":37602,"Nine":37603,"ĠRamadan":37604,"å½":37605,"itto":37606,"Ġadrenaline":37607,"Cert":37608,"Ġsmelled":37609,"Ġimpunity":37610,"Ġagendas":37611,"ĠReborn":37612,"ĠConcent":37613,"ĠSeems":37614,"Ġomega":37615,"ĠDustin":37616,"Ġbacker":37617,"ĠSauce":37618,"ĠBoyle":37619,"WIN":37620,"Ġspins":37621,"Ġpauses":37622,"upt":37623,"Ġshredded":37624,"Ġstrapped":37625,"ĠCorruption":37626,"Ġscratches":37627,"Ġni":37628,"Ġattire":37629,"ĠSAF":37630,"FactoryReloaded":37631,"ĠIPS":37632,"Ġ(%":37633,"Ġseminar":37634,"focus":37635,"civil":37636,"Ġ1860":37637,"intosh":37638,"Ġcontinual":37639,"Ġabbrevi":37640,"ĠSok":37641,"ocobo":37642,"XM":37643,"Ġfrantic":37644,"Ġunavoidable":37645,"Ġartery":37646,"Ġannotations":37647,"bath":37648,"Climate":37649,"Ġdors":37650,"ĠSlide":37651,"coord":37652,"ĠReload":37653,"ĠLDL":37654,"ĠLovecraft":37655,"Ġunimagin":37656,"Ġresembled":37657,"Ġbarracks":37658,"np":37659,"Ġsurrogate":37660,"Ġcategorized":37661,"ãĤ©":37662,"Ġvaccinated":37663,"Ġdrainage":37664,"Ġindist":37665,"ĠWhatsApp":37666,"Ġ1870":37667,"olerance":37668,"invoke":37669,"amorph":37670,"Ġreconnect":37671,"Ġemanc":37672,"Ġblindness":37673,"Ġ1280":37674,"internet":37675,"collar":37676,"Ġaltru":37677,"Ġabyss":37678,"ĠTRI":37679,"657":37680,"Ġinfused":37681,"HEAD":37682,"Ġforestry":37683,"ĠWoody":37684,"ĠCi":37685,"wi":37686,"sam":37687,"784":37688,"holiday":37689,"Ġmogul":37690,"ĠFees":37691,"ĠDEN":37692,"Internal":37693,"urbed":37694,"fusc":37695,"atom":37696,"ĠIllusion":37697,"Ġpolled":37698,"Ġflap":37699,"Ġcoax":37700,"LGBT":37701,"Analy":37702,"ĠSections":37703,"ĠCaliforn":37704,"emn":37705,"Ġhither":37706,"ĠNIGHT":37707,"Ġnailed":37708,"ĠPipeline":37709,"391":37710,"oof":37711,"ĠPrimal":37712,"verend":37713,"Ġslashing":37714,"Ġretri":37715,"aviour":37716,"Ġdeparting":37717,"gil":37718,"ISC":37719,"Ġmidway":37720,"Ġultrasound":37721,"Ġbehaving":37722,"ĠTara":37723,"classes":37724,"Virtual":37725,"ĠColonial":37726,"Ġstripping":37727,"Ġorchestrated":37728,"ĠGraves":37729,"452":37730,"ĠIronically":37731,"ĠWriters":37732,"Ġlends":37733,"ĠManz":37734,"Ġraven":37735,"Ġoxidative":37736,"Ġ266":37737,"ELF":37738,"actually":37739,"ascar":37740,"Draft":37741,"Ġfavourable":37742,"Ġhumiliating":37743,"Ġfidelity":37744,"ĠHof":37745,"ĠXuan":37746,"496":37747,"Ġlayered":37748,"atis":37749,"790":37750,"Ġpaycheck":37751,"iton":37752,"Kar":37753,"ĠVMware":37754,"ĠFarmer":37755,"Ġservic":37756,"glomer":37757,"Ġslump":37758,"ĠFabric":37759,"ĠDOC":37760,"esting":37761,"Ġreassure":37762,"Ġphyl":37763,"volt":37764,"itory":37765,"Rules":37766,"Ġoxidation":37767,"Ġprized":37768,"Ġmistress":37769,"ĠDjango":37770,"WARN":37771,"åij":37772,"Ġencode":37773,"ĠFeedback":37774,"Ġstupidity":37775,"Ian":37776,"ĠYugoslavia":37777,"ר":37778,"acl":37779,"UTE":37780,"1977":37781,"Ġqualifies":37782,"Ġpulses":37783,"pretty":37784,"Ġfroze":37785,"Ġss":37786,"Iterator":37787,"Ġurgently":37788,"Ġmailed":37789,"ĠCham":37790,"Ġsustaining":37791,"Ġbasil":37792,"Ġpuppies":37793,"ilant":37794,"ĠPLEASE":37795,"lap":37796,"aceous":37797,"Fear":37798,"ĠMastery":37799,"automatic":37800,"ĠTAG":37801,"Ġantim":37802,"agles":37803,"473":37804,"frames":37805,"Ġwhispers":37806,"ĠWhoever":37807,"Ġbravery":37808,"ĠUKIP":37809,"ractions":37810,"\"\"\"":37811,"Ġtame":37812,"Ġparted":37813,"everything":37814,"CONT":37815,"Ġindebted":37816,"Ġaddr":37817,"rek":37818,"IRED":37819,"Ġeminent":37820,"clinton":37821,"Ġousted":37822,"Ġreviewer":37823,"Ġmeltdown":37824,"Ġrearr":37825,"ĠYao":37826,"thereal":37827,"abyte":37828,"Ġstumbling":37829,"Ġbatches":37830,"Ġ259":37831,"Ġcontraceptive":37832,"Ġprostitute":37833,"ensis":37834,"Decl":37835,"ĠStrikes":37836,"Military":37837,"ĠOath":37838,"vacc":37839,"ppings":37840,"052":37841,"ĠpartName":37842,"amping":37843,"Reports":37844,"KI":37845,"CHR":37846,"Ġsubtly":37847,"swers":37848,"Blake":37849,"usual":37850,"Ġcontestants":37851,"Ġcartridges":37852,"ĠGREAT":37853,"Ġblush":37854,"ĠâĢº":37855,"472":37856,"Ġreasoned":37857,"ãĥ¤":37858,"paralleled":37859,"Ġdyn":37860,"agate":37861,"Ġnightly":37862,"åĨ":37863,"556":37864,"Ġsemantic":37865,"ĠAdvoc":37866,"Ġ!!":37867,"Ġdisagrees":37868,"ĠBW":37869,"Veh":37870,"Ġharming":37871,"Ġembraces":37872,"Ġstrives":37873,"Ġinland":37874,"ĠKard":37875,"Ġheats":37876,"ĠGinny":37877,"utan":37878,"ernaut":37879,"ylene":37880,"ĠElev":37881,"JD":37882,"Ġhars":37883,"ĠStarr":37884,"Ġskysc":37885,"Ġcollaborators":37886,"Usually":37887,"Ġrevolutions":37888,"ĠSTATS":37889,"Ġdismantle":37890,"Ġconfidently":37891,"Ġkinetic":37892,"Ali":37893,"Ġpercentile":37894,"Ġextracting":37895,"illian":37896,"estead":37897,"Ġphysicists":37898,"ĠMarshal":37899,"Ġfellowship":37900,"Ġdashed":37901,"ĠUR":37902,"ĠSioux":37903,"ĠCompact":37904,"amide":37905,"Python":37906,"ĠLeigh":37907,"ĠPharmac":37908,"istrates":37909,"herical":37910,"Ġfue":37911,"ĠEmin":37912,"Ġ({":37913,"ĠNeighborhood":37914,"Ġdisrupting":37915,"ĠDup":37916,"Ġgland":37917,"ĠSev":37918,"ĠMarian":37919,"argon":37920,"ĠDund":37921,"Ġ":46904,"ĠPhilips":46905,"ĠKafka":46906,"Ġupheaval":46907,"Ġsentimental":46908,"Ġsax":46909,"ĠAkira":46910,"serial":46911,"Matrix":46912,"Ġelecting":46913,"Ġcommenter":46914,"ĠNebula":46915,"plets":46916,"ĠNadu":46917,"ĠAdren":46918,"Ġenshr":46919,"ĠRAND":46920,"financial":46921,"ĠClyde":46922,"utherford":46923,"Ġsignage":46924,"Ġdeline":46925,"Ġphosphate":46926,"roversial":46927,"fascist":46928,"ĠVall":46929,"ĠBethlehem":46930,"Ġfors":46931,"Ġenglish":46932,"Solid":46933,"Nature":46934,"Ġva":46935,"ĠGuests":46936,"Ġtantal":46937,"Ġautoimmune":46938,";;;;;;;;;;;;":46939,"ĠTotally":46940,"ĠOv":46941,"Ġdefences":46942,"ĠCoconut":46943,"Ġtranquil":46944,"Ġploy":46945,"Ġflavours":46946,"ĠFlask":46947,"ãĤ¨ãĥ«":46948,"ĠWeston":46949,"ĠVolvo":46950,"870":46951,"Ġmicrophones":46952,"verbal":46953,"RPG":46954,"Ġiii":46955,";}":46956,"028":46957,"Ġheadlined":46958,"Ġprimed":46959,"Ġhoard":46960,"ĠShad":46961,"ĠENTER":46962,"Ġtriangular":46963,"Ġcapit":46964,"lik":46965,"ĠAncients":46966,"Ġlash":46967,"Ġconvol":46968,"Ġcolonel":46969,"enemy":46970,"Gra":46971,"Ġpubs":46972,"utters":46973,"Ġassigns":46974,"ĠPenet":46975,"ĠMonstrous":46976,"ĠBowen":46977,"ilver":46978,"Haunted":46979,"ĠDing":46980,"started":46981,"plin":46982,"Ġcontaminants":46983,"ĠDOE":46984,"ffen":46985,"ĠTechnician":46986,"Ry":46987,"Ġrobbers":46988,"Ġhotline":46989,"ĠGuardiola":46990,"ĠKaufman":46991,"rower":46992,"ĠDresden":46993,"ĠAlpine":46994,"Elf":46995,"Ġfmt":46996,"ĠSard":46997,"urses":46998,"gpu":46999,"Unix":47000,"Ġunequivocally":47001,"ĠCitizenship":47002,"quad":47003,"mire":47004,"ĠSweeney":47005,"Battery":47006,"615":47007,"Ġpancakes":47008,"Ġoats":47009,"Maps":47010,"ĠContrast":47011,"mbudsman":47012,"ĠEPS":47013,"Ġsubcommittee":47014,"Ġsourcing":47015,"Ġsizing":47016,"ĠBuffer":47017,"ĠMandatory":47018,"Ġmoderates":47019,"ĠPatterns":47020,"ĠChocobo":47021,"ĠZan":47022,"ĠSTATES":47023,"ĠJudging":47024,"ĠInher":47025,"*:":47026,"Ġbil":47027,"ĠYen":47028,"Ġexhilar":47029,"ollower":47030,"zers":47031,"Ġsnug":47032,"maximum":47033,"Ġdespicable":47034,"ĠPACK":47035,"ĠAnnex":47036,"Ġsarcastic":47037,"Ġlatex":47038,"Ġtamp":47039,"ĠSao":47040,"bah":47041,"ĠReverend":47042,"ĠChinatown":47043,"ĠAUT":47044,"documented":47045,"ĠGABA":47046,"ĠCanaan":47047,"ĠÙħ":47048,"Ġgoverns":47049,"prev":47050,"Esc":47051,"ĠEstimates":47052,"OSP":47053,"Ġendeavour":47054,"ĠClosing":47055,"ometime":47056,"everyone":47057,"Ġworsen":47058,"Ġscanners":47059,"Ġdeviations":47060,"ĠRobotics":47061,"ĠCompton":47062,"Ġsorcerer":47063,"Ġendogenous":47064,"Ġemulation":47065,"ĠPiercing":47066,"ĠAph":47067,"ĠSocket":47068,"Ġbould":47069,"ĠOU":47070,"ĠBorderlands":47071,"Ġ1863":47072,"Gordon":47073,"ĠWTO":47074,"Ġrestricts":47075,"Ġmosaic":47076,"Ġmelodies":47077,"çĦ":47078,"Tar":47079,"Ġdisson":47080,"ĠProvides":47081,"Ġ......":47082,"bek":47083,"FIX":47084,"Ġbroom":47085,"anship":47086,"Doctors":47087,"Ġnerds":47088,"ĠRegions":47089,"naissance":47090,"Ġmete":47091,"Ġcrept":47092,"plings":47093,"Ġgirlfriends":47094,"knit":47095,"igent":47096,"owe":47097,"Ġushered":47098,"ĠBaz":47099,"Mobil":47100,"434":47101,"ĠPresents":47102,"origin":47103,"Ġinsomnia":47104,"ĠAux":47105,"439":47106,"ĠChili":47107,"irsch":47108,"GAME":47109,"Ġgestation":47110,"algia":47111,"romising":47112,"$,":47113,"crow":47114,"ĠInspection":47115,"atomic":47116,"Relations":47117,"JOHN":47118,"roman":47119,"ĠClockwork":47120,"ĠBakr":47121,"mone":47122,"MET":47123,"Ġthirsty":47124,"Ġbc":47125,"Ġfaculties":47126,"Rum":47127,"Ġnuance":47128,"ĠDarius":47129,"pleting":47130,"fters":47131,"etchup":47132,"Registration":47133,"ĠKE":47134,"Rah":47135,"Ġpreferential":47136,"ĠLash":47137,"ĠHH":47138,"Valid":47139,"ĠNAV":47140,"Ġstarve":47141,"ĠGong":47142,"zynski":47143,"ĠActress":47144,"Ġwik":47145,"Ġunaccompanied":47146,"lvl":47147,"Bride":47148,"ADS":47149,"ĠCommando":47150,"ĠVaughn":47151,"Wallet":47152,"Ġhopping":47153,"ĠVie":47154,"Ġcaveats":47155,"Ġalas":47156,"ifled":47157,"abuse":47158,"661":47159,"Ġibn":47160,"Ġgul":47161,"Ġrobbing":47162,"til":47163,"ILA":47164,"Ġmitigating":47165,"Ġaptly":47166,"Ġtyrant":47167,"Ġmidday":47168,"ĠGilmore":47169,"ĠDecker":47170,"Ġ§§":47171,"partial":47172,"Exactly":47173,"Ġphenotype":47174,"Ġ[+]":47175,"ĠPlex":47176,"ĠIps":47177,"versions":47178,"Ġebook":47179,"Ġchic":47180,"gross":47181,"\":\"\"},{\"":47182,"ĠSurprisingly":47183,"Morgan":47184,"Ġresidues":47185,"ĠConfederation":47186,"infeld":47187,"Ġlyr":47188,"moderate":47189,"Ġperpendicular":47190,"VK":47191,"Ġsynchronized":47192,"Ġrefreshed":47193,"Ġadore":47194,"ĠTorment":47195,"olina":47196,"Ġ2600":47197,"ItemTracker":47198,"Ġpies":47199,"ĠFAT":47200,"ĠRHP":47201,"048":47202,"ĠRESP":47203,"ĠBJ":47204,"allows":47205,"Pand":47206,"Ġunwelcome":47207,"ĠVoc":47208,"ĠBastard":47209,"ĠOW":47210,"ĠLAR":47211,"ĠHealer":47212,"Environmental":47213,"ĠKenyan":47214,"ĠTrance":47215,"ĠPats":47216,"Ġaliases":47217,"ĠGarfield":47218,"Ġcampaigner":47219,"Ġadvancements":47220,"ĠOkinawa":47221,"ĠCoh":47222,"owsky":47223,"Ġstarved":47224,"Ġsizeable":47225,"Ġ:-)":47226,"ĠmRNA":47227,"Ġsuspensions":47228,"istar":47229,"Scotland":47230,"Prin":47231,"------------------------------------------------":47232,"Ġ502":47233,"Ġteaspoons":47234,"Ġ1050":47235,"Ġcoercive":47236,"ĠMasonic":47237,"edded":47238,"ĠPassenger":47239,"Ġlatt":47240,"Ġbraces":47241,"ĠSteal":47242,"ĠNYT":47243,"ĠKats":47244,"ĠCelest":47245,"aez":47246,"Tu":47247,"ĠCoulter":47248,"ðŁĺ":47249,"Flickr":47250,"ĠWilmington":47251,"iths":47252,"++;":47253,"Ġvending":47254,"Ġnegro":47255,"ĠPhi":47256,"ĠYellowstone":47257,"Callback":47258,"Ġshampoo":47259,"ĠShades":47260,"wat":47261,"Ġsuperhuman":47262,"Ġridiculed":47263,"Ġholiest":47264,"ombo":47265,"Ġinterns":47266,"Ġhone":47267,"ĠParagu":47268,"URI":47269,"Ġdangling":47270,"ãĤ»":47271,"sov":47272,"ictional":47273,"availability":47274,"Ġrevocation":47275,"Ġdow":47276,"inic":47277,"ĠTHEIR":47278,"Ġiso":47279,"Ġoutings":47280,"ĠLethal":47281,"Ġ)))":47282,"Ġinaccur":47283,"Ġoutlandish":47284,"Ġanus":47285,"letico":47286,"idon":47287,"lol":47288,"Ġunregulated":47289,"Ġsuccumbed":47290,"Ġcuff":47291,"ĠWasteland":47292,"letal":47293,"Ġsubstr":47294,"Ġcoffers":47295,"Ġautomakers":47296,"ovi":47297,"ĠXue":47298,"ĠDaytona":47299,"Ġjarring":47300,"Ġfumes":47301,"Ġdisbanded":47302,"zik":47303,"itton":47304,"Ġstrikingly":47305,"Ġspores":47306,"Adapter":47307,".):":47308,"ĠLyndon":47309,"ivalry":47310,"Ġorally":47311,"Ġtumultuous":47312,"Ġdispleasure":47313,"Ġcones":47314,"orrect":47315,"Ġappease":47316,"Ġderby":47317,"ĠTripoli":47318,"ĠAless":47319,"Ġpoked":47320,"ĠGuilty":47321,"vP":47322,"Enough":47323,"Ġoriginals":47324,"699":47325,"Ġrabbi":47326,"Ġproverbial":47327,"Ġpostpone":47328,"elope":47329,"ĠMisty":47330,"Ġstaffed":47331,"ĠUnemployment":47332,"reditary":47333,"Ġdiligent":47334,"recomm":47335,"measures":47336,"asin":47337,"825":47338,"Ġponds":47339,"Ġmmol":47340,"ĠSAR":47341,"ĠCARE":47342,"Ġ371":47343,"Ġclenched":47344,"ĠCorsair":47345,"Ġcaricature":47346,"zn":47347,"attach":47348,"ĠSchro":47349,"speak":47350,"painted":47351,"ĠSuc":47352,"ĠENT":47353,"Ġcellul":47354,"ĠPaid":47355,"diagn":47356,"WHERE":47357,"Ġtexted":47358,"Barn":47359,"Ġretracted":47360,"ĠReferred":47361,"Sav":47362,"Ġupkeep":47363,"Ġworkplaces":47364,"ĠTokens":47365,"Ġamplify":47366,"clinical":47367,"Ġmultic":47368,"mberg":47369,"Ġconvoluted":47370,"Region":47371,"565":47372,"ĠTopic":47373,"Ġsnail":47374,"Ġsaline":47375,"Ġinsurrection":47376,"ĠPetr":47377,"forts":47378,"BAT":47379,"ĠNavajo":47380,"Ġrudimentary":47381,"ĠLaksh":47382,"ONDON":47383,"Measure":47384,"Ġtransformer":47385,"ĠGoddard":47386,"Ġcoincides":47387,"irin":47388,"Rex":47389,"ĠBok":47390,"quit":47391,"Ġshotguns":47392,"Ġproletarian":47393,"Ġscorp":47394,"ĠAda":47395,"514":47396,"Ġslander":47397,"recorded":47398,"Ġembell":47399,"risome":47400,"Ġapologizing":47401,"ĠMulcair":47402,"ĠGibraltar":47403,"Cla":47404,"Ġallot":47405,"ĠAttention":47406,"Ġ433":47407,"leave":47408,"Ġwhine":47409,"ĠIssa":47410,"ĠFaust":47411,"ĠBarron":47412,"heny":47413,"Ġvictimized":47414,"Jews":47415,"Ġnurturing":47416,"ettel":47417,"Winged":47418,"ĠSubtle":47419,"Ġflavorful":47420,"ĠReps":47421,"enged":47422,"callback":47423,"Ġdirectional":47424,"Ġclasp":47425,"ĠDirections":47426,"planet":47427,"iculture":47428,"Helper":47429,"icion":47430,"acia":47431,"Ġç¥ŀ":47432,"Ġsurges":47433,"Ġcanoe":47434,"ĠPremiership":47435,"been":47436,"Ġdefied":47437,"ĠTrooper":47438,"Ġtripod":47439,"Ġgasp":47440,"ĠEuph":47441,"ĠAds":47442,"vernight":47443,"highly":47444,"Role":47445,"Ġentangled":47446,"ĠZeit":47447,"618":47448,"ĠRusty":47449,"Ġhavens":47450,"ĠVaughan":47451,"HAEL":47452,"ĠSERVICE":47453,"/,":47454,"Ġstricken":47455,"Ġdelusions":47456,"Ġbis":47457,"ĠHaf":47458,"Ġgratification":47459,"Ġenticing":47460,"UNCH":47461,"Adams":47462,"ĠOLED":47463,"ĠBeetle":47464,"Ġ1899":47465,"ĠSOFTWARE":47466,"ategor":47467,"VL":47468,"ĠTotem":47469,"ĠGators":47470,"ATURES":47471,"Ġimpedance":47472,"Registered":47473,"ĠCary":47474,"ĠAerial":47475,"onne":47476,"enium":47477,"Ġdred":47478,"ĠBeg":47479,"Ġconcurrently":47480,"Ġsuperpower":47481,"ĠXan":47482,"jew":47483,"imester":47484,"ĠDickinson":47485,"âĶģ":47486,"Fla":47487,"Ġpree":47488,"ĠRollins":47489,"©¶æ":47490,"Ġdenomination":47491,"ĠLana":47492,"516":47493,"Ġinciting":47494,"scribed":47495,"juries":47496,"ĠWonders":47497,"approximately":47498,"Ġsuspending":47499,"Ġmountainous":47500,"ĠLaugh":47501,"oidal":47502,"Ns":47503,"Detect":47504,")=":47505,"ĠLuthor":47506,"ĠSchwarzenegger":47507,"ĠMuller":47508,"ĠDevi":47509,"ecycle":47510,"Jar":47511,"613":47512,"ĠLongh":47513,"Bah":47514,"ĠSPORTS":47515,"nw":47516,"Ġrefinement":47517,"Ġwaterways":47518,"Ġdiner":47519,"Blade":47520,"683":47521,"Fac":47522,"Ġinitials":47523,"Ġrog":47524,"Ġparanormal":47525,"BUT":47526,"Ġ[(":47527,"ĠSwanson":47528,"ĠMesh":47529,"âĸ¬":47530,"Improve":47531,"ĠRadiation":47532,"ĠEsther":47533,"ĠEsk":47534,"ĠAly":47535,"iky":47536,"Ġirrad":47537,"ĠBuckingham":47538,"Ġrefill":47539,"Ġ._":47540,"Repe":47541,"CONCLUS":47542,"Ġdifferentiated":47543,"Ġchirop":47544,"ĠAtkins":47545,"Pattern":47546,"Ġexcise":47547,"Ġcabal":47548,"NSA":47549,"ĠSTA":47550,"ĠSIL":47551,"ĠParaly":47552,"Ġrye":47553,"ĠHowell":47554,"ĠCountdown":47555,"nesses":47556,"alysed":47557,"Ġresize":47558,"ãĤ½":47559,"Ġbudgetary":47560,"ĠStras":47561,"wang":47562,"Ġapiece":47563,"Ġprecincts":47564,"Ġpeach":47565,"Ġskyline":47566,"Ġ353":47567,"popular":47568,"Appearances":47569,"ĠMechanics":47570,"ĠDevOnline":47571,"Sullivan":47572,"Zen":47573,"Ġpu":47574,"opolis":47575,"544":47576,"Ġdeform":47577,"Ġcounteract":47578,"ĠLange":47579,"Ġ417":47580,"Console":47581,"774":47582,"Ġnodding":47583,"Ġpopulism":47584,"Ġhep":47585,"Ġcounselling":47586,"compliance":47587,"UFF":47588,"Ġundeniably":47589,"Ġrailing":47590,"ĠHorowitz":47591,"ĠSimone":47592,"ĠBungie":47593,"Ġak":47594,"ĠTalks":47595,"xff":47596,"flake":47597,"Crash":47598,"Ġsweaty":47599,"Ġbanquet":47600,"ĠOFFIC":47601,"Ġinventive":47602,"Ġastronomer":47603,"ĠStamford":47604,"ĠScare":47605,"ĠGREEN":47606,"olicited":47607,"Ġrusher":47608,"Ġcentrist":47609,"ighting":47610,"Ġsubclass":47611,"Ġdisav":47612,"Ġdefund":47613,"ĠNanto":47614,"ociate":47615,"mast":47616,"Ġpacif":47617,"Ġmend":47618,"eers":47619,"immigration":47620,"ESSION":47621,"Ġnumbering":47622,"Ġlaughable":47623,"ĠEnded":47624,"viation":47625,"emark":47626,"Pitt":47627,"Ġmeticulous":47628,"ĠLF":47629,"Ġcongratulated":47630,"ĠBirch":47631,"Ġswayed":47632,"Ġsemifinals":47633,"Ġhumankind":47634,"matter":47635,"ĠEquip":47636,"opausal":47637,"Said":47638,"ĠLayout":47639,"Ġvoicing":47640,"Ġthug":47641,"Ġpornographic":47642,"IPS":47643,"Ġmoaning":47644,"Ġgrievance":47645,"Ġconfessions":47646,"escal":47647,"TEXTURE":47648,"Authent":47649,"osaurus":47650,"Purchase":47651,"Ġrelegation":47652,"alter":47653,"Ġ³³":47654,"Ġriddled":47655,"Ġogre":47656,"ĠLowell":47657,"Occup":47658,"Eat":47659,"ĠHyder":47660,"ĠAdviser":47661,"Commerce":47662,"Hunt":47663,"ĠOrth":47664,"ĠCompetitive":47665,"ĠCLA":47666,"CDC":47667,"Ġsalads":47668,"Fle":47669,"Ġindustrialized":47670,"`,":47671,"ĠOWN":47672,"Ġbeck":47673,"ĠParticularly":47674,"oubt":47675,"ĠmM":47676,"ĠHussain":47677,"ĠChennai":47678,"Ġ920":47679,"Ġappointing":47680,"ĠCullen":47681,",,,,,,,,":47682,"Ġpores":47683,"verified":47684,"Ġbiochemical":47685,"emate":47686,"Ġcowardly":47687,"ĠHelsinki":47688,"ĠEthiopian":47689,"SOURCE":47690,"ERC":47691,"estro":47692,"Ġbiotech":47693,"ĠSour":47694,"Ġbrewer":47695,"Bloomberg":47696,"Ġintensify":47697,"Glass":47698,"anco":47699,"ĠFDR":47700,"greSQL":47701,"ĠFires":47702,"©¶æ¥µ":47703,"eco":47704,"1001":47705,"ĠHomeless":47706,"Ġinstantaneous":47707,"ĠHaste":47708,"igel":47709,"Diamond":47710,"Ġpaving":47711,"Ġlandfill":47712,"Ġdads":47713,"houn":47714,":]":47715,"Ġincendiary":47716,"ĠLivingston":47717,"ĠHilbert":47718,"ĠChecks":47719,"styles":47720,"inators":47721,"ĠClive":47722,"phrine":47723,"Ġchimpanzees":47724,"Ġpall":47725,"ĠJM":47726,"ĠAadhaar":47727,"ðĿ":47728,"Ġachievable":47729,"disabled":47730,"PET":47731,"OOOOOOOO":47732,"Mot":47733,"Ġintangible":47734,"Ġballet":47735,"ĠWebs":47736,"ĠEstimated":47737,"Effects":47738,"Ġbailed":47739,"Joshua":47740,"Ġturbulence":47741,"Ġoccupant":47742,"ĠDaylight":47743,"Ġ361":47744,"meet":47745,"Ġstatically":47746,"Ġonlook":47747,"Ġki":47748,"illegal":47749,"Ġvelvet":47750,"Ġdehydration":47751,"Ġacquies":47752,"ĠRez":47753,"akura":47754,"ĠUpton":47755,"atro":47756,"Ġincomprehensible":47757,"Ġbackdoor":47758,"ĠRhino":47759,"727":47760,"Ġmaths":47761,")+":47762,"Ġheresy":47763,"Ġdf":47764,"ĠRoche":47765,"ĠLydia":47766,"Ġpancreat":47767,"reply":47768,"arrell":47769,"Ġsolicitation":47770,"Ġcircadian":47771,"BIP":47772,"Ġforay":47773,"Ġcryptic":47774,"izu":47775,"imeo":47776,"ĠTomato":47777,"ĠHoms":47778,"examination":47779,"Ġquarry":47780,"ĠValiant":47781,"ĠJericho":47782,"ĠINCLUD":47783,"Ġ1840":47784,"519":47785,"Ġresists":47786,"Ġsnapshots":47787,"ĠSpur":47788,"ĠAntiqu":47789,"Login":47790,"Ġbestselling":47791,"Ġantic":47792,"ĠSutherland":47793,"ãĤ¢ãĥ«":47794,"Ġ~/":47795,"ĠParm":47796,"èĥ":47797,"Pages":47798,"intensity":47799,"Ġimmobil":47800,"Ġ1865":47801,"zzo":47802,"Ġnifty":47803,"Ġfentanyl":47804,"ĠPreservation":47805,"ophen":47806,"Ġdarts":47807,"ĠDinosaur":47808,"pointers":47809,"ĠRite":47810,"suggest":47811,"awareness":47812,"ĠSheridan":47813,"Ġstances":47814,"Ġsorcery":47815,"Ġperjury":47816,"ĠNikola":47817,"iever":47818,"Ġfiance":47819,"ĠJordanian":47820,"ĠBalloon":47821,"Ġnab":47822,"Ġkb":47823,"Ġhumanities":47824,"ĠTanaka":47825,"hillary":47826,"Ġconsultancy":47827,"ĠZub":47828,"Ġremission":47829,"Ġconfid":47830,"CHQ":47831,"ĠFug":47832,"Ġimprovis":47833,"Yep":47834,"/_":47835,"Ġunwillingness":47836,"Ġportfolios":47837,"055":47838,"ĠInstructor":47839,"aiman":47840,"Ġclaimants":47841,"Mbps":47842,"ĠBye":47843,"received":47844,"Tweet":47845,"Ġindemn":47846,"riz":47847,"amara":47848,"Nat":47849,"Ġevaluates":47850,"ĠLur":47851,"epad":47852,"FOX":47853,"ĠThro":47854,"Ġrusty":47855,"Ġbedrock":47856,"ĠOprah":47857,"JB":47858,"Ġmanipulative":47859,"Ġwillful":47860,"Ġrelapse":47861,"Ġextant":47862,"Theme":47863,"Sensor":47864,"ĠStability":47865,"govern":47866,"Ġpoppy":47867,"Ġknack":47868,"Ġinsulated":47869,"ĠTile":47870,"ĠExtrem":47871,"Ġuntold":47872,"Ġconverge":47873,"Ġrefuel":47874,"igroup":47875,"Ġdistortions":47876,"Ġravaged":47877,"Ġmechanically":47878,"ĠReilly":47879,"ĠNose":47880,"ĠIncarnation":47881,"ĠBecky":47882,"abbling":47883,"Ġtaco":47884,"Ġrake":47885,"Ġmelancholy":47886,"Ġillustrious":47887,"ĠDartmouth":47888,"Guide":47889,"ĠRazer":47890,"ĠBenz":47891,"Ultimate":47892,"ĠSurprise":47893,"Ġpageant":47894,"offer":47895,"Whoever":47896,"Ġwiser":47897,"Ġchemist":47898,"ĠHELL":47899,"ĠBulk":47900,"Ġplutonium":47901,"ĠCOVER":47902,"Ö¼":47903,"failed":47904,"Ġtirelessly":47905,"Ġinfertility":47906,"ĠTrident":47907,"ĠShowtime":47908,"ĠCiv":47909,"Vice":47910,"requires":47911,"ittance":47912,"Ġuncontrolled":47913,"interesting":47914,"561":47915,"Ġinnovate":47916,"ategic":47917,"Lie":47918,"ĠSelling":47919,"Ul":47920,"Ġsavior":47921,"ĠTosh":47922,"Ġswast":47923,"PASS":47924,"Ġrink":47925,"Ġcardio":47926,"ĠIro":47927,"udi":47928,"Ġvantage":47929,"Ġvans":47930,"ĠNiño":47931,"+=":47932,"Ġpropagate":47933,"":49029,"Ġleukemia":49030,"Ġeluc":49031,"Ġannouncer":49032,"ĠLithuan":49033,"ĠArmageddon":49034,"åĩ":49035,"Lenin":49036,"ĠRuk":49037,"Ġpepp":49038,"ĠRomantic":49039,"ĠPIT":49040,"ĠInterstellar":49041,"ĠAtkinson":49042,"Raid":49043,"Js":49044,"Goal":49045,"Course":49046,"Ġvanishing":49047,"esley":49048,"ĠRounds":49049,"Elsa":49050,"593":49051,"Ġredundancy":49052,"ĠSTAND":49053,"Ġprophetic":49054,"Ġhabitable":49055,"ryu":49056,"Ġfaintly":49057,"MODE":49058,"Ġflanked":49059,"IRC":49060,"Awesome":49061,"Ġspurious":49062,"ĠZah":49063,"ĠMSG":49064,"Ġshading":49065,"Ġmotivational":49066,"ĠSantana":49067,"ĠSPR":49068,"Ġexcruciating":49069,"omial":49070,"ĠMiko":49071,"ĠLeopard":49072,"Abyss":49073,"Ġ[|":49074,"dirty":49075,"Ġbaths":49076,"Ġdemoral":49077,"andre":49078,"PB":49079,"Ġunification":49080,"Ġsacrament":49081,"Ġ[&":49082,"Ġpriceless":49083,"Ġgelatin":49084,"Ġemanating":49085,"ĠAllaah":49086,"986":49087,"Ġoutburst":49088,"Ġeras":49089,"ĠXVI":49090,"ĠSPI":49091,"Ott":49092,"ĠLazarus":49093,"PLIED":49094,"Flying":49095,"blogs":49096,"Wisconsin":49097,"Raven":49098,"Ġrebate":49099,"Ġcreeps":49100,"ĠSpan":49101,"ĠPainter":49102,"ĠKira":49103,"ĠAmos":49104,"ĠCorvette":49105,"Consumer":49106,"ĠRecover":49107,"cki":49108,"Ġpesky":49109,"ĠInvention":49110,"Companies":49111,"Ġchallengers":49112,"ademic":49113,"ĠUkrainians":49114,"ĠNeurolog":49115,"ĠForsaken":49116,"Ġentrants":49117,"Ġembattled":49118,"Ġdefunct":49119,"ĠGlacier":49120,"Ġpoisons":49121,"ĠHorses":49122,"makes":49123,"ĠDirt":49124,"Ġ423":49125,"hhh":49126,"ĠTransformation":49127,"QUIRE":49128,"..................":49129,"Ġtraveller":49130,"ĠSexy":49131,"ĠKern":49132,"ipolar":49133,"Ġransomware":49134,"oooooooooooooooo":49135,"Ec":49136,"ruby":49137,"Professional":49138,"ĠOutbreak":49139,"argument":49140,"Grey":49141,"ĠFifa":49142,"ĠCHO":49143,"ĠFORM":49144,"ĠAmtrak":49145,"-[":49146,"Ġcradle":49147,"Ġantioxidants":49148,"ãģ®å®":49149,"736":49150,"ĠNASL":49151,"ĠContributions":49152,"Indiana":49153,"ĠSTEP":49154,"CSS":49155,"Ġsalient":49156,"Ġallocations":49157,"yrights":49158,"Ġmashed":49159,"ĠCutter":49160,"Sexual":49161,"Ġpounded":49162,"Ġfanbase":49163,"Ġcasc":49164,"ĠTransparency":49165,"Ġanalytic":49166,"ĠSummoner":49167,"×ŀ":49168,"ĠADC":49169,"detail":49170,"Ġvanquished":49171,"Ġcrabs":49172,"arie":49173,"Destroy":49174,"ĠSack":49175,"Ġtransistor":49176,"Alabama":49177,"ĠKoen":49178,"ĠFisheries":49179,"cone":49180,"Ġannexed":49181,"ĠMGM":49182,"esa":49183,"Ġfaked":49184,"ĠCongratulations":49185,"Ġhindered":49186,"Ġcorrectional":49187,"ĠITV":49188,"leeve":49189,"Ġinappropriately":49190,"licks":49191,"Ġtrespass":49192,"Ġpaws":49193,"Ġnegotiator":49194,"ĠChristensen":49195,"limits":49196,"ĠDianne":49197,"Ġelegance":49198,"ĠContracts":49199,"anke":49200,"Obj":49201,"Ġvigilance":49202,"Ġcastles":49203,"ĠNAD":49204,"ĠHolo":49205,"Ġemphatically":49206,"ĠTitus":49207,"ĠServing":49208,"ĠRichie":49209,"ĠPigs":49210,"568":49211,"Ġanimosity":49212,"ĠAttributes":49213,"ĠUriel":49214,"MQ":49215,"myra":49216,"ĠApplicant":49217,"Ġpsychiatrists":49218,"ĠVij":49219,"ĠAbby":49220,"agree":49221,"Push":49222,"ĠkWh":49223,"hiba":49224,"Ġincite":49225,"ĠWeasley":49226,"ĠTaxi":49227,"ministic":49228,"hyper":49229,"ĠFarn":49230,"Ġ601":49231,"ĠNationwide":49232,"Fake":49233,"952":49234,"Ġmaize":49235,"Ġinteracted":49236,"Ġtransitioned":49237,"Ġparasitic":49238,"Ġharmonic":49239,"Ġdecaying":49240,"Ġbaseless":49241,"nsics":49242,"Ġtranspired":49243,"Ġabundantly":49244,"ĠForensic":49245,"Ġtreadmill":49246,"ĠJav":49247,"aband":49248,"Ġsshd":49249,"Ġfrontman":49250,"ĠJakarta":49251,"oller":49252,"drops":49253,"ĠSERVICES":49254,"romptu":49255,"ophical":49256,"hospital":49257,"bledon":49258,"645":49259,"Ġmidrange":49260,"ĠEVENT":49261,"culated":49262,"rawled":49263,"Ġperched":49264,"Ġoverboard":49265,"ĠPeel":49266,"ĠPwr":49267,"ĠCarth":49268,"ĠCOMPLE":49269,"coe":49270,"shall":49271,"Ġdeterrence":49272,"METHOD":49273,"ĠAbsent":49274,"MEN":49275,"Ġsill":49276,"ĠLEVEL":49277,"York":49278,"Ġsinners":49279,"ĠOPEC":49280,"ĠNur":49281,"ĠDesigns":49282,"selection":49283,"Ġunworthy":49284,"CHA":49285,"Ġstrengthens":49286,"883":49287,"edly":49288,"Ġslicing":49289,"Ġmalnutrition":49290,"Ġfilmmaking":49291,"ĠPolk":49292,"urated":49293,"Ġ421":49294,"breakers":49295,"!'\"":49296,"Ġwetlands":49297,"ĠDiscrimination":49298,"Ġallowable":49299,"Ġsteered":49300,"ĠSicily":49301,"SAM":49302,"Ġmustache":49303,"Ġmids":49304,"Ġclipped":49305,"Ġcirculate":49306,"Ġbrittle":49307,"ĠBuildings":49308,"raised":49309,"ĠRoundup":49310,"Ġwealthier":49311,"Ġoverwrite":49312,"Ġoverpowered":49313,"ĠGerrard":49314,"sites":49315,"PDATED":49316,"Ġacutely":49317,"ĠGamble":49318,"Ġpim":49319,"ĠKus":49320,"Typically":49321,"Deploy":49322,"ĠMoroccan":49323,"potion":49324,"combe":49325,"Ġvigilante":49326,"Ġ363":49327,"Stew":49328,"ĠBagg":49329,"Ġresided":49330,"ĠSpo":49331,"Ġremnant":49332,"Ġemptiness":49333,"brainer":49334,"Ġoutpatient":49335,"priority":49336,"Ġleptin":49337,"ĠPayton":49338,"ĠGleaming":49339,"ĠShed":49340,"ĠPolo":49341,"ĠMormonism":49342,"restricted":49343,"arlane":49344,"wx":49345,"Ġcreatine":49346,"ĠAnon":49347,"ĠSTUD":49348,"ĠJUL":49349,"ĠTee":49350,"528":49351,"089":49352,"Ġhatched":49353,"Dispatch":49354,"ĠComposite":49355,"Ġ451":49356,"puff":49357,"ĠXCOM":49358,"ĠOrn":49359,"ĠTHANK":49360,"ENDED":49361,"ĠAsheville":49362,"ĠÃľ":49363,"Ġmango":49364,"ĠSlightly":49365,"worldly":49366,"ĠWander":49367,"ĠExpand":49368,"ĠChr":49369,"Mist":49370,"Ġorthodoxy":49371,"ĠUNESCO":49372,"regate":49373,"Elsewhere":49374,"kie":49375,"irled":49376,"Ġtopple":49377,"Ġadoptive":49378,"ĠLegs":49379,"dress":49380,"ĠSagan":49381,"bare":49382,"ĠGlou":49383,"Crunch":49384,"Ġhelpers":49385,"Ġchronically":49386,"ĠHuma":49387,"10000":49388,"Ġaccommodating":49389,"äºĶ":49390,"Ġwrinkles":49391,"Ġdodged":49392,"fourth":49393,"Ġprecon":49394,"Ġcompressor":49395,"ĠKare":49396,"Ġevict":49397,"ĠWarwick":49398,"imar":49399,"Ġmodernization":49400,"Ġbandwagon":49401,"Ġrefuted":49402,"Ġnetted":49403,"ĠNaples":49404,"ĠGenie":49405,"perors":49406,"Ġfielded":49407,"Ġdere":49408,"ĠParables":49409,"lees":49410,"Ġtrout":49411,"aspers":49412,"Ġnihil":49413,"Ġhappiest":49414,"Ġfloppy":49415,"ĠLoft":49416,"ĠHeard":49417,"Ġunison":49418,"Ġlug":49419,"ĠRedmond":49420,"classic":49421,"Supporters":49422,"SHIP":49423,"GMT":49424,"Ġfuelled":49425,"çIJ":49426,"Ġdd":49427,"ĠEminem":49428,"Ġ1897":49429,"NYSE":49430,"Ġsecretaries":49431,"ĠFIA":49432,"ĠCanaveral":49433,"Favorite":49434,"Ġpomp":49435,"Ġdetainee":49436,"ership":49437,"aimon":49438,"iour":49439,"ĠApex":49440,"Ġplantations":49441,"amia":49442,"acion":49443,"Rust":49444,"Ġtowed":49445,"ĠTruly":49446,"577":49447,"Ġsheltered":49448,"rider":49449,"Wo":49450,"Ġlair":49451,"ĠIntelligent":49452,"improve":49453,"matically":49454,"Ġetiquette":49455,"adra":49456,"allo":49457,"ĠJuno":49458,"anything":49459,"ĠStruggle":49460,"ĠPredict":49461,"ĠGrimes":49462,"ĠAMERICA":49463,"ctx":49464,"ĠSituation":49465,"WOOD":49466,"Ġsoluble":49467,"meier":49468,"Ġintolerable":49469,"angering":49470,"Ġuninterrupted":49471,"Ġtooltip":49472,"Ġinterrogated":49473,"Ġgunned":49474,"ĠSneak":49475,"æŃ¦":49476,"Ġtether":49477,"Ġcrumble":49478,"Lens":49479,"Ġclustered":49480,"ĠSyl":49481,"ĠHasan":49482,"Ġdystopian":49483,"wana":49484,"Ġjoystick":49485,"ĠThib":49486,"ammu":49487,"Tomorrow":49488,"546":49489,"Ġovercame":49490,"Ġminimized":49491,"ceptor":49492,"Runner":49493,"ENGTH":49494,"ĠBrenda":49495,"ĠAchievements":49496,"Ġtorches":49497,"Ġrapport":49498,"ĠInvestigator":49499,"ĠHandling":49500,"relation":49501,"grey":49502,"815":49503,"Ġkcal":49504,"ĠCommands":49505,"dq":49506,"Ġcurls":49507,"Ġbearer":49508,"Ġcynicism":49509,"itri":49510,"ĠUseful":49511,"Bee":49512,"DCS":49513,"Ġabras":49514,"Pract":49515,"BILITIES":49516,"712":49517,"Ġdebugger":49518,"Ġdebtor":49519,"ĠLia":49520,"ĠKers":49521,"Ġexacerbate":49522,"ĠStacy":49523,"ĠBland":49524,"ĠScenes":49525,"Ġbranching":49526,"âĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪâĸĪ":49527,"apeake":49528,"Ġsalsa":49529,"Ġmishand":49530,"ĠKonami":49531,"ĠNib":49532,"Ġanecdote":49533,"Ġagreeable":49534,"Ïī":49535,"ĠNathaniel":49536,"ĠHeisman":49537,"ĠBeware":49538,"Ġ1886":49539,"spective":49540,"691":49541,"522":49542,"Ġinhibits":49543,"Ġhashing":49544,"Ġ1889":49545,"å°Ĩ":49546,"vich":49547,"Pure":49548,"Ġsolidly":49549,"Ġaspirin":49550,"imaru":49551,"Ġstreetcar":49552,"ĠUCS":49553,"ĠJudd":49554,"Ġflashbacks":49555,"pins":49556,"Ġ1440":49557,"ĠUNHCR":49558,"ĠSymptoms":49559,"TIT":49560,"538":49561,"Fra":49562,"%);":49563,"Ġooz":49564,"Ġcurfew":49565,"Ġcalmed":49566,"Ġparticipates":49567,"TeX":49568,"Ġnonsensical":49569,"Ġfullback":49570,"ĠDeL":49571,"monkey":49572,"hari":49573,"Ġmetabolites":49574,"Ġlooted":49575,"ĠALWAYS":49576,"ĠBCC":49577,"Lt":49578,"ochet":49579,"Bone":49580,"Ġvetoed":49581,"Ġgcc":49582,"ĠCLICK":49583,"Ġ1888":49584,"saf":49585,"Ġstiffness":49586,"Ġlowly":49587,"ĠGeh":49588,"verson":49589,"orset":49590,"Ġunforeseen":49591,"Ġanesthesia":49592,"ĠOptical":49593,"Ġreconstructed":49594,"ĠTup":49595,"shows":49596,"NEWS":49597,"ĠNewspaper":49598,"ĠASA":49599,"tera":49600,"Numbers":49601,"Ġinexplicable":49602,"×ij":49603,"Ġhardness":49604,"untarily":49605,"ĠAcer":49606,"gradient":49607,"ARDIS":49608,"Ġwoodland":49609,"Ġmetaphors":49610,"ĠWembley":49611,"ĠPavel":49612,"philis":49613,"Ġrewriting":49614,"Ġperceptual":49615,"Ġ1070":49616,"worms":49617,"ĠDowns":49618,"Ġunsurprisingly":49619,"Ġtagging":49620,"flame":49621,"Ġlitres":49622,"Ġbounces":49623,"ĠBabe":49624,"shut":49625,"Ġoverdoses":49626,"ĠSheila":49627,"ĠChau":49628,"ĠBless":49629,"Capture":49630,"ĠSignificant":49631,"ĠScion":49632,"Ġ389":49633,"ĠMcH":49634,"ĠTitanium":49635,"ĠMeal":49636,"ameda":49637,"agents":49638,"aggressive":49639,"Billy":49640,"763":49641,"ĠSaying":49642,"DERR":49643,"itone":49644,"Collins":49645,"Bound":49646,"Ġbolted":49647,"ĠDMCA":49648,"953":49649,"Ġuniqueness":49650,"Ġepigen":49651,"unci":49652,"antam":49653,"Ġreckoning":49654,"chairs":49655,"OGR":49656,"ĠSenegal":49657,"Ġ1862":49658,"relevant":49659,"Ġ¯":49660,"Ġpharmacies":49661,"ĠGeral":49662,"vier":49663,"Yan":49664,"ORPG":49665,"Ġrabid":49666,"bending":49667,"ĠUNITED":49668,"Ġ465":49669,"Assembly":49670,"Ġweep":49671,"Ġbehest":49672,"ĠMothers":49673,"ĠJace":49674,"hid":49675,"Ġwhirlwind":49676,"ĠUNIVERS":49677,"Ġutopian":49678,"Ġkidnap":49679,"Philipp":49680,"Kin":49681,"893":49682,"Ġlivestream":49683,"ĠMISS":49684,"Ġsubversive":49685,"ĠTechniques":49686,"ĠJUSTICE":49687,"ĠBASE":49688,"Ġ387":49689,"Ġassailants":49690,"ĠHardcore":49691,"Ġsprinkled":49692,"ĠPse":49693,"éļ":49694,"printed":49695,"ĠHau":49696,"ORGE":49697,"ĠTOUR":49698,"Ġlaced":49699,"Ġitch":49700,"Giving":49701,"Ġported":49702,"781":49703,"////////////////////////////////":49704,"breeding":49705,"Ġlogger":49706,"ĠHOL":49707,"innie":49708,"Firstly":49709,"Ġembryonic":49710,"Ġdelegated":49711,"pai":49712,"OIL":49713,"Ġcentrally":49714,"ĠRx":49715,"ĠScouting":49716,"Dutch":49717,"Ġhereditary":49718,"ĠCruiser":49719,"sat":49720,"529":49721,"ĠMarriott":49722,"othermal":49723,"Ġprohibitions":49724,"Earn":49725,"ĠStab":49726,"ĠColleges":49727,"ĠBelief":49728,"stretched":49729,"ĠLH":49730,"ĠEntityItem":49731,"CIA":49732,"Ġunrem":49733,"Ġlaureate":49734,"Ġdenominations":49735,"summary":49736,"hler":49737,"Spect":49738,"ĠKlaus":49739,"ĠBeans":49740,"Ġinsur":49741,"ĠPAX":49742,"Ġfielder":49743,"ĠVet":49744,"ĠSparrow":49745,"zie":49746,"ĠSQ":49747,"ĠMondays":49748,"ĠOffline":49749,"ĠLerner":49750,"ĠExtensions":49751,"Ireland":49752,"Ġpatronage":49753,"Ġcontrasted":49754,"ĠMania":49755,"hirt":49756,"Moscow":49757,"Ġcondemns":49758,"ĠAnge":49759,"Ġcomposing":49760,"ĠPepe":49761,"ĠPaddock":49762,"Ġheterogeneity":49763,"Ġideologically":49764,"Ġfishes":49765,"Ġcursing":49766,"ĠRutherford":49767,"ĠFloating":49768,"ĠAmelia":49769,"Tea":49770,"Synopsis":49771,"Ġstunts":49772,"Ġbead":49773,"Ġstocking":49774,"ĠMILL":49775,"obook":49776,"massive":49777,"\\<":49778,"Ġhump":49779,"ĠPreferences":49780,"EngineDebug":49781,"geist":49782,"ĠNieto":49783,"omever":49784,"ishy":49785,"evaluate":49786,"colonial":49787,"Alternative":49788,"ĠGoPro":49789,"ĠVortex":49790,"ĠNETWORK":49791,"ansky":49792,"Secure":49793,"ĠThrust":49794,"Snake":49795,"Ġparcels":49796,"Ġsamurai":49797,"Ġactresses":49798,"Nap":49799,"MF":49800,"iferation":49801,"Beer":49802,"523":49803,"ĠIly":49804,"ointment":49805,"Ping":49806,"Ġstriped":49807,"ĠMellon":49808,"ossession":49809,"Ġneutron":49810,"endium":49811,"Ġaph":49812,"ĠFlavoring":49813,"Ġ383":49814,"Ġresponsiveness":49815,"ĠJindal":49816,"ĠHitchcock":49817,"Denver":49818,"ĠDRAGON":49819,"smanship":49820,"ĠDupl":49821,"Ġsly":49822,"Ġwebcam":49823,"ĠTwain":49824,"ĠDarling":49825,"iliate":49826,"consumer":49827,"DIT":49828,"Ġnamesake":49829,"Ġunorthodox":49830,"Ġfuner":49831,"ĠPLoS":49832,"ĠCONTROL":49833,"ozyg":49834,"oglobin":49835,"FACE":49836,"ERG":49837,"ĠDia":49838,"ĠFiesta":49839,"cele":49840,"034":49841,"Ġenclave":49842,"âĸ¬âĸ¬":49843,"onement":49844,"alist":49845,"Mand":49846,"Ġhomegrown":49847,"ĠFancy":49848,"Ġconceptions":49849,"ĠContains":49850,"ureen":49851,"Ġreiterate":49852,"Ġmeager":49853,"Ġinstallments":49854,"Spawn":49855,"627":49856,"Ġphotoc":49857,"ĠCabrera":49858,"ĠRosenthal":49859,"ĠLansing":49860,"isner":49861,"Ġinvests":49862,"ĠUFOs":49863,"EXP":49864,"Hardware":49865,"Ġtragically":49866,"Ġconcedes":49867,"ieft":49868,"cham":49869,"borgh":49870,"ĠSchr":49871,"ĠMelanie":49872,"ĠHoy":49873,"Ġvisitation":49874,"Ġidiosyncr":49875,"Ġfractions":49876,"Ġforeskin":49877,"obos":49878,"Ġpoaching":49879,"ĠVIEW":49880,"Ġstimulates":49881,"ĠGork":49882,"canon":49883,"MIC":49884,"ĠNemesis":49885,"ĠIndra":49886,"ĠDMV":49887,"Ġ529":49888,"Ġinspecting":49889,"Ġgrandma":49890,"ĠWhedon":49891,"ĠShant":49892,"ĠPurg":49893,"ikan":49894,"ĠTeg":49895,"ĠCLR":49896,"zac":49897,"Victoria":49898,"ĠVerify":49899,"ionics":49900,"Ġpartying":49901,"ĠMou":49902,"colour":49903,"Ġtestimonies":49904,"lations":49905,"Ġpressuring":49906,"hiro":49907,"acers":49908,"Ġfid":49909,"angler":49910,"ĠCSI":49911,"Ġhereafter":49912,"Ġdissidents":49913,"reporting":49914,"iphany":49915,"chev":49916,"Ġsolitude":49917,"Ġlobe":49918,"Ġindis":49919,"Ġcredential":49920,"recent":49921,"adult":49922,"ĠNirvana":49923,"ĠFranchise":49924,"Layer":49925,"Hyp":49926,"ĠBerkshire":49927,"Ġwills":49928,"tif":49929,"Ġtotem":49930,"ĠJudah":49931,"repair":49932,"Instant":49933,"548":49934,"Ġembassies":49935,"Ġbottleneck":49936,"Ġbount":49937,"Ġtypew":49938,"ĠAlvin":49939,"jing":49940,"imilar":49941,"Rush":49942,"Ġbrim":49943,"ĠHELP":49944,"Aim":49945,"]'":49946,"Ġpassively":49947,"Ġbounded":49948,"ĠRated":49949,"Ġcriminality":49950,"Ġbiomark":49951,"Ġdispatcher":49952,"ĠTowards":49953,"Ġ+++":49954,"righteous":49955,"frog":49956,"ĠPanc":49957,"Carter":49958,"032":49959,"æ©Ł":49960,"Ġultraviolet":49961,"ĠLicensed":49962,"ĠTata":49963,"ĠBlessing":49964,"ĠGAM":49965,"Ġchemically":49966,"ĠSeaf":49967,"ĠRELE":49968,"ĠMercenary":49969,"capitalist":49970,"Ġformulations":49971,"Ġannihilation":49972,"ĠVerb":49973,"ĠArgon":49974,"Ġunloaded":49975,"Ġmorphed":49976,"Ġconquering":49977,"backer":49978,"IELD":49979,"Ġthefts":49980,"Ġfrontrunner":49981,"ĠRoyale":49982,"ĠFundamental":49983,"elight":49984,"Chip":49985,"necessary":49986,"ayn":49987,"ĠSlip":49988,"Ġ448":49989,"cerned":49990,"Pause":49991,"Ġshockingly":49992,"ĠABV":49993,"Ġcomposure":49994,"733":49995,"ĠMotorsport":49996,"ahime":49997,"Murray":49998,"Mach":49999,"Ġgrids":50000,"Ġdebian":50001,"Ġfurthermore":50002,"Ġdexterity":50003,"ĠCollections":50004,"oslov":50005,"ilage":50006,"bj":50007,"ĠMonteneg":50008,"ĠstrutConnector":50009,"Ġmassacres":50010,"Ġbriefs":50011,"fetched":50012,"uvian":50013,"olition":50014,"Failure":50015,"emonic":50016,"Ġflared":50017,"Ġclaimant":50018,"Ġcures":50019,"Ġgiveaways":50020,"ĠSubstance":50021,"alions":50022,"Ġcringe":50023,"ĠKul":50024,"Ġaristocracy":50025,"ĠUlster":50026,"olated":50027,"housing":50028,"ĠMIS":50029,"Ġglared":50030,"ĠWilhelm":50031,"needs":50032,"lambda":50033,"builders":50034,"ĠVIS":50035,"Ġradiator":50036,"ĠGhostbusters":50037,"Ġ436":50038,"actual":50039,"Ġherds":50040,"ça":50041,"watching":50042,"Ġcountering":50043,"Charge":50044,"Ġcharred":50045,"Ġwarheads":50046,"Ġiodine":50047,"ĠMacy":50048,"041":50049,"Ġdepartures":50050,"ĠSins":50051,"Ġdyed":50052,"ĠConcepts":50053,"gado":50054,"713":50055,"Ġquotations":50056,"Ġgist":50057,"ĠChristy":50058,"Ġantigen":50059,"ĠHemp":50060,"ĠDrawn":50061,"ĠBarg":50062,"ezvous":50063,"Ġpaternity":50064,"Ġardu":50065,"ĠAnchorage":50066,"ĠRik":50067,"Ġoverloaded":50068,"ĠUsername":50069,"ĠTammy":50070,"ĠNau":50071,"ĠCellular":50072,"Ġwaning":50073,"Ġrodent":50074,"ĠWorcester":50075,"ilts":50076,"ĠTad":50077,"Ġdwellings":50078,"Ġbullish":50079,"431":50080,"Ġretaliate":50081,"Ġmigraine":50082,"ĠChevron":50083,"CHECK":50084,"Ġdonkey":50085,"crim":50086,"SPA":50087,"ĠAnalog":50088,"Ġmarquee":50089,"ĠHaas":50090,"Bir":50091,"ĠGDDR":50092,"ĠDownloads":50093,"Ġwillpower":50094,"ĠForth":50095,"ĠRecorded":50096,"Ġimpossibility":50097,"ĠLogged":50098,"ĠFranks":50099,"ĠRatt":50100,"initions":50101,"Ġcleaners":50102,"Ġsorely":50103,"Ġflickering":50104,"ĠExamination":50105,"catching":50106,"alloween":50107,"Msg":50108,"Ġdunno":50109,"Fa":50110,"Ġdysph":50111,"crazy":50112,".''.":50113,"Ġmainline":50114,"Ġcs":50115,"Ġptr":50116,"ĠWally":50117,"igun":50118,"951":50119,"ĠBigfoot":50120,"fights":50121,"Ġretrieving":50122,"Jr":50123,"Ġduplication":50124,"ĠExplan":50125,"Ġrelational":50126,"Ġquaint":50127,"Ġbiscuits":50128,"Ġado":50129,"Ġshudder":50130,"Ġantidote":50131,"blooded":50132,"ksh":50133,"Ġsauces":50134,"Ġreinvest":50135,"Ġdispensary":50136,"ĠDiver":50137,"Ġ9000":50138,"student":50139,"Ġinsepar":50140,"escap":50141,"Ġtoddlers":50142,"ĠGPIO":50143,"ĠAssignment":50144,"headers":50145,"Ġlackluster":50146,"Ġaback":50147,"956":50148,"Ġtoolbar":50149,"745":50150,"Ġoust":50151,"Ġcontemplation":50152,"ĠPRESIDENT":50153,"Ġ458":50154,"======":50155,"Ġguaranteeing":50156,"ĠHeist":50157,"ĠCannes":50158,"Ͻ":50159,"Ġcollaborator":50160,"ĠAmp":50161,"Ġgou":50162,"ĠSHALL":50163,"stories":50164,"783":50165,"Ġmobilized":50166,"Ġbrood":50167,"ĠLU":50168,"ĠðŁij":50169,"Ġrefin":50170,"ĠAnthropology":50171,"vind":50172,"illi":50173,"Ġwarranties":50174,"ĠBabel":50175,"Ġswath":50176,"Ġcaches":50177,"Ġantagonists":50178,"artifacts":50179,"Ġhotly":50180,"ĠStarts":50181,"ĠGö":50182,"zag":50183,"!!!!!":50184,"Ġscourge":50185,"Ġconspiring":50186,"ruits":50187,"reverse":50188,"ĠSheen":50189,"ĠJesuit":50190,"ĠGiovanni":50191,"adies":50192,"Ġbuttocks":50193,"earcher":50194,"acan":50195,"Ġvolleyball":50196,"Ġshrouded":50197,"Ġscoreboard":50198,"bats":50199,"ĠIPM":50200,"Ġasses":50201,"Ġderegulation":50202,"ĠTelegram":50203,"ĠReboot":50204,"Ġ7000":50205,"ĠCanary":50206,"Ġkernels":50207,"ĠFrançois":50208,"ĠDuff":50209,"ĠPon":50210,"ĠLeica":50211,"ĠGarmin":50212,"Ġorphans":50213,"ĠClaudia":50214,"Ġcalendars":50215,"ĠLeilan":50216,"ento":50217,"Rocket":50218,"Ġbrunch":50219,"ĠHawking":50220,"ainers":50221,"Ġsensibilities":50222,"ĠkW":50223,"ĠKand":50224,"Ġreclaimed":50225,"Ġinterestingly":50226,"ש":50227,"romy":50228,"JM":50229,"ĠEnhancement":50230,"bush":50231,"Skip":50232,"Ġrappers":50233,"Ġgazing":50234,"pedia":50235,"athlon":50236,"Revolution":50237,"Ġsnipers":50238,"Ġreverted":50239,"Ġconglomerate":50240,"Terry":50241,"794":50242,"Ġharsher":50243,"Ġdesolate":50244,"ĠHitman":50245,"Commission":50246,"Ġ(/":50247,"â̦.\"":50248,"Compar":50249,"Ġamplification":50250,"ominated":50251,"Ġregress":50252,"ĠCollider":50253,"Ġinformants":50254,"Ġgazed":50255,"<|endoftext|>":50256},"merges":["Ġ t","Ġ a","h e","i n","r e","o n","Ġt he","e r","Ġ s","a t","Ġ w","Ġ o","e n","Ġ c","i t","i s","a n","o r","e s","Ġ b","e d","Ġ f","in g","Ġ p","o u","Ġa n","a l","a r","Ġt o","Ġ m","Ġo f","Ġ in","Ġ d","Ġ h","Ġan d","i c","a s","l e","Ġt h","i on","o m","l l","en t","Ġ n","Ġ l","s t","Ġ re","v e","Ġ e","r o","l y","Ġb e","Ġ g","Ġ T","c t","Ġ S","i d","o t","Ġ I","u t","e t","Ġ A","Ġ is","Ġ on","i m","a m","o w","a y","a d","s e","Ġth at","Ġ C","i g","Ġf or","a c","Ġ y","v er","u r","Ġ u","l d","Ġs t","Ġ M","' s","Ġ he","Ġ it","at ion","it h","i r","c e","Ġy ou","i l","Ġ B","Ġw h","o l","Ġ P","Ġw ith","Ġ 1","t er","c h","Ġa s","Ġw e","Ġ (","n d","i ll","Ġ D","i f","Ġ 2","a g","er s","k e","Ġ \"","Ġ H","e m","Ġc on","Ġ W","Ġ R","he r","Ġw as","Ġ r","o d","Ġ F","u l","at e","Ġa t","r i","p p","o re","ĠT he","Ġs e","u s","Ġp ro","Ġh a","u m","Ġa re","Ġd e","a in","an d","Ġo r","ig h","es t","is t","a b","r om","Ġ N","t h","Ġc om","Ġ G","u n","o p","0 0","Ġ L","Ġn ot","es s","Ġe x","Ġ v","re s","Ġ E","e w","it y","an t","Ġb y","e l","o s","or t","o c","q u","Ġf rom","Ġha ve","Ġs u","i ve","ou ld","Ġs h","Ġth is","n t","r a","p e","igh t","ar t","m ent","Ġa l","u st","en d","- -","al l","Ġ O","ac k","Ġc h","Ġ le","i es","re d","ar d","â Ģ","ou t","Ġ J","Ġa b","e ar","i v","al ly","ou r","o st","g h","p t","Ġp l","as t","Ġc an","a k","om e","u d","T he","Ġh is","Ġd o","Ġg o","Ġh as","g e","' t","Ġ U","r ou","Ġs a","Ġ j","Ġb ut","Ġw or","Ġa ll","e ct","Ġ k","am e","Ġw ill","o k","Ġw he","Ġthe y","id e","0 1","f f","ic h","p l","t her","Ġt r",". .","Ġin t","i e","u re","ag e","Ġn e","i al","a p","in e","ic e","Ġm e","Ġo ut","an s","on e","on g","ion s","Ġwh o","Ġ K","Ġu p","Ġthe ir","Ġa d","Ġ 3","Ġu s","at ed","ou s","Ġm ore","u e","o g","ĠS t","in d","i ke","Ġs o","im e","p er",". \"","b er","i z","a ct","Ġon e","Ġsa id","Ġ -","a re","Ġyou r","c c","ĠT h","Ġc l","e p","a ke","ab le","i p","Ġcon t","Ġwh ich","i a","Ġ im","Ġab out","Ġwe re","ver y","u b","Ġh ad","Ġ en","Ġcom p",", \"","ĠI n","Ġu n","Ġa g","i re","ac e","a u","ar y","Ġw ould","as s","r y","Ġ âĢ","c l","o ok","e re","s o","Ġ V","ig n","i b","Ġof f","Ġt e","v en","Ġ Y","i le","o se","it e","or m","Ġ2 01","Ġre s","Ġm an","Ġp er","Ġo ther","or d","ul t","Ġbe en","Ġl ike","as e","an ce","k s","ay s","ow n","en ce","Ġd is","ct ion","Ġan y","Ġa pp","Ġs p","in t","res s","ation s","a il","Ġ 4","ic al","Ġthe m","Ġhe r","ou nt","ĠC h","Ġa r","Ġ if","Ġthe re","Ġp e","Ġy ear","a v","Ġm y","Ġs ome","Ġwhe n","ou gh","ac h","Ġth an","r u","on d","ic k","Ġo ver","ve l","Ġ qu","Ċ Ċ","Ġs c","re at","re e","ĠI t","ou nd","p ort","Ġal so","Ġp art","f ter","Ġk n","Ġbe c","Ġt ime","en s","Ġ 5","op le","Ġwh at","Ġn o","d u","m er","an g","Ġn ew","-- --","Ġg et","or y","it ion","ing s","Ġj ust","Ġint o","Ġ 0","ent s","o ve","t e","Ġpe ople","Ġp re","Ġit s","Ġre c","Ġt w","i an","ir st","ar k","or s","Ġwor k","ad e","o b","Ġs he","Ġo ur","w n","in k","l ic","Ġ1 9","ĠH e","is h","nd er","au se","Ġh im","on s","Ġ [","Ġ ro","f orm","i ld","at es","ver s","Ġon ly","o ll","Ġs pe","c k","e ll","am p","Ġa cc","Ġb l","i ous","ur n","f t","o od","Ġh ow","he d","Ġ '","Ġa fter","a w","Ġat t","o v","n e","Ġpl ay","er v","ic t","Ġc ould","it t","Ġa m","Ġf irst","Ġ 6","Ġa ct","Ġ $","e c","h ing","u al","u ll","Ġcom m","o y","o ld","c es","at er","Ġf e","Ġbe t","w e","if f","Ġtw o","oc k","Ġb ack",") .","id ent","Ġu nder","rou gh","se l","x t","Ġm ay","rou nd","Ġp o","p h","is s","Ġd es","Ġm ost","Ġd id","Ġad d","j ect","Ġin c","f ore","Ġp ol","on t","Ġag ain","cl ud","ter n","Ġkn ow","Ġne ed","Ġcon s","Ġc o","Ġ .","Ġw ant","Ġse e","Ġ 7","n ing","i ew","ĠTh is","c ed","Ġe ven","Ġin d","t y","ĠW e","at h","Ġthe se","Ġp r","Ġu se","Ġbec ause","Ġf l","n g","Ġn ow","ĠâĢ ĵ","c om","is e","Ġm ake","Ġthe n","ow er","Ġe very","ĠU n","Ġse c","os s","u ch","Ġe m","Ġ =","ĠR e","i ed","r it","Ġin v","le ct","Ġsu pp","at ing","Ġl ook","m an","pe ct","Ġ 8","ro w","Ġb u","Ġwhe re","if ic","Ġyear s","i ly","Ġd iff","Ġsh ould","Ġre m","T h","I n","Ġe v","d ay","' re","ri b","Ġre l","s s","Ġde f","Ġr ight","Ġs y",") ,","l es","00 0","he n","Ġth rough","ĠT r","_ _","Ġw ay","Ġd on","Ġ ,","Ġ1 0","as ed","Ġas s","ub lic","Ġre g","ĠA nd","i x","Ġ very","Ġin clud","ot her","Ġim p","ot h","Ġsu b","ĠâĢ Ķ","Ġbe ing","ar g","ĠW h","= =","ib le","Ġdo es","an ge","r am","Ġ 9","er t","p s","it ed","ation al","Ġb r","Ġd own","Ġman y","ak ing","Ġc all","ur ing","it ies","Ġp h","ic s","al s","Ġde c","at ive","en er","Ġbe fore","il ity","Ġwe ll","Ġm uch","ers on","Ġth ose","Ġsu ch","Ġ ke","Ġ end","ĠB ut","as on","t ing","Ġl ong","e f","Ġth ink","y s","Ġbe l","Ġs m","it s","a x","Ġo wn","Ġpro v","Ġs et","if e","ment s","b le","w ard","Ġsh ow","Ġp res","m s","om et","Ġo b","Ġs ay","ĠS h","t s","f ul","Ġe ff","Ġg u","Ġin st","u nd","re n","c ess","Ġ ent","ĠY ou","Ġgo od","Ġst art","in ce","Ġm ade","t t","st em","ol og","u p","Ġ |","um p","Ġhe l","ver n","ul ar","u ally","Ġa c","Ġm on","Ġl ast","Ġ2 00","1 0","Ġst ud","u res","ĠA r","sel f","ar s","mer ic","u es","c y","Ġm in","oll ow","Ġc ol","i o","Ġm od","Ġc ount","ĠC om","he s","Ġf in","a ir","i er","âĢ Ķ","re ad","an k","at ch","e ver","Ġst r","Ġpo int","or k","ĠN ew","Ġs ur","o ol","al k","em ent","Ġus ed","ra ct","we en","Ġs ame","ou n","ĠA l","c i","Ġdiff ere","Ġwh ile","---- ----","Ġg ame","ce pt","Ġs im",".. .","Ġin ter","e k","Ġre port","Ġpro du","Ġst ill","l ed","a h","Ġhe re","Ġwor ld","Ġth ough","Ġn um","ar ch","im es","al e","ĠS e","ĠI f","/ /","ĠL e","Ġre t","Ġre f","Ġtr ans","n er","ut ion","ter s","Ġt ake","ĠC l","Ġcon f","w ay","a ve","Ġgo ing","Ġs l","u g","ĠA meric","Ġspe c","Ġh and","Ġbet ween","ist s","ĠD e","o ot","I t","Ġe ar","Ġagain st","Ġh igh","g an","a z","at her","Ġex p","Ġo p","Ġin s","Ġg r","Ġhel p","Ġre qu","et s","in s","ĠP ro","is m","Ġf ound","l and","at a","us s","am es","Ġp erson","Ġg reat","p r","Ġs ign","ĠA n","' ve","Ġs omet","Ġs er","h ip","Ġr un","Ġ :","Ġt er","ire ct","Ġf ollow","Ġd et","ic es","Ġf ind","1 2","Ġm em","Ġc r","e red","e x","Ġex t","ut h","en se","c o","Ġte am","v ing","ou se","as h","at t","v ed","Ġsy stem","ĠA s","d er","iv es","m in","Ġle ad","ĠB l","c ent","Ġa round","Ġgo vern","Ġc ur","vel op","an y","Ġc our","al th","ag es","iz e","Ġc ar","od e","Ġl aw","Ġre ad","' m","c on","Ġre al","Ġsupp ort","Ġ1 2",".. ..","Ġre ally","n ess","Ġf act","Ġd ay","Ġb oth","y ing","Ġs erv","ĠF or","Ġth ree","Ġw om","Ġm ed","od y","ĠThe y","5 0","Ġex per","t on","Ġe ach","ak es","Ġc he","Ġc re","in es","Ġre p","1 9","g g","ill ion","Ġg rou","ut e","i k","W e","g et","E R","Ġm et","Ġs ays","o x","Ġd uring","er n","iz ed","a red","Ġf am","ic ally","Ġha pp","ĠI s","Ġch ar","m ed","v ent","Ġg ener","i ent","p le","i et","re nt","1 1","v es","pt ion","Ġ2 0","form ation","Ġc or","Ġoff ic","ie ld","Ġto o","is ion","Ġin f","Ġ Z","t he","o ad","Ġp ublic","Ġpro g","r ic","* *","Ġw ar","Ġp ower","v iew","Ġf ew","Ġl oc","Ġdiffere nt","Ġst ate","Ġhe ad","' ll","Ġp oss","Ġst at","re t","ant s","Ġv al","Ġis s","Ġc le","i vers","an c","Ġex pl","Ġan other","Ġ Q","Ġa v","th ing","n ce","W h","Ġch ild","Ġs ince","i red","l ess","Ġl ife","Ġde velop","itt le","Ġde p","Ġp ass","ã ĥ","Ġt urn","or n","Th is","b ers","ro ss","ĠA d","Ġf r","Ġres p","Ġsec ond","o h","Ġ /","Ġdis c","Ġ &","Ġsomet hing","Ġcomp le","Ġ ed","Ġf il","Ġmon th","a j","u c","Ġgovern ment","Ġwith out","Ġle g","Ġd ist","Ġp ut","Ġqu est","an n","Ġpro t","2 0","Ġne ver","i ence","Ġle vel","Ġar t","Ġth ings","Ġm ight","Ġeff ect","Ġcont ro","Ġc ent","Ġ1 8","Ġall ow","Ġbel ie","ch ool","ot t","Ġinc re","Ġfe el","Ġres ult","Ġl ot","Ġf un","ot e","Ġt y","ere st","Ġcont in","Ġus ing","Ġb ig","2 01","Ġas k","Ġb est","Ġ )","I N","Ġo pp","3 0","Ġnum ber","in ess","S t","le ase","Ġc a","Ġm ust","Ġd irect","Ġg l","Ġ <","Ġop en","Ġp ost","Ġcom e","Ġse em","ord ing","Ġwe ek","ate ly","it al","Ġe l","ri end","Ġf ar","Ġt ra","in al","Ġp ri","ĠU S","Ġpl ace","Ġfor m","Ġto ld","\" :","ain s","at ure","ĠTr ump","Ġst and","Ġ #","id er","ĠF r","Ġne xt","Ġs oc","Ġp ur","Ġle t","Ġl ittle","Ġh um","Ġ i","r on","1 5","Ġ1 5","Ġcomm un","Ġm ark","ĠThe re","Ġw r","ĠTh at","Ġin formation","w ays","Ġb us","a pp","Ġinv est","m e","Ġh ard","ain ed","e ad","Ġim port","Ġapp ro","Ġt est","Ġt ri","Ġre st","os ed","Ġf ull","Ġc are","ĠS p","Ġc ase","O N","Ġs k","Ġl ess","Ġ +","Ġpart ic","ĠP l","ab ly","u ck","is hed","ch n","b e","Ġl ist","at or","Ġto p","Ġad v","ĠB e","ru ct","Ġd em","r ation","l ing","g y","re en","g er","Ġh ome","Ġle ft","Ġbet ter","Ġd ata","Ġ1 1","Ġatt ack","Ġpro ble","l ine","ard s","Ġbe h","r al","ĠH ow","ĠS he","ar ge","Ġ --",": //","Ġb ro","ĠP h","at s","Ġbu ild","w w","id ed","a im","as es","en cy","Ġm ain","in ed","Ġinclud ing","Ġ {","Ġg ot","Ġint erest","Ġke ep","Ġ X","Ġe as","ain ing","Ġcl ass","âĢ ¦","ĠN o","Ġv ar","Ġsm all","amp le","A T","Ġ ide","ĠS o","Ġre ce","Ġpol it","Ġm ov","Ġpl an","Ġper cent","iv ing","Ġc amp","Ġp ay","1 4","s c","is ed","Ġu nt","one y","pl oy","== ==","Ġdid n","ĠI nd","el s","ert ain","Ġp os","__ __","i ver","Ġpro cess","Ġprog ram","if ied","ĠR ep","1 6","u ro","olog y","at ter","in a","Ġn ame","ĠA ll","Ġf our","Ġret urn","v ious","b s","Ġcall ed","Ġm ove","ĠS c","ir d","Ġgrou p","Ġb re","Ġm en","Ġc ap","t en","e e","Ġd ri","le g","he re","uth or","Ġp at","Ġcur rent","id es","Ġp op","t o","ent ion","Ġal ways","Ġm il","Ġwom en","Ġ1 6","Ġo ld","iv en","ra ph","ĠO r","r or","ent ly","Ġn ear","ĠE x","re am","s h","Ġ1 4","Ġf ree","iss ion","st and","ĠC on","al ity","us ed","1 3","Ġdes ign","Ġch ange","Ġch ang","Ġb o","Ġv is","em ber","Ġb ook","read y","Ġk ill","2 5","pp ed","Ġa way","Ġab le","Ġcount ry","Ġcon st","ar n","Ġor der","A R","i or","i um","or th","1 8","ail able","Ġs w","Ġm illion","Ġ1 3","at ic","t ed","ĠG o","Ġo per","en g","Ġth ing","aj or","con om","ĠCom m","Ġwh y","u red","ur al","Ġs chool","b y","ĠM ar","Ġa ff","Ġd ays","Ġan n","us h","an e","I f","e g","Ġpro f","Ġhe alth","ou th","B ut","ion al",". ,","Ġs ol","Ġal ready","Ġ3 0","Ġchar act","H e","Ġf riend","E S","i ans","ic le","' d","ĠO n","Ġle ast","Ġp rom","Ġd r","Ġh ist","it her","Ġ est","i qu","1 7","s on","Ġte ll","Ġt alk","oh n","o int","le ction","A N","Ġunt il","au gh","Ġl ater","Ġ ve","Ġv iew","end ing","iv ed","Ġwor d","w are","Ġc ost","Ġen ough","Ġg ive","ĠUn ited","Ġte chn","are nt","O R","Ġp ar","ĠD r","Ġ201 6","r ist","er ing","Ġ Â","Ġl arge","s ide","ac y","cc ess","Ġw in","Ġimport ant","Ġ19 9","Ġdoes n","Ġ1 7","Ġbus iness","Ġcle ar","Ġre se","\" ,","ur y","Ġe qu","as ter","al f","ĠAmeric an","n ect","Ġex pect","ivers ity","Ġo cc","ĠF l","Ġk ind","Ġme an","Ġp ast","Ġde v","Ġb as","le t","ra ft","Ġor gan","Ġde l","Ġper form","Ġst ory","Ġse ason","ĠC ol","Ġcl aim","Ġc ame","Ġwith in","Ġl ine","Ġpro ject","ĠA t","Ġcontro l","end ed","ĠS y","Ġa ir","iz ation","Ġ *","le y","Ġm oney","id d","Y ou","f or","Ġfam ily","Ġm aking","Ġb it","Ġpol ice","Ġhapp en","Ġ vers","on y","u ff","ĠW hen","Ġs it","ide o","l f","is on","Ġsu re","g in","Ġapp ear","Ġl ight","Ġ es","o f","Ġw ater","Ġt imes","n ot","Ġg row","Ġcomp any","ĠT e","ow s","Ġm ar","our ce","i ol","ar m","b r","Ġex ample","Ġcon c","Ġf ore","ĠT o","p ro","E N","ri es","Ġ2 5","ĠC an","ne y","Ġact ually","Ġe ver","ur ity","ak en","ap s","Ġt ax","Ġm ajor","am a","Ġof ten","er al","Ġhum an","Ġj ob","is ter","Ġav ailable","oc r","en n","a id","iv id","Ġrec ord","? \"","Ġs ing","ĠA m","id ence","Ġnew s","st er","Ġe conom","Ġfollow ing","ĠB r","is ing","Ġh our","m ost","um ent","Ġse x","Ġdes c","Ġbec ome","ĠE d","Ġto ok","Ġha ving","Ġprodu ct","a ult","A s","ar ing","Ġme ans","Ġh op","un e","Ġch o","Ġc ertain","Ġn on","Ġde al","2 4","le ment","oc i","en e","Ġs ide","ĠP r","ĠM ay","Ġre ason","u ed","c hed","ul ation","Ġe lect","Ġoffic ial","Ġposs ible","Ġh old","and s","ot s","Ġc ity","or ies","Ġse ver","Ġchild ren","Ġon ce","Ġact iv","l er","Ġn ight","it ions","ĠJ ohn","a pe","pl ay","Ġd one","Ġl im","Ġwork ing","ĠP res","or ld","e b","ĠC o","Ġb ody","ail s","ut es","ĠM r","Ġwhe ther","Ġa uthor","ro p","Ġpro per","Ġse en",") ;","Ġf ac","ĠS u","Ġcon d","it ing","Ġcour se","Ġ }","-------- --------","a ign","Ġev ent","Ġen g","Ġp ot","Ġin tern","i am","Ġsh ort","em pt","ã Ĥ","ĠG od","il ar","8 0","Ġor ig","I S","our n","ab ility","it ive","Ġd am","Ġ1 00","Ġp ress","Ġdo ing","Ġprot ect","r ing","Ġthough t","Ġquest ion","re w","ĠW ar","Ġsever al","ĠSt ate","Ġg iven","Ġf und","ĠT w","Ġw ent","an ces","w ork","p or","m y","4 0","Ġar g","art ment","ust om","Ġpol ic","Ġme et","Ġc reat","2 2","ĠSt ates","Ġg ames","ra w","ut ure","Ġunder stand","ur s","ĠO b","l ish","s y","Ġm akes","Ġw on","ag on","Ġh tt","Ġl ove","ent ial","Ġcomple te","p ar","ĠI m","A L","Ġacc ount"," ł","ore d","ver t","Ġ ident","Ġ201 5","Ġother s","ĠM in","i ber","ver age","The re","ition al","d d","Ġpro b","Ġyou ng","Ġal ong","Ġacc ording","Ġy et","Ġmem bers","ĠWh at","o id","ĠM an","A nd","Ġam ong","a i","Ġem ploy","ĠR es","Ġ >","Ġinv ol","Ġl ow","a f","ĠC ar","Ġh ig","ĠO ne","ĠS ec","in ation","Ġlike ly","Ġan t","ag ed","ĠR uss","Ġb en","Ġre le","F or","b ack","ĠN ot","Ġpres ident","b all","Ġacc ess","ivid ual","ĠD em","ĠE uro","6 0","Ġkn own","ir l","ĠG r","Ġear ly","u se","iet y","âĢ ĵ","Ġf ight","Ġs ent","Ġto day","Ġmark et","\" .","Ġb ased","Ġstr ong","ur ther","Ġde b","m ber","Ġproble m","Ġde ath","Ġsoc ial","im ate","A S","ort un","Ġcamp aign","er y","C h","Ġe y","i ally","Ġm us","w h","p os","Ġ er","Ġsa f","Ġmonth s","ir on","Ġv iol","Ġf ive","Ġst re","Ġplay ers","in c","al d","y ear","a un","Ġsu ccess","Ġpres ent","ere nce","Ġ201 4","Ġsu gg","Ġpartic ular","Ġtr y","Ġsugg est","ĠCh rist","on es","Ġpri v","2 3","Ġc rit","Ġl and","Ġloc al","if y","2 9","Ġa ut","E D","ĠG u","Ġm ult","Ġpolit ical","Ġask ed","Ġfor mer","it ter","ri pt","Ġcl ose","Ġp ract","ĠY ork","Ġget ting","Ġac ross","Ġcom b","Ġbelie ve","Ġ z","Ġto get","Ġtoget her","ĠC ent","ir c","Ġind ividual","ĠM c","2 7","is k","ĠE ng","Ġf ace","Ġ2 4","Ġval ue","Ġare a","e v","Ġw rit","ĠPres ident","Ġv ot","Ġke y","Ġm om","p ut","Ġany thing","Ġexper ience","att le","Ġm ind","a ff","om m","Ġf uture","g ed","Ġc ut","Ġto t","it ch","Ġv ideo","Ġinvest ig","Ġn et","ĠM y","r ict","i en",". )","Ġimp ro","th ough","ward s","Ġcon nect","ĠM ed","sel ves","ens ive","m b","o ber","at ors","A n","Ġ5 0","Ġre du","res ent","Ġab ove","Ġf re","ĠEuro pe","s w","Ġam ount","ĠA pp","Ġe ither","Ġmil it","Ġan al","Ġf ail","ĠE n","al es","Ġspec ial","Ġbl ack","I T","c her","Ġlook ing","Ġf ire","y n","Ġal most","o on","Ġstud y","Ġm iss","c hes","ro wn","Ġt re","Ġcommun ity","Ġmed ia","Ġf ood","Ġcom es","ĠUn iversity","Ġsing le","Wh at","u ly","Ġh alf","ag ue","h od","ĠRep ublic","Ġstart ed","Ġqu ick","ot o","b ook","Ġiss ue","it or","Ġel se","Ġcons ider","2 6","ro du","Ġt aken","2 8","9 9","ĠW ith","Ġtr ue","Ġw a","Ġtr ad","Ġag o","Ġm ess","ie f","Ġadd ed","o ke","Ġb ad","Ġf av","3 3","Ġsim ilar","as k","ĠD on","Ġcharact er","ort s","ĠH ouse","Ġreport ed","Ġty pe","v al","i od","ĠHow ever","Ġt arg","Ġent ire","pp ing","Ġhist ory","Ġl ive","ff ic",".... ....","ed eral","Ġtr ying","Ġdisc uss","ĠH ar","ac es","l ished","Ġse lf","os p","re st","Ġro om","el t","Ġf all","ol ution","Ġe t","Ġ x","Ġis n","Ġide a","b o","Ġs ound","ĠD ep","Ġsome one","ci ally","ull y","Ġf oc","Ġob ject","if t","ap er","Ġplay er","Ġr ather","Ġserv ice","as hing","ĠD o","ĠP art","ru g","m on","p ly","Ġm or","Ġnot hing","Ġprov ide","I C","un g","Ġpart y","Ġex ist","Ġm ag","7 0","Ġr ul","Ġh ouse","Ġbeh ind","Ġhow ever","ĠW orld","Ġs um","Ġapp lic","Ġ ;","Ġfun ction","g r","ĠP ol","Ġfr ont","2 00","Ġser ies","Ġt em","Ġty p","ill s","Ġo pt","Ġpoint s","Ġbel ow","itt ed","Ġspec ific","Ġ201 7","um b","Ġr a","Ġpre vious","Ġpre t","re me","Ġc ustom","Ġcour t","ĠM e","Ġre pl","Ġwho le","g o","c er","Ġt reat","ĠA ct","Ġprob ably","Ġle arn","end er","ĠA ss","Ġvers ion","n ow","Ġche ck","ĠC al","R E","min ist","O n","our ces","Ġben ef","Ġd oc","Ġdet er","Ġen c","Ġsu per","Ġadd ress","Ġv ict","Ġ201 3","Ġme as","t r","Ġf ield","W hen","Ġsign ific","u ge","Ġfe at","Ġcomm on","l oad","Ġbe gin","Ġbr ing","Ġa ction","er man","Ġdesc rib","Ġind ust","Ġwant ed","ri ed","m ing","Ġatt empt","4 5","f er","Ġd ue","ress ion","# #","Ġsh all","Ġs ix","o o","Ġst ep","Ġp ub","Ġhim self","Ġ2 3","Ġc op","Ġd est","Ġst op","A C","ib ility","Ġl ab","ic ult","Ġhour s","Ġcre ate","Ġf urther","ĠAmeric a","ĠC ity","Ġd ou","he ad","S T","ĠN orth","c ing","Ġn ational","u le","ĠIn st","Ġt aking","ĠQ u","ir t","Ġre d","Ġrese arch","v iron","ĠG e","Ġbre ak","an a","Ġsp ace","ater ial","Ġrec ent","ĠA b","Ġgener al","Ġh it","Ġper iod","Ġevery thing","ive ly","Ġph ys","Ġsay ing","an ks","Ġc ou","Ġc ult","ac ed","e al","u ation","Ġc oun","l u","Ġinclud e","Ġpos ition","ĠA fter","ĠCan ad","ĠE m","Ġim m","ĠR ed","Ġp ick","Ġcom pl","Ġm atter","re g","e xt","ang u","is c","o le","a ut","Ġcomp et","e ed","f ect","Ġ2 1","ĠS en","ĠThe se","as ing","Ġcan not","Ġin it","Ġrel ations","ac hed","Ġb ar","Ġ4 0","ĠT H","Ġ201 2","Ġv ol","Ġg round","Ġsec urity","Ġup d","il t","3 5","Ġconc ern","ĠJ ust","Ġwh ite","Ġseem s","ĠH er","pe cially","i ents","Ġann oun","Ġf ig","ight s","Ġst ri","l ike","id s","Ġs us","Ġw atch","Ġ â","Ġw ind","ĠC ont","Ġit self","Ġm ass","A l","y le","iqu e","ĠN ational","Ġab s","Ġp ack","Ġout side","Ġan im","Ġp ain","et er","Ġman ag","du ct","og n","Ġ ]","ĠSe pt","se c","o ff","ĠJ an","Ġf oot","ad es","Ġth ird","Ġm ot","Ġev idence","int on","Ġth reat","a pt","pl es","c le","Ġl o","Ġde cl","Ġit em","med i","Ġrep resent","om b","am er","Ġsignific ant","og raph","s u","Ġc al","i res","00 00","I D","A M","Ġsim ply","Ġlong er","Ġf ile","O T","c he","S o","ate g","or g","ĠH is","Ġen er","Ġd om","Ġup on","il i","\": \"","Ġthem selves","Ġcom ing","Ġqu ite","Ġdiff icult","ĠB ar","il ities","re l","end s","c ial","6 4","Ġwom an","ra p","y r","Ġne cess","ip s","Ġte xt","Ġrequ ire","Ġmilit ary","Ġre view","Ġresp ons","7 5","Ġsub ject","Ġinst ead","Ġiss ues","Ġg en","\" ,\"","Ġmin utes","Ġwe ap","r ay","am ed","t ime","b l","H ow","Ġc ode","ĠS m","Ġhig her","ĠSt e","r is","Ġp age","Ġstud ents","ĠIn tern","Ġmet hod","ĠA ug","ĠP er","ĠA g","Ġpolic y","ĠS w","Ġex ec","Ġac cept","um e","rib ut","Ġword s","Ġfin al","Ġchang es","ĠDem ocr","Ġfriend s","Ġres pect","Ġe p","Ġcomp an","iv il","Ġdam age","** **","og le","viron ment","Ġne g","ent al","Ġa p","Ġtot al","iv al","! \"","l im","Ġneed s","Ġag re","Ġdevelop ment","Ġa ge","ip le","2 1","Ġresult s","ĠA f","S h","Ġg un","ĠOb ama","ro ll","Ġ @","Ġright s","ĠB rit","Ġrun ning","Ġwas n","Ġp ort","Ġr ate","Ġpret ty","Ġtarg et","Ġsa w","Ġc irc","Ġwor ks","ic ro","al t","o ver","ww w","Th at","l ier","Ġevery one","ud e","Ġp ie","idd le","ra el","Ġr ad","Ġbl ock","Ġw alk","T o","ã ģ","n es","ĠA ust","a ul","ro te","ĠS outh","ess ion","op h","Ġshow s","Ġs ite","Ġj o","Ġr isk","cl us","l t","Ġin j","id ing","ĠS pe","Ġch all","ir m","Ġ2 2","itt ing","st r","Ġh y","L E","ke y","Ġbe gan","at ur","ashing ton","l am","ĠD av","b it","Ġs ize","ĠP ar","3 8","ourn al","f ace","Ġdec ision","Ġl arg","Ġj ud","re ct","Ġcontin ue","ĠO ct","ove red","ĠI nt","==== ====","Ġp arent","ĠW ill","Ġeas y","Ġd rug","ang er","Ġs ense","Ġd i","id ay","Ġener gy","ist ic","Ġass oci","ar ter","ob al","e ks","ĠE l","ur ch","Ġg irl","o e","it le","Ġ2 8","ĠC he","Ġrequ est","Ġso on","Ġh ost","k y","Ġst ates","om es","Ġm aterial","le x","Ġmom ent","Ġan sw","on se","Ġes pecially","Ġn orm","Ġserv ices","p ite","r an","Ġro le","4 4",") :","Ġc red","C l","____ ____","Ġm at","Ġl og","ĠCl inton","O U","Ġoff ice","Ġ2 6","Ġch arg","Ġtr ack","m a","Ġhe art","Ġb all","Ġperson al","Ġbuild ing","n a","s et","b ody","ĠBl ack","Ġincre ase","itt en","Ġneed ed","3 6","3 2","= \"","Ġl ost","Ġbec ame","Ġgrou ps","ĠM us","Ġw rote","ĠP e","Ġpro p","j oy","à ©","ĠWh ite","Ġde ad",". '","Ġhtt p","Ġwe bs","O S","Ġins ide","Ġwr ong","Ġstat ement","Ġ ...","y l","Ġfil m","Ġmus ic","Ġsh are","ific ation","Ġre lease","Ġfor ward","Ġst ay","Ġcomp ut","it te","s er","Ġorig inal","Ġc ard","Ġc and","Ġd iv","at ural","Ġfav or","O M","Ġc ases","us es","Ġse ction","Ġle ave","g ing","ov ed","ĠW ashington","3 9","ĠG l","Ġrequ ired","act ion","ap an","o or","it er","ĠK ing","Ġcount ries","ĠG erman","ll ing","Ġ2 7","3 4","Ġquest ions","Ġpr im","Ġc ell","Ġsh oot","Ġany one","ĠW est","Ġaff ect","ep end","Ġon line","ĠIs rael","ĠSept ember","Ġab ility","Ġcont ent","is es","Ġre ve","Ġl aun","Ġind ic","Ġfor ce","c ast","Ġso ld","av ing","f l","Ġso ft","Ġcompan ies","ce ed","Ġart icle","Ġa ud","Ġre v","Ġed uc","Ġplay ing","0 5","Ġhe ld","ct or","Ġrele ased","Ġf ederal","3 7","Ġad minist","Ġinter view","Ġinst all","Ġrece ived","Ġs ource","u k","P h","Ġser ious","Ġcre ated","Ġc ause","Ġim medi","Ġdef in","u el","ĠDep artment","ct ions","ĠC our","ĠN ow","z e","it es","it ution","Ġl ate","Ġspe ak","n ers","Ġleg al","ar i","ĠC or","Ġwe eks","Ġmod el","Ġp red","Ġex act","B C","ĠB y","IN G","os ing","Ġt akes","Ġreg ard","Ġopp ortun","Ġpr ice","Ġ19 8","ĠA pr","f ully","Ġor d","Ġproble ms","ru ction","h am","ĠC ount","le ge","Ġlead ers","E T","le v","Ġde ep","olog ical","es e","h aps","ĠS ome","Ġp ers","Ġcont ract","Ġrelations hip","s p","ou d","Ġb ase","4 8","m it","A d","anc ial","Ġcons um","Ġpot ential","Ġl angu","re m","et h","Ġrel ig","ress ed","6 6","Ġl ink","Ġl ower","ay er","ĠJ une","Ġf em","un t","er c","ur d","Ġcont act","Ġ ill","Ġm other","Ġest ab","h tt","ĠM arch","ĠB ro","ĠCh ina","Ġ2 9","Ġs qu","Ġprov ided","Ġa verage","as ons","Ġ201 1","Ġex am","l in","5 5","n ed","Ġper fect","Ġt ou","al se","u x","Ġbu y","Ġsh ot","Ġcol lect","Ġph ot","Ġplay ed","Ġsur pr","Ġofficial s","Ġsim ple","av y","Ġindust ry","Ġhand s","g round","Ġp ull","Ġr ound","Ġus er","Ġr ange","u ary","Ġpriv ate","op s","e es","Ġw ays","ĠM ich","Ġve h","Ġex cept","Ġter ms","im um","pp er","I ON","ore s","ĠDr agon","ou l","Ġd en","Ġperform ance","Ġb ill","c il","4 7","Ġen vironment","Ġex c","ad d","Ġwor th","Ġp ict","Ġch ance","Ġ201 8","b or","Ġspe ed","ict ion","Ġal leg","ĠJ apan","at ory","re et","Ġm atch","ĠI I","Ġst ru","ord er","Ġst e","Ġl iving","Ġst ruct","in o","Ġse par","her n","Ġresp onse","Ġen joy","Ġv ia","A D","um ents","ace book","Ġmem ber","ib r","iz ing","Ġto ol","ĠM on","ĠWh ile","h ood","ĠA ng","ĠD ef","Ġoff er","T r","a ur","Ġturn ed","ĠJ uly","d own","an ced","Ġrec ently","ĠE ar","Ġc e","ĠSt ar","ĠC ong","rough t","Ġbl ood","Ġhop e","Ġcom ment","ain t","Ġar ri","il es","Ġpartic ip","ough t","ri ption","0 8","4 9","Ġg ave","Ġse lect","Ġkill ed","sy ch","Ġgo es","i j","Ġc oll","Ġimp act","at ives","ĠS er","0 9","ĠAug ust","Ġb oy","d e","ĠD es","Ġf elt","U S","Ġexpect ed","Ġim age","ĠM ark","cc ording","o ice","E C","ĠM ag","en ed","h old","ĠP ost","Ġpre vent","N o","Ġinvol ved","Ġey es","Ġquick ly","A t","un k","Ġbeh av","Ġ ur","Ġl ed","c ome","e y","Ġcand id","Ġear lier","Ġfoc us","et y","P ro","led ge","ix ed","ill ed","Ġpop ular","A P","Ġset t","l ight","Ġvar ious","in ks","Ġlevel s","Ġro ad","ell ig","ab les","he l","itte e","ĠG ener","y pe","Ġhe ard","ic les","Ġm is","Ġus ers","ĠS an","Ġimpro ve","Ġf ather","Ġse arch","The y","v il","Ġprof ess","Ġkn ew","Ġl oss","Ġev ents","6 5","Ġb illion","0 7","0 2","ĠNew s","ĠA M","Ġco ver","w here","ens ion","Ġb ott","Ġare as","en ces","op e","ĠTw itter","a el","Ġget s","ĠGo ogle","Ġs n","i ant","Ġv ote","Ġnear ly","Ġinclud ed","Ġrec ogn","z z","m m","al ed","Ġhappen ed","0 4","Ġh ot","Ġwho se","Ġc ivil","Ġsu ff","o es","it iz","ĠSy ri","Ġresp ond","Ġh on","Ġfeat ures","Ġeconom ic","ĠApr il","r im","Ġtechn ology","Ġo ption","ag ing","Ġpur ch","R e","Ġl at","ch ie","is l","Ġrec omm","u f","Ġtr aining","Ġeffect s","Ġf ast","Ġ201 0","Ġocc ur","Ġwebs ite","Ġem ail","Ġs ens","e ch","Ġo il","Ġinf lu","Ġcurrent ly","ĠS ch","ĠAd d","Ġgo al","Ġsc ient","Ġcon v","1 00","em y","Ġdec ided","Ġtra vel","Ġm ention","L L","0 3","Ġe lection","Ġph one","Ġlook s","Ġsit uation","Ġc y","Ġh or","b ed","ĠCour t","a ily","av es","Ġqu ality","ĠCom p","w ise","Ġt able","Ġst aff","ĠW ind","et t","Ġtri ed","ide red","Ġadd ition","Ġb ox","Ġl ack","ar ily","Ġw ide","Ġm id","Ġbo ard","ys is","Ġant i","h a","Ġd ig","en ing","Ġd ro","C on","6 8","Ġsl ow","b ased","se qu","Ġp ath","E x","ak er","Ġwork ed","Ġp en","Ġeng ine","Ġlook ed","ĠSu per","ĠS erv","Ġvict im","U n","Ġproper ty","Ġint rodu","Ġexec ut","ĠP M","L e","Ġcol or","ĠM ore","Ġ6 0","Ġnet work","Ġd ate","c ul","id ge","Ġext ra","3 1","Ġs le","6 7","Ġw ond","Ġreport s","j ust","ĠAust ral","Ġcap ital","Ġen s","Ġcomm and","Ġallow ed","Ġpre p","Ġca pt","h ib","Ġnum bers","ch an","Ġf air","m p","om s","Ġre ach","W ith","t ain","Ġbro ad","Ġcou ple","ec ause","ly ing","ĠF eb","Ġsc reen","Ġl ives","Ġpri or","ĠCong ress","A r","Ġappro ach","Ġe mer","ar ies","ĠD is","s erv","ĠN e","Ġbu ilt","c ies","Ġre pe","Ġrul es","for ce","ĠP al","Ġfin ancial","Ġcons idered","ĠCh ar","n ces","ĠI S","Ġb rought","Ġb i","i ers","ĠS im","O P","Ġproduct s","Ġvis it","Ġdoc ument","Ġcon duct","Ġcomplete ly","in ing","ĠCal if","ib ly","Ġwr itten","ĠT V","em ents","Ġd raw","O ne","Ġpub lished","Ġsec ret","r ain","he t","ĠF acebook","ond ay","ĠU p","Ġsex ual","Ġth ous","ĠP at","Ġ ess","Ġstand ard","Ġar m","g es","ect ion","Ġf ell","Ġfore ign","an i","ĠFr iday","Ġreg ular","in ary","Ġincre ased","Ġus ually","Ġdem on","Ġd ark","Ġadd itional","ro l","ĠO f","Ġprodu ction","! !","und red","Ġintern ational","id ents","ĠF ree","rou p","Ġr ace","Ġm ach","Ġh uge","A ll","le ar","ove mber","Ġto wn","Ġatt ention","ĠO ff","y ond","ĠThe n","f ield","Ġter ror","ra z","ĠB o","Ġmeet ing","ĠP ark","Ġar rest","Ġf ear","Ġa w","ĠV al","or ing","' ,","Ġext reme","ar r","Ġwork ers","A fter","Ġ3 1","n et","am ent","Ġdirect ly","Ġpop ulation","ub e","ĠOct ober","ĠI N","ĠJan uary","5 9","ĠDav id","Ġc ross","ce mber","ĠF irst","Ġmess age","ir it","Ġn ation","Ġp oll","is ions","Ġansw er","n y","is ode","Ġcar ry","ĠRuss ia","Ġhe ar","eng th","ro y","Ġn atural","in ally","Ġdo g","m itted","Ġtr ade","Ġsub st","Ġmult iple","ĠAf ric","Ġf ans","Ġs ort","Ġgl obal","ic ation","ĠW ed","ar a","Ġa chie","Ġlangu age","ve y","Ġt al","Ġnecess ary","Ġdet ails","Ġs en","ĠS und","ĠRe g","ĠR ec","0 6","Ġs il","ress ive","Ġmed ical","un ch","orn ia","Ġu nd","f ort","oc ks","ĠM onday","ues day","c raft","7 7","ur t","Ġ ver","ĠH ill","Ġrece ive","Ġmor ning","es tern","Ġb ank","Ġs at","ir th","ĠH igh","Ġdev ice","ĠTH E","ĠCent er","Ġsaf e","Ġp le","ĠCanad a","Ġsystem s","Ġass ist","Ġsur v","Ġb attle","ĠS oc","vert is","S he","Ġp aper","Ġgrow th","Ġc ast","S c","Ġpl ans","ll ed","Ġpart s","Ġw all","Ġmove ment","Ġpract ice","im ately","Ġdis play","Ġsomet imes","om p","ĠP aul","ĠY es","k ing","5 8","o ly","Ġs on","Ġav oid","ok es","ĠJ ew","Ġto wards","as c","Ġ //","ĠK ore","Ġtalk ing","Ġcor rect","Ġsp ent","ic ks","i able","e ared","Ġter m","Ġwant s","om ing","Ġ ut","Ġdou b","Ġfor ces","Ġp lease","6 9","ĠN ovember","at form","ond on","Ġon es","Ġimmedi ately","ĠRuss ian","ĠM et","Ġde g","Ġparent s","C H","ĠAmeric ans","al y","ĠM od","Ġsh own","Ġcond itions","Ġst uff","Ġre b","ĠY our","Ġinclud es","n own","ĠS am","Ġexper ien","m ission","ĠE ven","augh t","Ġannoun ced","ĠRepublic an","Ġdeter min","Ġdescrib ed","ĠCount y","( )","Ġdo or","Ġchang ed","Ġne igh","ĠH ere","Ġcle an","Ġp an","ĠDe cember","ĠEurope an","ir ing","ap ter","Ġcl ub","ĠT uesday","Ġp aid","ĠN et","Ġattack s","Ġcharact ers","Ġal one","Ġdirect or","d om","Ġ3 5","Ġl oad","Ġr out","ĠCalif ornia","Ġfin ally","Ġr ac","Ġcont r","Ġexact ly","res h","p ri","ĠIs lam","Ġn ature","Ġcare er","Ġlat est","Ġcon vers","ĠS l","p ose","ci ent","ĠIn c","iv ity","8 8","ĠA tt","ĠM or","nes day","Ġwe ight","k en","Ġnot e","Ġteam s","Ġ \\","air s","ĠG reen","Ġh undred","on ent","Ġstre ng","Ġcons ist","ic ated","Ġreg ul","Ġl ic","ast ic","Ġt en","urs day","ellig ence","ous ly","ĠU K","B I","Ġcost s","Ġind epend","ĠA P","Ġnorm al","Ġh om","Ġob vious","Ġs we","Ġst ar","Ġread y","ac her","Ġimp lement","g est","Ġs ong","ĠG et","ĠL ab","Ġinterest ing","us ing","Ġg iving","ĠSund ay","Ġet c","Ġm iddle","Ġrem ember","r ight","os ition","ut ions","Ġm ax","4 6","Ġyour self","Ġdem and","Ġtreat ment","Ġd anger","ĠC ons","Ġgu y","ĠBrit ish","Ġphys ical","Ġrel ated","Ġrem ain","Ġcould n","Ġref er","Ġc itiz","b ox","EN T","bo ard","Ġin n","I G","er o","ĠSt reet","osp ital","ren ch","cher s","Ġst ra","O L","ag er","ĠA N","Ġeas ily","I A","en ge","in y","Ġcl os","ock ed","Ġus es","ĠC oun","I m","u ild","? ?","m ore","Ġan g","Ġwr ite","ol ute","5 7","Ġlead er","Ġread ing","< /","Ġaut om","est s","4 3","Ġleg isl","ĠG old","Ġdesign ed","ĠS T","ĠLe g","a res","Ġbe aut","ĠT ex","Ġappear s","Ġstru gg","ĠR om","Ġ 00","Ġcho ice","Ġparticular ly","ĠF rom","op er","ĠL ondon","ann ed","Ġallow s","ob ile","Ġdiffere nce","âĢ ¢","ĠV iew","ĠWed nesday","Ġal though","Ġrel ative","Ġapplic ation","ate ver","Ġare n","Ġmy self","Ġim ag","Ġdis e","Ġsoc iety","Ġfre qu","ĠEng lish","Ġpo or","ĠD ay","Ġwrit ing","Ġse ven","Ġstart ing","Ġb ud","Ġpr int","ĠTr ans","uf act","ĠSt ud","n ew","Ġcr im","Ġg ives","Ġco ol","a e","i ance","ĠGener al","Ġthink ing","Ġsa ve","Ġlim ited","ĠPart y","Ġmean ing","p en","ow ers","ĠJ ack","E M","Ġn ice","ru pt","Ġg as","Ġe ight","Ġfe et","Ġeff ort","Ġ ign","ic it","B l","co in","Ġop in","Ġbr ain","Wh ile","he st","ĠTh ursday","Ġwould n","augh ter","Ġtou ch","le ments","Ġstud ies","Ġcent er","c ont","or ge","Ġcomput er","Ġinvestig ation","P l","or ks","Ġ200 8","Ġincre asing","Ġst ore","Ġcom ments","Ġb al","m en","Ġdo ll","Ġl iber","Ġw ife","Ġlaw s","atur day","it ness","Ġmod ern","ĠS k","Ġadminist ration","Ġopportun ity","Ġs al","Ġpower ful","M y","Ġclaim s","ĠEar th","ord s","Ġt itle","Ġes c","n ame","N ot","om en","Ġbe yond","Ġc amer","Ġse ll","it ute","ear ch","Ġapp l","im ent","4 2","ĠAr t","Ġun f","Ġviol ence","ur g","ĠE ast","Ġcomp ared","Ġopt ions","Ġthrough out","Ġv s","ig r",". [","ac hes","7 8","Ġfil es","F L","E L","ar ian","ĠJ ames","ĠA ir","an ch","Ġdet ail","Ġpie ce","P S","Ġn amed","Ġeduc ation","Ġdri ve","Ġitem s","Ġstud ent","ic ed",": :","ic o","Ġth row","Ġsc ene","Ġcomple x","Ġ200 9","Ġpre c","ĠB re","7 9","Ġcon cept","Ġstat us","am ing","Ġd ied","Ġknow ledge","Ġbegin ning","O D","ru ary","Ġcertain ly","Ġgu ys","Ġsl ight","in n","ound s","Ġf ine","Ġf at","ic ations","Ġper haps","ĠA nt","Ġinc ome","Ġhtt ps","Ġmajor ity","port s","st on","Ġgreat er","Ġfe ed","ent ially","Ġsaf ety","Ġun ique","and om","Ġg one","Ġshow ed","Ġhist or","Ġcoun ter","i us","id a","Ġlead ing","i pe","Ġs end","ĠDon ald","er ve","Ġdef ense","ines e","Ġy es","ĠF ire","ĠMus lim","ra q","Ġcontin ued","os h","Ġprov ides","Ġpr ison","ĠP re","Ġhapp y","Ġeconom y","Ġtr ust","ag s","ĠG ame","Ġweap ons","um an","ĠC le","it ation","Ġanal ysis","ĠT imes","Ġsc ience","- >","Ġfig ure","Ġdis app","ent y","Ġsoft ware","Ġu lt","Ġoffic ers","N ew","I s","Ġrem ains","ĠInd ia","Ġp sych","ri ef","Ġc at","es c","Ġob serv","Ġst age","ĠD ark","Ġent er","ch ange","Ġpass ed","Ġdes pite","ĠO ut","Ġmov ie","r s","Ġv oice","m ine","ĠPl ay","Ġto ward","ĠT er","Ġreg ion","Ġval ues","or ters","Ġm ount","Ġoffic er","ĠO ther","b an","Ġh ous","w ood","ro om","I V","ĠS un","se e","ĠO ver","ro g","9 0","Ġl ay","ĠT ur","a wn","Ġpress ure","ĠS ub","Ġbook s","ed om","ĠS and","A A","ag o","Ġre asons","f ord","Ġactiv ity","U T","N ow","ĠSen ate","ce ll","n ight","Ġcall s","in ter","Ġlet ter","ĠR ob","ĠJ e","Ġcho ose","ĠL aw","G et","B e","Ġro b","Ġtyp es","Ġpl atform","Ġqu arter","R A","ĠT ime","Ġmay be","ĠC r","9 5","p re","Ġmov ing","Ġl if","Ġgo ld","Ġs om","Ġpat ients","Ġtr uth","ĠK e","ur ance","ant ly","m ar","Ġchar ge","ĠG reat","Ġce le","---------------- ----------------","Ġro ck","ro id","an cy","Ġcred it","a ud","B y","ĠE very","Ġmov ed","ing er","rib ution","Ġn ames","Ġstra ight","ĠHe alth","ĠW ell","Ġfe ature","Ġr ule","Ġsc he","in ated","ĠMich ael","ber g","4 1","il ed","b and","Ġcl ick","ĠAng el","on ents"," Ń","ĠI raq","ĠS aturday","Ġa ware","p art","Ġpat tern","O W","ĠL et","Ġgr ad","ign ed","Ġassoci ated","Ġst yle","n o","i ation","a ith","il ies","Ġst ories","ur ation","Ġindividual s","ĠâĢ ¦","m iss","ĠAss oci","ish ing","ab y","Ġsum mer","ĠB en","Ġ3 2","Ġar ch","ut y","ĠTex as","h ol","Ġfull y","Ġm ill","Ġfollow ed","ĠB ill","ĠInd ian","ĠSec ret","ĠB el","ĠFeb ruary","Ġjob s","Ġseem ed","ĠGo vern","i pped","Ġreal ity","Ġl ines","Ġp ark","Ġmeas ure","ĠO ur","I M","Ġbro ther","Ġgrow ing","Ġb an","Ġest im","Ġc ry","ĠS chool","Ġme chan","ĠO F","ĠWind ows","Ġr ates","ĠO h","Ġpos itive","Ġcult ure","ist ics","ic a","Ġh ar","y a","ite ly","i pp","Ġm ap","en cies","ĠWill iam","I I","ak ers","5 6","ĠM art","ĠR em","Ġal tern","it ude","Ġco ach","row d","D on","Ġk ids","Ġj ournal","Ġcor por","Ġf alse","Ġwe b","Ġsle ep","Ġcont ain","Ġst o","Ġb ed","iver se","ĠR ich","ĠCh inese","Ġp un","Ġme ant","k nown","Ġnot ice","Ġfavor ite","a ven","Ġcond ition","Ġpur pose",") )","Ġorgan ization","Ġchall eng","Ġman ufact","Ġsus p","ĠA c","Ġcrit ic","un es","uc lear","Ġm er","vent ion","Ġ8 0","Ġm ist","ĠU s","ĠT or","htt p","ol f","Ġlarg er","Ġadv ant","Ġrese ar","Ġact ions","m l","Ġke pt","Ġa im",", '","c ol","Ġbenef its","if ying","Ġact ual","ĠIntern ational","Ġveh icle","Ġch ief","Ġeff orts","ĠLe ague","ĠM ost","Ġwa it","Ġad ult","Ġover all","Ġspe ech","Ġhigh ly","Ġfem ale","Ġer ror","Ġeffect ive","5 4","Ġenc our","w ell","Ġfail ed","Ġcons erv","Ġprogram s","Ġt rou","Ġa head","5 00","vertis ement","I P","ĠF ound","p ir","Ġ %","Ġcr ime","and er","Ġloc ation","ĠI ran","Ġbehav ior","az ing","Ġr are","Ġem b","Ġca used","Ġsh ip","Ġact ive","Ġcont ribut","Ġg reen","Ġac qu","Ġref lect","ven ue","Ġf irm","Ġb irth","] .","Ġclear ly","Ġem ot","Ġag ency","ri age","Ġmem ory","9 8","S A","ĠSe e","ac ing","C C","Ġbig gest","Ġr ap","Ġbas ic","Ġb and","e at","Ġsus pect","ĠM ac","Ġ9 0","m ark","ist an","Ġsp read","am s","k i","as y","ra v","ĠR ober","Ġdemon str","r ated","Ġabs olute","Ġpl aces","Ġim pl","ibr ary","Ġc ards","Ġdest roy","Ġv irt","ve re","Ġapp eared","y an","p oint","Ġbe g","Ġtem per","s pe","ant ed","ear s","ĠD irect","Ġl ength","Ġbl og","am b","Ġint eg","Ġres ources","ac c","if ul","Ġsp ot","Ġfor ced","Ġthous ands","ĠMin ister","Ġqu al","ĠF rench","at ically","Ġgener ally","Ġdr ink","Ġth us","I L","od es","Ġappro pri","ĠRe ad","Ġwh om","Ġey e","Ġcol lege","Ġ4 5","ire ction","Ġens ure","Ġapp arent","id ers","Ġrelig ious","Ġmin or","ol ic","Ġt ro","ĠWh y","rib ute","m et","Ġprim ary","Ġdevelop ed","Ġpe ace","Ġsk in","st e","av a","Ġbl ue","Ġfam ilies","Ġ ir","Ġapp ly","Ġin form","ĠSm ith","C T","i i","Ġlim it","Ġres ist","........ ........","um n","Ġconf lic","Ġtw e","ud d","ĠT om","Ġl iter","qu e","b on","Ġha ir","Ġevent ually","Ġp us","Ġhelp ed","Ġag g","or ney","ĠApp le","Ġf it","ĠS ur","Ġpre m","Ġs ales","Ġsecond s","Ġstreng th","Ġfeel ing","¿ ½","Ġt our","Ġknow s","o om","Ġex erc","Ġsom ew","ï ¿½","> >","Ġsp okes","Ġide as","Ġreg ist","so ft","ĠD el","ĠP C","Ġpro pos","Ġlaun ch","Ġbott om","T H","ĠP lease","v est","it z","ĠIn ter","Ġsc ript","Ġr at","ar ning","Ġ il","ĠJ er","ĠA re","Ġwh atever","ok en","ci ence","Ġmod e","Ġag ree","Ġs ources","Ġinit ial","Ġrest rict","Ġwond er","us ion","## ##","ĠS il","vil le","Ġb urn","t w","as ion","Ġ £","Ġn or","u ing","Ġre ached","Ġs un","Ġc ateg","ig ration","Ġc ook","Ġprom ot","Ġm ale","Ġcl imate","Ġf ix","Ġalleg ed","U R","all ed","Ġim ages","C ont","ot a","Ġschool s","i os","Ġd rop","Ġst ream","ĠM o","Ġprevious ly","al ing","Ġp et","Ġdou ble","Ġ( @","ann el","Ġdef ault","t ies","Ġr ank","ĠD ec","ĠCoun cil","Ġweap on","Ġst ock","Ġanal y","ĠSt r","Ġpict ure","ĠPol ice","f erence","Ġcent ury","Ġcitiz ens","Ġon to","Ġexp and","Ġhe ro","ĠS ol","Ġw ild","Ġupd ate","Ġcustom ers","r ont","d ef","Ġl ik","Ġcrim inal","ĠChrist ian","S P","7 6","Ġle aving","Ġother wise","ĠD ist","Ġbas is","5 2","5 3","ic ip","ĠB er","Ġrecomm end","Ġfl oor","Ġc rowd","ol es","Ġ7 0","Ġcent ral","ĠE v","Ġd ream","Ġdown load","Ġconf ir","ĠTh om","Ġwind ow","Ġhapp ens","Ġun it","Ġt end","Ġs pl","Ġbec omes","Ġfight ing","Ġpred ict","ĠP ress","ĠP ower","Ġhe avy","ak ed","Ġf an","or ter","ate gy","B A","iz es","Ġsp end","H ere","Ġ200 7","Ġad op","ĠH am","Ġfoot ball","ĠP ort","od ay","5 1","amp ions","Ġtrans fer","h t","Ġ3 8","ter m","ac ity","Ġb ur","] ,","tern al","r ig","b ut","Ġthere fore","ĠB ecause","res p","re y","Ġm ission","S ome","Ġnot ed","Ġass um","Ġdise ase","Ġed it","Ġprog ress","r d","ĠB rown","oc al","Ġadd ing","Ġra ised","ĠAn y","Ġt ick","Ġsee ing","ĠPe ople","Ġagre ement","Ġser ver","Ġw at","Ġdeb ate","Ġsupp osed","il ing","Ġlarg est","Ġsuccess ful","ĠP ri","ĠDemocr atic","Ġj ump","ĠSyri a","Ġown ers","Ġoff ers","Ġshoot ing","Ġeff ic","se y","Ġha ven","ver se","te red","ĠL ight","im al","ĠB ig","Ġdef end","Ġbe at","Ġrecord s","% )","Ġsc en","Ġemploy ees","Ġdev ices","he m","Ġcom mer","ĠM ex","Ġbenef it","ĠPro f","Ġil leg","Ġsur face","ĠAl so","Ġh arm","ing ly","w ide","ĠA lex","Ġsh ut","ĠC ur","Ġl ose","p m","Ġchall enge","se mb","Ġst ation","Ġint elligence","Ġacc ur","ĠFl or","Ġrequ ires","ĠM al","b um","Ġh ospital","Ġsp irit","Ġoff ered","Ġprodu ce","ĠComm un","Ġcreat ing","Ġcr is","s pect","Ġend ed","Ġd aily","Ġvot ers","land s","i as","i h","on a","Ġsm art","ĠOff ice","ĠL ord","ri al","ĠIntern et","Ġcirc um","Ġextreme ly","' .","Ġopin ion","ĠM il","Ġg ain","B S","ĠF in","y p","Ġuse ful","Ġbud get","Ġcom fort","is f","Ġback ground","el ine","Ġep isode","Ġen emy","Ġtri al","Ġestab lish","d ate","ĠC ap","Ġcontin ues","Ġshow ing","ĠUn ion","w ith","Ġpost ed","ĠSy stem","Ġe at","ri an","Ġr ise","ĠGerman y","il s","Ġsign ed","Ġv ill","Ġgr and","m or","ĠEng land","Ġproject s","um ber","Ġconf erence","z a","Ġrespons ible","ĠAr ab","Ġlearn ed","âĢĶ âĢĶ","i pping","ĠGe orge","O C","Ġreturn ed","ĠAustral ia","Ġb rief","Q u","Ġbr and","ill ing","ab led","Ġhig hest","Ġtr ain","ĠComm ission","wh ile","Ġn om","cept ion","Ġm ut","ĠBl ue","Ġinc ident","v ant","8 6","ĠI D","Ġn uclear","7 4","ĠL ike","ĠR E","ĠM icro","l i","m ail","Ġcharg es","8 9","Ġad just","ad o","Ġear th","N A","Ġpr ices","P A","Ġd raft","Ġrun s","Ġcandid ate","ens es","Ġmanag ement","ĠPh il","ĠM iss","Ġte ach","g ram","Ġunderstand ing","a it","ic ago","A dd","ĠE p","sec ut","Ġsepar ate","Ġinst ance","Ġe th","Ġun less","**** ****","ĠF ore","in ate","Ġoper ations","S p","Ġf aith","g ar","ĠCh urch","ron ic","Ġconf ig","os ure","Ġactiv ities","Ġtrad itional","Ġ3 6","Ġd irection","Ġmach ine","Ġsur round","Ġp ush","un ction","ĠE U","Ġeas ier","Ġarg ument","G B","Ġm icro","Ġsp ending","iz ations","Ġthe ory","ad ow","Ġcall ing","ĠL ast","Ġd er","Ġinflu ence","Ġcomm it","Ġph oto","Ġun c","ist ry","g n","ast e","ack s","Ġdis p","ad y","d o","ĠG ood","Ġ `","Ġw ish","Ġreve aled","Âł Âł","l ig","Ġen force","ĠComm ittee","Ġche m","Ġmil es","Ġinterest ed","Ġsol ution","ic y","in ct","Ġ- >","ĠD et","Ġrem oved","Ġcomp ar","e ah","Ġpl ant","ĠS ince","Ġachie ve","Ġadvant age","Ġslight ly","b ing","Ġpl aced","u nder","201 5","ĠM ad","Ġt im","os es","Ġc ru","ĠR ock","Ġmost ly","Ġneg ative","Ġset ting","Ġprodu ced","Ġm ur","Ġconnect ion","ĠM er","Ġdri ver","Ġexecut ive","Ġass ault","Ġb orn","ĠV er","t ained","Ġstruct ure","Ġredu ce","Ġdec ades","Ġd ed","u ke","ĠM any","idd en","Ġle ague","S e","Ġjo in","Ġdis co","Ġd ie","c ks","act ions","Ġass ess","ag n","Ġgo als","our s","I R","Ġsen ior","ill er","m od","ip ment","oc ol","u y","ĠQ ue","Ġpart ies","ir gin","Ġle arning","it able","Ġstre et","Ġcamer a","A pp","Ġsk ills","b re","c ious","Ġcele br","ĠFr anc","Ġexist ing","Ġwill ing","l or","Ġ id","ĠSp ace","Ġcrit ical","ĠL a","ortun ately","Ġser ve","Ġc old","Ġspec ies","T S","Ġanim als","ĠB ay","Ġold er","ĠU nder","est ic","ĠT re","Ġte acher","Ġpre fer","v is","Ġth read","ĠM att","Ġmanag er","ãĥ »","Ġprofess ional","ĠV ol","Ġnot es","The se","ul a","Ġf resh","ent ed","u zz","ed y","clus ion","ĠR el","Ġdoub t","E O","Ġopen ed","ĠB it","Ad vertisement","Ġgu ess","ĠU N","Ġse qu","Ġexpl ain","ott en","Ġatt ract","ak s","Ġstr ing","Ġcont ext","oss ible","ĠRepublic ans","Ġsol id","Ġc ities","Ġask ing","Ġr andom","u ps","ur ies","ar ant","dd en","g l","ĠFlor ida","Ġdep end","ĠSc ott","Ġ3 3","Ġi T","ic on","Ġmention ed","Ġ2 000","Ġclaim ed","Ġdefin itely","ul f","Ġc ore","Ġopen ing","ĠCon st","wh ich","ĠT ra","A G","7 2","Ġbelie ved","ad a","Ġ4 8","ĠSec urity","yr ight","ĠP et","ĠL ou","Ġhold ing","======== ========","Ġ ice","Ġb row","Ġauthor ities","h ost","w ord","Ġsc ore","ĠD iv","Ġcell s","Ġtrans l","Ġneigh bor","Ġrem ove","u ct","Ġdist rict","ĠA ccording","Ġwor se","Ġconcern s","Ġpresident ial","Ġpolic ies","ĠH all","7 3","Ġh us","A Y","Ġ200 6","ĠJ ud","Ġindepend ent","ĠJust ice","ili ar","pr int","igh ter","Ġprotect ion","z en","Ġsu dden","h ouse","ĠJ es","P R","ĠIn f","Ġb ul","Ġ _","ĠServ ice","ĠP R","Ġstr ategy","ff ect","Ġgirl s","Ġmiss ing","oy al","ĠTe am","ul ated","Ġd at","Ġpolit ics","ab or","A ccording","Ġspe ll","Ġg raph","ort hern","T C","A b","Ġlab or","is her","Ġk ick","ĠiT unes","Ġstep s","pos es","Ġsmall er","E n","ber t","Ġro ll","Ġresear chers","Ġcl osed","Ġtrans port","Ġlaw y","________ ________","ĠCh icago","Ġas pect","Ġn one","Ġmar riage","9 6","Ġe lements","ĠF re","ĠS al","Ġd ram","F C","t op","e qu","Ġhe aring","Ġsupport ed","Ġtest ing","co hol","Ġmass ive","Ġst ick","Ġgu ard","is co","ph one","F rom","How ever","Ġb order","Ġcop y","ograph y","l ist","7 1","Ġown er","cl ass","ru it","r ate","ĠO nce","Ġdig ital","Ġt ask","ER S","Ġinc red","t es","+ +","ĠFr ance","Ġb reat","ow l","Ġiss ued","ĠW estern","Ġdet ect","Ġpart ners","Ġsh ared","ĠC all","Ġcan cer","ac he","rib e","Ġexpl ained","Ġhe at","{ \"","Ġinvest ment","ĠB ook","Ġw ood","Ġtool s","ĠAl though","Ġbelie f","Ġcris is","Ġg e","ĠM P","Ġoper ation","ty pe","~ ~","g a","Ġcont ains","ant a","Ġexp ress","ĠG roup","ĠJ ournal","k a","Ġam b","ĠUS A","Ġfind ing","Ġfund ing","h ow","Ġestab lished","ide os","Ġdeg ree","Ġdanger ous","ang ing","Ġfre edom","pp ort","out hern","Ġch urch","Ġc atch","ĠTw o","Ġpres ence","ĠGu ard","U p","Ġauthor ity","ĠPro ject","Ġbut ton","Ġcon sequ","Ġval id","Ġwe ak","Ġstart s","Ġref erence","ĠM em","\" )","U N","or age","ĠO pen","Ġcol lection","y m","g ency","Ġbeaut iful","ro s","Ġtell s","Ġwa iting","n el","Ġprov iding","ĠDemocr ats","Ġd aughter","Ġm aster","Ġpur poses","ĠJapan ese","Ġequ al","Ġturn s","Ġdoc uments","Ġwatch ing","R es","Ġr an","201 4","Ġre ject","ĠKore a","Ġvictim s","Le vel","ere nces","Ġw itness","Ġ3 4","Ġre form","com ing","Ġocc up","Ġc aught","Ġtra ffic","ad ing","Ġmod els","ar io","Ġserv ed","Ġb atter","u ate","ĠSecret ary","Ġagre ed","Ġtr uly","yn am","ĠR et","Ġun its","ĠRes earch","h and","az ine","ĠM ike","Ġvar iety","ot al","Ġam azing","Ġconfir med","Ġentire ly","Ġpurch ase","Ġe lement","Ġc ash","Ġdeter mine","D e","Ġc ars","ĠW all","â ĸ","Ġview s","Ġdrug s","Ġdep artment","ĠSt ep","u it","Ġ3 9","as ure","ĠCl ass","Ġc overed","ĠB ank","Ġme re","u ana","Ġmult i","Ġm ix","Ġun like","lev ision","Ġsto pped","Ġs em","ĠG al","ul es","Ġwe l","ĠJohn son","l a","Ġsk ill","Ġbec oming","ri e","Ġappropri ate","f e","ell ow","ĠPro t","ul ate","oc ation","Ġweek end","od ies","Ġsit es","Ġanim al","ĠT im","Ġsc ale","Ġcharg ed","Ġinst ruct","ill a","Ġmethod s","Ġc ert","Ġjud ge","ĠH el","Ġdoll ars","Ġstand ing","ĠS qu","Ġdeb t","l iam","Ġdri ving","ĠS um","ĠEd ition","Ġal bum","and on","I F","ĠU k","6 3","ad er","Ġcommer cial","es h","ĠGovern ment","Ġdisc overed","Ġout put","ĠHill ary","ĠCar ol","Ġ200 5","Ġab use","anc ing","Ġsw itch","Ġann ual","T w","Ġst ated","ag ement","in ner","Ġdem ocr","Ġres idents","Ġallow ing","Ġfact ors","od d","Ġf uck","em ies","Ġoccur red","ot i","Ġn orth","ĠP ublic","Ġinj ury","Ġins urance","C L","oll y","ã Ģ","Ġrepe ated","Ġar ms","ang ed","Ġconst ruction","Ġf le","P U","ic ians","Ġfor ms","ĠMc C","ant ic","Ġm ental","p ire","Ġequ ipment","Ġf ant","Ġdiscuss ion","Ġregard ing","k in","ar p","Ġch air","og ue","Ġpro ceed","ĠI d","O ur","Ġmur der","M an","Ġ4 9","as p","Ġsupp ly","Ġin put","Ġwe alth","liam ent","Ġpro ced","or ial","ĠSt at","ĠN FL","hen s","ĠInst itute","Ġput ting","ourn ament","et ic","Ġloc ated","Ġk id","er ia","r un","Ġpr inc","Ġ !","go ing","ĠB et","Ġcl ot","Ġtell ing","Ġprop osed","i ot","or ry","Ġfund s","g ment","ĠL ife","Ġb aby","ĠB ack","Ġsp oke","Im age","Ġear n","ĠA T","g u","Ġex change","ĠL in","ov ing","Ġp air","M ore","az on","Ġarrest ed","Ġkill ing","c an","ĠC ard","y d","Ġident ified","Ġm obile","Ġthan ks","ony m","ĠF orm","Ġhundred s","ĠCh ris","ĠC at","Ġtre nd","h at","ĠA v","om an","Ġelect ric","ĠW il","S E","O f","Ġrest aur","ot ed","Ġtr ig","Ġn ine","Ġb omb","Wh y"," ¯","Ġco verage","Ġapp eal","ĠRober t","ĠS up","Ġfin ished","Ġfl ow","Ġdel iver","Ġcal cul","Ġphot os","Ġph il","Ġpie ces","Ġapp re","k es","Ġr ough","D o","Ġpart ner","Ġconcern ed","Ġ3 7","ĠG en","C ol","ct ors","Ġ= >","st ate","Ġsuggest ed","ĠFor ce","C E","Ġher self","ĠPl an","w orks","o oth","ren cy","Ġcor ner","Ġhus band","Ġintern et","ĠA ut","em s","os en","ĠAt l","g en","Ġbal ance","6 2","Ġsound s","te xt","Ġar r","ov es","Ġmill ions","Ġrad io","Ġsat isf","ĠD am","M r","G o","S pe","Ġcomb at","r ant","ĠG ree","Ġf uel","Ġdist ance","Ġtest s","Ġdec re","ĠE r","Ġman aged","D S","Ġt it","Ġmeas ures","ĠL iber","Ġatt end","as hed","ĠJ ose","ĠN ight","d it","ĠN ov","ĠE nd","out s","Ġgener ation","Ġadv oc","y th","Ġconvers ation","ĠS ky","act ive","ce l","ri er","ĠFr ank","Ġg ender","Ġcon cent","Ġcar ried","and a","ĠV irgin","Ġarri ved","ic ide","ad ed","Ġfail ure","Ġmin imum","le ts","Ġwor st","Ġkeep ing","Ġint ended","Ġilleg al","Ġsub sc","Ġdetermin ed","Ġtri p","Y es","Ġra ise","Ġ ~","Ġfeel s","Ġpack age","ĠJ o","h i","201 6","re al","Ġf ra","Ġsy mb","M e","uck y","p ret","ĠK h","ĠEd it","ĠWe b","em ic","ĠCol or","Ġjust ice","I nt","Ġfar m","ck now","\" >","el ess","Ġredu ced","Ġ5 00","x x","ĠR ad","ĠW ood","Ġcl in","Ġhy p","il er","ur a","k ins","8 5","6 1","ĠThe ir","ĠM ary","Ġs an","Ġno vel","ĠWh o","Ġcap acity","Ġimp ossible","Ġpl ays","Ġmin ister","ij uana","ic ate","ĠS et","Ġf ram","Ġ ing","Ġcommun ities","ĠF BI","it a","Ġb on","Ġstr ateg","Ġinterest s","l ock","g ers","m as","ĠAN D","Ġconflic t","Ġrequire ments","Ġs ac","Ġoper ating","in i","rel ated","Ġcomm itted","Ġrelative ly","Ġs outh","¯ ¯","Ġaff ord","Ġident ity","Ġdec isions","Ġacc used","pl ace","Ġvict ory","o ch","i at","N ame","C om","t ion","ed s","Ġsee k","Ġt ight","ĠIm ages","Ġinit i","Ġhum ans","Ġfam iliar","Ġaud ience","Ġintern al","vent ure","Ġs ides","ĠT O","Ġd im","Ġcon clud","Ġapp oint","Ġenforce ment","ĠJ im","ĠAssoci ation","Ġcircum st","ĠCanad ian","Ġjo ined","Ġdiffere nces","ĠL os","Ġprot est","Ġtw ice","w in","Ġgl ass","ars h","ĠAr my","Ġexp ression","Ġdec ide","Ġplan ning","an ia","Ġhand le","ĠMicro soft","ĠN or","Ġmax imum","ĠRe v","Ġse a","Ġev al","Ġhel ps","re f","Ġb ound","Ġm outh","Ġstand ards","Ġcl im","ĠC amp","ĠF ox","cl es","Ġar my","ĠTe chn","ack ing","x y","S S","Ġ4 2","Ġbu g","ĠUk rain","ĠM ax","ĠJ ones","ĠSh ow","l o","Ġplan et","Ġ7 5","Ġwin ning","Ġf aster","Ġspe ct","Ġbro ken","T R","Ġdef ined","Ġhealth y","Ġcompet ition","htt ps","ĠIs land","ĠF e","Ġannoun ce","ĠC up","ĠInst ead","Ġcl ient","Ġposs ibly","se ction","ock et","l ook","Ġfin ish","Ġcre w","Ġres erv","Ġed itor","Ġh ate","Ġs ale","Ġcontro vers","Ġp ages","w ing","Ġnum er","Ġopp osition","Ġ200 4","Ġref uge","Ġfl ight","Ġap art","ĠL at","A meric","ĠAfric a","Ġapplic ations","ĠPal est","ĠB ur","Ġg ar","ĠSoc ial","Ġup gr","Ġsh ape","Ġspe aking","ans ion","a o","ĠS n","Ġwor ry","ĠBrit ain","P lease","rou d","Ġh un","Ġintrodu ced","Ġd iet","I nd","ĠSec ond","Ġfun ctions","ut s","ĠE ach","ĠJe ff","Ġst ress","Ġaccount s","Ġgu arant","ĠAn n","ed ia","Ġhon est","Ġt ree","ĠAfric an","ĠB ush","} ,","Ġs ch","ĠOn ly","Ġf if","ig an","Ġexerc ise","ĠEx p","Ġscient ists","Ġlegisl ation","ĠW ork","ĠS pr","à Ĥ","ĠH uman","Ġ è","Ġsur vey","Ġr ich","ri p","Ġmain tain","Ġfl o","Ġleaders hip","st ream","ĠIslam ic","Ġ 01","ĠCol lege","Ġmag ic","ĠPr ime","Ġfig ures","201 7","ind er","x ual","ĠDe ad","Ġabsolute ly","Ġfour th","Ġpresent ed","resp ond","rib le","Ġal cohol","at o","ĠD E","por ary","Ġgr ab","Ġvar i","Ġqu ant","ĠPh oto","Ġpl us","r ick","ar ks","Ġaltern ative","Ġp il","Ġappro x","th at","Ġobject s","ĠR o","ĠAnd roid","Ġsignificant ly","ĠR oad","k ay","R ead","av or","Ġa cknow","ĠH D","ĠS ing","O r","ĠM ont","Ġun s","pro f","Ġneg oti","ĠAr ch","ik i","Ġte levision","ĠJew ish","Ġcomm ittee","Ġmot or","Ġappear ance","Ġs itting","Ġstri ke","ĠD own","com p","ĠH ist","Ġf old","ac ement","ĠLou is","Ġbel ong","ĠâĢ ¢","Ġm ort","Ġprep ared","Ġ6 4","ĠM aster","Ġind eed","ĠD en","Ġre nt","T A","our ney","ar c","S u","9 7","Ġadv ice","Ġchang ing","Ġlist ed","Ġlaun ched","is ation","ĠP eter","is hes","Ġl ived","ĠM el","ĠSup reme","ĠF ederal","Ġ) ;","ruct ure","Ġset s","Ġphil os","u ous","Ġ ł","Ġappl ied","ĠN OT","Ġhous ing","ĠM ount","Ġo dd","Ġsu st","D A","ffic ient","Ġ ?","ol ved","Ġp owers","Ġth r","Ġrem aining","ĠW ater","L C","Ġca uses","ãģ ®","Ġman ner","ad s","Ġsuggest s","Ġend s","stand ing","f ig","ĠD un","id th","Ġg ay","Ġter min","ĠAngel es","M S","Ġscient ific","Ġco al","ap ers","b ar","ĠThom as","Ġsy m","ĠR un","th is","P C","igr ants","Ġmin ute","ĠDist rict","cell ent","Ġle aves","Ġcomple ted","am in","Ġfoc used","Ġmon itor","Ġveh icles","M A","ĠM ass","ĠGr and","Ġaffect ed","itution al","Ġconst ruct","Ġfollow s","Ġt on","re ens","Ġh omes","ĠE xt","ĠLe vel","r ast","ĠI r","Ġel im","Ġlarge ly","ĠJ oe","Ġvot es","all s","Ġbusiness es","ĠFound ation","ĠCent ral","Ġy ards","Ġmaterial s","ul ner","Ġgu ide","Ġclos er","um s","Ġsp orts","ed er","J ust","Ġtax es","8 4","ĠO ld","Ġdec ade","ol a","Ġv ir","Ġdro pped","Ġdel ay","it ect","Ġsec ure","ste in","le vel","Ġtre ated","Ġfil ed","ain e","Ġv an","Ġm ir","Ġcol umn","ict ed","e per","Ġro t","Ġcons ult","Ġent ry","Ġmar ijuana","ĠD ou","Ġapparent ly","ok ing","clus ive","Ġincre ases","an o","Ġspecific ally","Ġte le","ens ions","Ġrelig ion","ab ilities","Ġfr ame","ĠN ote","ĠLe e","Ġhelp ing","Ġed ge","ost on","Ġorgan izations","à ĥ","ĠB oth","hip s","Ġbig ger","Ġbo ost","ĠSt and","Ġro w","ul s","ab ase","Ġr id","L et","are n","ra ve","Ġst ret","P D","Ġv ision","Ġwe aring","Ġappre ci","Ġa ward","ĠU se","Ġfact or","w ar","ul ations",") (","Ġg od","Ġter rit","Ġpar am","ast s","8 7","Ġen emies","ĠG ames","F F","Ġacc ident","W ell","ĠMart in","T ER","Ġat h","ĠHe ll","Ġfor g","Ġve ter","ĠMed ic","f ree","Ġst ars","Ġexp ensive","Ġac ad","ra wn","ĠW he","Ġl ock","Ġform at","Ġsold iers","s m","Ġag ent","Ġrespons ibility","or a","ĠS cience","Ġrap id","Ġt ough","ĠJes us","Ġbelie ves","M L","Ġwe ar","le te","Ãĥ ÃĤ","ĠD ri","Ġcomm ission","ĠB ob","O h","ap ed","Ġwar m","ÃĥÃĤ ÃĥÃĤ","Ġ200 3","ort ion","Ġhas n","ust er","Ġun ivers","ĠI ll","Ġk ing","olog ies","9 4","ĠT em","ĠM os","Ġpat ient","ĠMex ico","ce an","ĠDe ath","ĠSand ers","y ou","ĠC ast","ĠComp any","pt y","Ġhappen ing","F P","ĠB attle","Ġb ought","A m","M od","U s","ut ers","ĠC re","ĠTh ose","Ġ4 4","is er","Ġs oul","ĠT op","ĠHar ry","ĠA w","Ġse at","ff ee","Ġrev olution","Ġ( \"","ĠD uring","et te","Ġr ing","Ġoff ensive","Ġreturn s","Ġv ideos","Ġdis cl","Ġfam ous","en ced","ĠS ign","ĠR iver","Ġ3 00","P M","ĠB us","ĠC H","Ġcandid ates","ard en","Ġpercent age","Ġvis ual","Ġthan k","Ġtrou ble","ner gy","Ġ200 1","Ġpro ve","ash ion","Ġen h","ĠL ong","U M","Ġconnect ed","Ġposs ibility","O ver","Ġexper t","Ġl ibrary","art s","ĠDirect or","Ġfell ow","9 2","ir ty","Ġd ry","Ġsign s","ĠL ove","Ġqu iet","f oot","Ġp ure","ĠH un","Ġf illed","ph as","ĠE lect","end ment","ĠEx pl","Ġun able","n s","m o","Ġv ast","ob e","Ġident ify","app ing","ĠCarol ina","g ress","Ġpro te","Ġf ish","Ġcircumst ances","raz y","ĠPh ot","Ġb odies","ĠM ur","Ġdevelop ing","ĠA R","Ġexperien ced","Ġsubst ant","ĠBo ard","es ome","Ġdom estic","Ġcomb ined","ĠP ut","Ġchem ical","ĠCh ild","Ġpo ol","ĠC y","Ġe gg","c ons","st ers","Ġh urt","Ġmark ets","Ġconserv ative","Ġsupp orters","Ġag encies","id el","O b","ur b","Ġ4 3","ĠDef ense","y e","ĠA p","du le","Ġtemper ature","Ġconduct ed","ĠCh ief","Ġpull ed","Ġf ol","L ast","ont o","os is","V ER","D es","ĠP an","F irst","Ġadv ance","Ġlic ense","r ors","ĠJ on","Ġimag ine","Ġhe ll","Ġf ixed","Ġinc or","os ite","ĠL og","ick en","] :","Ġsurpr ise","h ab","Ġc raft","ol t","ĠJ ul","Ġd ial","Ġrele vant","Ġent ered","Ġlead s","ĠA D","ĠCle an","Ġpict ures","ess or","Ġal t","Ġpay ing","P er","ĠMark et","Ġupd ates","am ily","ĠT ype","ĠH ome","Ġ5 5","semb ly","rom e","8 3","Ġgreat est","Ġhe ight","Ġhe av","ain ts","Ġlist en","as er","ĠS H","Ġcap able","ac le","Ġpers pect","in ating","Ġoff ering","ry pt","ĠDe velop","ab in","r c","Ġbr ight","al ty","ar row","Ġsupp l","ind ing","ack ed","gy pt","ĠAn other","p g","ĠVirgin ia","ĠL u","Ġpl anned","Ġp it","Ġswe et","T ype","ĠD i","Ġtyp ically","ĠFranc isco","Ġpro spect","ĠD an","Ġte en","re es","Ġsc hed","Ġh ol","Ġsc r","Ġlot s","l ife","Ġnews p","Ġfor get","ĠN one","ĠM iddle","ĠR yan","ed d","Ġse vere","Ġsu it","ll er","9 3","Ġcor respond","Ġexpl os","u ations","Ġfl ag","g ame","r id","Ġpr in","ĠD ata","Ġde ploy","ĠEn ter","su it","gh an","ĠM en","Ġthough ts","Ġmat ters","Ġad apt","ĠA ri","Ġf ill","Ġfor th","Ġs am","Ġ4 1","Ġpay ment","ĠH or","Ġsp ring","du c","Ġl osing","Ġbring ing","F O","al a","Ġdist ribution","he red","b our","ĠIsrael i","om a","Ġcomb ination","Ġpl enty","V E","C an","ĠH aw","Ġper man","ĠSpe cial","Ġto w","Ġsee king","Ġexam ples","Ġclass es","c r","Ġbe er","Ġmov es","ĠI P","ĠK n","Ġpan el","E ven","Ġproper ly","Ġr is","Ġpl ug","Ġestim ated","E very","Ġdef ensive","ag raph","Ġpre gn","Ġinst it","ĠV ict","Ġvol ume","Ġpos itions","Ġl inks","ĠPro gram","ĠWe ek","ag ues","Ġtrans form","k er","ĠC EO","Ġc as","Ġopp onent","Ġtwe et","ĠC ode","Ġsh op","Ġf ly","Ġtal ks","Ġb ag","Ph one","Ġa id","Ġpl ants","Ġ6 5","Ġatt orney","ar ters","qu est","ĠMag ic","Ġbeg ins","Ġmy ster","Ġenvironment al","Ġst orage","N N","Ġm arg","Ġs ke","Ġmet al","ell y","Ġord ered","Ġrem ained","Ġl oved","Ġprom pt","Ġupd ated","Ġexper ts","Ġwalk ing","Ġan cient","Ġperform ed","AT E","Ġne ither","i ency","Ġmanufact ure","ĠP ak","Ġselect ed","Ġm ine","Ġult imately","Ġexpl an","Ġlab el","ĠServ ices","ribut ed","Tr ump","Ġsy n","ĠU lt","S C","Ġme at","Ġg iant","ĠW ars","ĠO N","Ġad m","Ġinter pret","Ġeven ing","Ġev il","ĠB oston","ĠW ild","Ġ Ã","ĠBit coin","ĠAm azon","D r","ĠIn formation","Ġobvious ly","Ġadv anced","Ph oto","ol ar","Ġwe ather","Ġsymb ol","Ġso le","Ġpot entially","ost er","Ġorig inally","m un","3 00","az e","ess ions","Ġde ck","Ġst ood","Ġyou th","ĠB ern","R ep","ĠT est","Ġbas ically","ot ic","Ġinvol ve","ol it","ly n","S ee","Ġair craft","Ġconf irm","E W","Ġmess ages","ĠRich ard","Ġk it","Ġpro hib","Ġv ulner","is ters","Ġexist ence","Ġturn ing","ĠS P","Ġdes ire","Ġfl at","Ġm ent","se ason","ang es","Ġneighbor hood","ĠL ake","AT ION","Ġpoint ed","b ur","Ġinn ov","uc ks","U L","Ġprofess or","Ġexp ressed","A B","ic ious","Ġ200 2","ĠDe v","Ġs ession","Ġb are","s en","Ġdis s","ĠC ath","ĠP ass","ĠP oint","Ġdo ctor","or row","ail ed","ĠR ub","ĠD C","ĠChar l","p erson","Ġwrit er","igh ters","ure au","Ġob lig","Ġrecord ed","Ġbro ke","Ġord ers","il ty","Ġmot ion","in ity","l aw","ad ium","Ġimm igration","Ġcontr ast","Ġb att","Ġex cellent","Ġtechn ical","am i","Ġt un","Ġcl oud","ĠY ear","ge on","Ġcre ation","Ġstr ange","Ġa uth","Ġfor t","b orn","Ġext ent","ĠT oday","ĠCl ub","Ġr ain","Ġs ample","Ġaccept ed","Ġt act","Ġf ired","ĠS on","Ġstand s","Ġb oot","Ġ4 7","Ġstat ements","Ġvers ions","Ġse lling","ound ed","Ġ199 0","Ġwere n","ĠW atch","Ġexper iment","P ost","Ġret ail","ul ed","In st","un te","ãĥ ¼","Ġdep art","Ġb ond","i very","om pl","Ġre action","ĠSyri an","ĠP ac","app ed","ani el","D P","Ġres olution","Ġre act","Ġappro ved","on om","m ond","ĠO ffic","-- -","Ġrepl ace","Ġt ack","Ġsp ort","Ġch ain","Ġemer gency","r ad","ĠPalest in","Ġ4 6","Ġautom atically","Ġrout e","Ġp al","Ġb anks","ĠPar is","ĠMed ia","ro ad","ic ing","i xt","ist ed","Ġg rew","Ġco ord","ĠW here","om in","Ġsub s","� �","Ġ ±","Ġcorpor ate","Ġse lection","n oon","ĠRep ort","c s","clud ing","ord ers","anc he","ĠIt s","Ġslow ly","ĠE gypt","ĠA cc","Ġcol le","iqu es","E X","Ġattempt s","ur l","ĠC ross","Ġfind ings","ĠS C","ĠO R","Ġind ex","ens ity","ĠW ay","ĠL and","Ġsh ock","d is","Ġd ynam","Ġc art","m osp","S ince","i est","ĠB oy","Ġst orm","ĠCont in","201 3","he w","il it","Ġess ential","iqu id","O ther","ive red","Ġreason able","A ct","Ġsub sequ","ĠP ack","ĠF ort","Ġconsider ing","Ġun iversity","l og","Ġmar ried","Ġill ust","ĠTr ue","£ ı","Ġnumer ous","rast ructure","Ġserious ly","Ġrefer red","u a","Ġconsist ent","on na","ĠRe al","ru ption","ci ples","Ġfact s","9 1","ot es","er g","The n","Ġacc ompl","N ote","Ġre venue","Ġpass ing","Ġm al","e en","ĠY et","Ġg ather","ter day","ew ork","ĠA uthor","P e","Ġopt im","Ġr ub","Ġè £ı","Ġun known","st one","Ġun ion","ol ve","Ġopportun ities","Ġbrow ser","ĠW al","ĠC ost","Ġreport ing","st s","p et","Ġs and","Ġsudden ly","Ġsurpr ising","ĠV R","Ġsomew hat","ĠB as","ult ure","iz z","ĠC D","Ġchalleng es","Ġsett ings","Ġexperien ces","ĠF ull","Ġcan n","Ġrece iving","ES T","Ġj oint","Ġcult ural","Ġa st","8 2","as tern","ce ived","ĠC ru","Ġb ull","p ired","am m","Ġfac ing","p ower","Ġb oss","ĠH ol","Ġinst r","Ġincreasing ly","Ġsh ift","Ġstre ets","ĠWilliam s","ab b","Ġl ie","Ġl augh","ĠC a","P L","Ġadult s","Ġcustom er","Ġob tained","Ġsupport ing","ht ml","f ire","Ġdetail ed","Ġpick ed","ĠR ight","ld er","E E","st ood","ĠK im","Ġw ire","Ġs ight","Ġdevelop ers","Ġpers ons","Ġs ad","Ġc up","Ġwar ning","Ġboy s","l ong","Ġb ird","f o","Ġw al","Ġobserv ed","Ġz one","iven ess","Ġch annel","c ript","Ġref used","ĠAg ain","Ġsu c","Ġspokes man","ĠRe f","r ite","ou ston","ãĥ ³","ĠS her","Ġact s","ĠN ame","Ġstrugg le","ar ry","omet imes","Ġdisc rim","H T","Ġcateg ory","Ġreal ize","Ġemploy ee","ĠAf ghan","en ger","Ġgun s","ĠSte ve","ĠM ot","ĠO l","ok ed","Ġth ick","Ġfair ly","ill y","Ġsur ve","ĠM at","we ight","â Ķ","Ġtro ops","Ġag ents","Ġbatter y","Ġmot iv","à ¡","S ec","d en","o very","L S","Ġfl u","Ġconf ident","ĠO per","Ġem pty","Ġp hen","Ġse ctor","Ġexc ited","Ġrem ote","ap h","o en","Ġdestroy ed","Ġmor al","ĠH P","ĠR on","Ġd ress","ĠB at","Ġl it","ĠM S","Ġa f","H L","r um","is ms","Ġshould n","Ġsym pt","ĠTor onto","het ic","Ġcar bon","Ġinstall ed","Ġviol ent","Ġsol ar","j a","Ġpract ices","Ġr ide","ĠP enn","Ġimpro ved","Ġaud io","Ġbehav i","ĠP S","Ġe ating","D ata","ĠRe view","p ass","cl aim","u ated","ang ers","c hen","Ġproper ties","Ġany where","An other","Ġbl ow","ĠJack son","Ġp roud","Ġplan e","l ines","Ġsqu are","Ġpro of","ans as","Ġtalk ed","m akers","Ġs ister","Ġhold s","Ġres ident","Ġ= =","Ġresist ance","Ġspl it","Ġpro secut","Ġconf idence","res ents","Ġcut s","Ġexcept ion","Ġz ero","Get ty","Ġcop yright","Ġtot ally","orm al","ific ations","ĠAustral ian","Ġs ick","Ġ1 50","Ġhouse hold","Ġfe es","Ġdri vers","og en","ĠN Y","Ġnecess arily","Ġregul ations","ear ing","s l","Ġperspect ive","c are","ic ial","H is","Ġesc ape","Ġsurpr ised","ĠV an","ur rent","Ġv ac","8 1","ĠTh us","Ġem phas","ĠCh ampions","ĠI ce","Ġn arr","Ġhead s","Ġca using","b el","f ortunately","ĠM a","Ġtarg ets","ci pl","Ġafter noon","Ġadd s","ĠMay be","ĠF our","ess ed","ple te","Ġus ual","ch o","ing u","Ġwith d","ĠE nergy","ĠE conom","O O","Ġart icles","Ġinj ured","Ġman age","Ġexpl ains","Ġdi agn","R ec","at ures","Ġlink ed","Ġdiscuss ed","Ġexpl o","Ġocc asion","ath an","Ġopp osite","Ġfac es","Ġden ied","ĠK night","Ġn ut","Ġapprox imately","Ġdisapp oint","onym ous","ĠB est","ĠL o","ĠH y","ĠA ff","Ġvot ing","an while","ĠII I","Ġinstit utions","ag ram","ĠD aily","Ġdr ag","Ġnear by","Ġgu ilty","Ġcon ver","P re","s hip","Ġre ward","Ġphilos oph","ĠS S","u gh","Ġapp s","f riend","Ġu pper","Ġad vert","Ġs now","Ġfr ust","Ġour selves","F r","ĠD ie","amp ion","Ġdis miss","Ġc ere","Ġsign al","f rom","Ġ ).","Ġ5 2","Ġcr imes","it ors","est ival","use um","Ġcoun cil","ĠS aud","M ay","ĠG un","ic ian","et her","Ġsu fficient","ĠH en","so le","Ġhistor ical","ĠF ar","ĠT urn","Ġp in","Ġsuc ceed","m at","ly mp","Ġtrad ition","ĠO k","Ġc ro","Ġdesc ription","al le","Ġsk y","T e","Ġwide ly","Ġw ave","Ġdefin ition","ĠJew s","Ġcy cle","Ġref ere","Ġbr ings","us al","Ġal ive","Ġfrequ ently","Ġint ention","ĠCont rol","l v","y stem","Ġpriv acy","g ent","ren ce","ĠQu est","ĠChrist mas","Ġr ail","Ġco oper","Ġtest ed","ĠC apt","as ks","Ġcomfort able","Ġdel ivered","sc ape","Ġdep th","ĠG OP","Ġwrit es","Ġass ets","Ġsa v","im ents","Ġtrans ition","Ġart ist","ĠL ook","Ġl ob","Ġcomp onents","ar ity","Ġwalk ed","Ġro ot","Ġparticip ants","Ġnot iced","Ġres c","Ġn av","ĠAd minist","d a","ut ral","pl ate","Ġimport ance","Ġass ert","ious ly","c ription","Ġinj uries","ĠChe ck","Ġregist ered","Ġint ent","Ġmiss ed","ograph ic","Ġsent ence","oun ter","Ġassist ance","ev in","Ġdat abase","Ġbuild ings","Ġclass ic","Ġth inks","ĠOh io","P r","ug g","Ġfe e","p an","Ġeffect ively","Ġfac ility","Ġbe ar","Ġch apter","Ġdog s","ĠCol umb","Ġl atter","it ial","Ġad mitted","T V","ĠGe org","Ġpost s","\\ \\","Ġlawy er","Ġequ ival","Ġm and","Ġcontro lled","ĠW alk","ĠAnd rew","Ġmen u","am ental","Ġprotect ed","v a","Ġadminist r","or al","Ġre in","ĠS ar","Ġamount s","Ġn ative","ĠM oon","Ġrep resents","Ġab andon","Ġcarry ing","Ġt ank","m ary","Ġdecl ared","T ube","Ġh at","Ġpun ish","el lect","m es","Ġun iverse","ĠR od","ph y","Ġinf rastructure","Ġ5 1","Ġopp osed","ow nt","c a","ĠM ake","Ġhard ware","Ġco ffee","R el","b al","w orld","ĠS af","ĠSe a","in als","Ġown ed","Ġh all","ers ion","Ġdescrib e","ĠP ot","Ġport ion","Ġat mosp","Ġgovern ments","Ġdep ending","Ġoff ense","Ġtr ick","aw a","ĠL ine","ĠV is","ĠH ard","ĠOr ig","ĠCl ick","Ġdes k","ĠVal ley","ĠS ov","Ġmov ies","Ġrem ark","Ġm ail","Ġcons cious","Ġrul ing","ĠR ights","Ġmed ic","he nt","ĠW omen","> <","Ġrepl aced","ĠP rem","ĠTh anks","Ġre new","ĠB all","if orm","Ġsh ots","C omm","Ġar med","Ġconst ant","Ġt aste","Ġreal ized","Ġbu ff","Ġm o","Ġeffic ient","M ost","or ation","if ies","Ġcommun ication","Ġfl ood","Ġconsequ ences","Ġany way","ig g","ĠG M","ĠTh ank","Ġ iron","Ġev olution","ĠC op","tw itter","Ġ9 5","Ġrelationship s","ad el","ĠYou ng","Ġpropos al","ay ers","uild ing","ĠH ot","OR E","c os","Ġcoll abor","P G","ax y","Ġknow ing","Ġsupport s","ow ed","Ġcontrol s","Ġmere ly","um er","Ġath let","Ġf ashion","p ath","Ġg ift","Ġer a","AN D","Ġkind s","ĠKore an","Ġleg it","ul ous","Ġess entially","Ġthe rap","n ic","Ġsuff ered","Ġh ur","Ġprom ise","Ġex cess","Ġover w","Ġpr ime","ĠH ouston","er ry","ĠM s","R S","201 2","Ġst ores","ĠO lymp","Ġj ourney","Al though","S ub","ĠE duc","ĠCh apter","Ġrequest s","Ġconsum ers","Ġt iny","Ġis ol","ĠF air","b a","ĠY OU","Ġcr ash","ce ler","Ġemot ional","Ġgood s","Ġelect ed","Ġmod er","ĠLin ux","Ġbl ocks","Ġis land","ĠSoc iety","Ġelect ions","Ġbroad cast","Ġche ap","Ġn ations","Ġse asons","4 00","Ġwas te","ĠS at","Ġfield s","em ploy","Ġprof ile","Ġauth ors","AL L","ĠG ra","w est","ĠT y","Ġdeath s","Ġv acc","Ġfor med","Ġd u","Ġon going","ĠMuslim s","el f","ig ure","Ġass ume","ĠUkrain e","w ater","Ġco ast","Ġvot ed","g or","ĠA S","ĠMich igan","az a","ĠAr m","i ro","Ġf lex","as ters","' '","Ġwel come","ar l","Ġloc ations","ig ation","ĠF il","Ġbu ying","Ġarch itect","Ġhard er","ĠC ub","Ġinter face","Ġrestaur ant","Ġdisco ver","Ġex ceed","Ġfav our","ger y","Ġd uty","Ġp itch","ad or","ĠM ach","b oy","Ġrespond ed","Ġext ended","her s","M any","ra id","if er","ĠIn s","S er","Ġmed ium","s he","ĠS ports","Ġmag azine","ut ation","Ġlim its","ĠG all","Ġex ternal","raz il","Ġyoung er","t le","Ġrem ind","ĠC ON","Ġimmedi ate","Ġh idden","Ġvol unte","Ġsim pl","od cast","Ġph ase","d r","Ġpl ot","Ġexp osure","R I","og rap","v in","an ish","ĠAc ad","ĠEng ine","Ġexp ansion","ĠP ay","Y our","Ġpus hed","ĠE ll","ĠHe ad","Ġmarket ing","ĠA C","k et","Ġh its","Ġg ro","ĠA ge","ĠSc ot","] [","Ġst im","Ġi Phone","Ī Ĵ","Ġn arrow","ĠGet ty","ĠTur key","Ġperfect ly","Ġen able","ut ch","Ġprec ise","Ġreg ime","Ġsh if","Ġcomp ens","g un","d iv","Ġch osen","ĠK en","An y","Ġtre es","Ġrecomm ended","ĠR en","u able","ĠH T","F ollow","E G","ĠH and","ĠK enn","Ġarg uments","Ġex ists","Ġb ike","ĠCons erv","Ġbre aking","ĠG ar","Ġc razy","Ġvirt ual","ay lor","ix el","Ġ19 80","Ġper mission","ĠSer ies","Ġconsum er","Ġclose ly","c alled","Ġ5 4","Ġhop es","Ġar ray","ĠW in","ĠLab our","Ġsp ons","ĠI re","Ġp ow","Ġread ers","Ġemploy ment","Ġcreat ure","Ġresult ing","Ġaccur ate","Ġmom ents","Ġarg ued","Ġp ed","D uring","Ġ5 3","ĠT al","Ġs ought","Ġsuff ering","Ġ icon","le e","Ġ( $","al ian"," °","Ġp ra","Ġbon us","( \"","k o","Ġact ing","D E","f all","Ġcompar ison","Ġsm ooth","ĠN AS","u pp","ĠJose ph","ep ing","ĠT ake","ĠM id","Ġs ending","f ast","ĠF all","Ġdeal ing","us er","ĠOr gan","C o","Ġatt ached","Ġse es","% .","Ġtyp ical","AR T","Ġfind s","ĠAs ia","um in","ĠC ore","ĠE nt","in ent","u ce","ĠBl ood","ĠN ever","Ġem ails","Ġhigh light","Ġconf ront","at us","ut ed","Ġun us","Ġtop ic","ĠAd am","Ġb le","at i","Ġunder stood","S et","st ruct","T P","Ġm ob","a a","ĠSt art","pect ed","se ll","Ġded icated","ĠC A","u an","Ġsong s","esc ription","Ġte ch","Ġr ape","Ġas ide","Ġgr ant","Ġ5 6","s ub","Ġarg ue","Ġcont aining","Ġsche dule","Ġliber al","Ġpublic ly","Ġheav ily","ĠU t","in er","ĠS ection","ĠC are","we et","l s","D is","âĶ Ģ","ĠF ollow","B ack","ĠI T","Ġb es","j i","ĠH it","est ed","Ġevery body","ĠSw ed","Ġfem in","Ġfac ilities","Ġcon ven","C omp","ĠO S","c ore","Ġan x","Ġdiv ision","ĠC am","ĠSt an","m ates","Ġexpl ore","pl om","Ġsh ares","pl oad","an es","Ġide al","et ers","ĠB ase","Ġpl astic","Ġdist inct","ĠNet work","ĠSe attle","Ġtrad ing","ens us","int end","Ġex hib","Ġinit ially","ĠF ood","Ġthous and","ĠBus iness","act er","Ġpar agraph","Ġrough ly","Ġw ww","Ġcreat ive","ĠCon f","Ġconsum ption","Ġfil ms","ag an","Ġob tain","Ġt all","Ġt or","Ġacknow led","Ġg rown","al o","K E","Ġ4 00","end ers","t aining","U G","Ġsu icide","Ġwat ched","ĠL ist","al i","re hens","Ġsurround ing","Ġp ip","Ġf lying","ĠJ ava","ord an","Ġserv ing","in ations","p ost","Ġsh o","A v","Ġj ail","z y","Ġ199 9","Ġ< /","Ġliter ally","ĠS ir","Ġexp osed","Ġl ies","st ar","Ġb at","Ġear ned","ĠD ig","Ġspec ified","ĠSe ason","Ġdeg rees","Don ald","Ġcent re","Ġsh aring","Ġwin ter","ĠC O","C he","Ġ Î","M P","Ġun w","Ġfew er","ĠM ir","Ġsomew here","ĠK ey","Ġattack ed","ĠK ir","Ġdom ain","Ġstrong er","Ġ9 9","Ġpen alty","I d","Sc ript","Ġdecl ined","Ġne ck","Ġfra ud","Ġcur rency","Ġr ising","R C","â̦ â̦","H z","Ġt ab","Ġtal ent","n am","ĠN BA","Ġvill age","Ġleg s","ĠN ext","E d","Ġac id","Ġhy d","8 00","Ġinvol ving","ĠIm age","ĠBe fore","F l","Ġyes terday","S ource","Ġterror ist","Ġsu p","Ġsy nt","ĠSaud i","Ġw est","Ġr u","b urg","Ġvis ible","Ġstru ck","r ison","Ġaw esome","Ġd rawn","Ġansw ers","ĠG irl","ĠR am","Ġthreat s","Ġdef eat","os it","Ġv ent","atur ally","Americ an","end a","ĠH oly","Ġr um","% ,","c ase","ĠHist ory","ĠYou Tube","Ġsit uations","ĠD NA","S te","Ġsa ved","It em","Ġrec ip","olog ist","Ġfac ed","Ġel ig","O nce","ĠL i","u h","Ġmist ake","ĠDiv ision","ĠB ell","Ġsympt oms"," ®","Ġdom in","Ġfall ing","Ġend ing","as hes","Ġmat ches","ĠOn line","Ġexplan ation","D ef","red it","Ġany more","ĠT otal","ĠF OR","us hed","Ġlet ters","Ġris ks","ĠO K","Ġreported ly",": \\","Ġpl ate","Ġsubject s","Ġattempt ed","if ier","ian a","Ġunlike ly","ĠTh ough","um a","ĠIn vest","ĠPr in","ic an","ĠD ar","ĠColor ado","au g","Ġve get","a os","ri a","Ġshe l","Ġmark ed","Ġ( )","Ġsp r","p o","ĠL ink","Ġdef e","ĠJ r","Ġthem e","Ġpass ion","ĠP en","Ġinf o","iz er","Ġsh it","ĠC ivil","ap se","c re","Ġpo ly","Ġcomp onent","ĠChar les","ĠIre land","ĠPro v","Ġdo ctors","Ġgr anted","Ġpain t","Ġhon or","Ġsm oke","Ġpay ments","Ġprim arily","ĠKing dom","r ich","ate ll","Ġde als","Ġsched uled","Ġfund amental","Ġprote in","Ġnewsp aper","Ġcl ients","yth on","ĠD ate","h us","Ġfeed back","Ġstret ch","Ġc ock","Ġhot el","ĠQue en","Ġsu gar","Ġj u","Ġmil k","Ġappro val","ĠL ive","Ġequival ent","ef ully","Ġins ert","z ona","Ġext ension","d ri","J ohn","Ġacc omp","S m","ĠF und","Ġconst antly","Ġ` `","Ġgener ated","ĠA ction","ĠP sych","ĠT ri","Ġrecogn ize","Ġv ary","ph a","ĠR a","d f","et ch","ĠSov iet","Tw o","Ġpattern s","Ġprof ession","an ing","T ime","ĠL im","Ġcol ors","ĠA z","ĠT R","Ġinf ect","Ġphen omen","Ġshe ll","Al so","Ġput s","Ġdel ivery","Ġbro wn","Ġprocess ing","Ġlight s","ess age","ĠBro ok","ĠA ud","l ation","Ġindust rial","L ike","ĠB razil","rou s","ES S","ĠL uc","Ġsome how","Ġ8 5","Ġpro port","Ġpolit icians","Ġindic ate","Ġh ole","Ġtechn iques","Ġcompet itive","Ġph r","Ġv o","ist ent","ĠD ream","Ġcamp us","Ġaspect s","Ġhelp ful","Ġsh ield","or se","Ġtrig ger","m al","Ġ5 8","Ġt ort","Ġperson ally","Ġt ag","Ġkeep s","ĠV ideo","Ġben ch","Ġg ap","a ire","Ġe ast","Ġrec overy","per ial","Ġprof it","ĠM ic","Ġ5 7","Ġcol on","Ġstrong ly","st yle","Ġalleg ations","h an","Ġrep orters","j o","r ine","arg et","and al","Ġ0 3","Ġfl ash","tr ans","Ġstr ict","Ġpark ing","ĠPak istan","Ġl i","Ġwe ird","ĠE ric","Ġreg ions","ĠJ un","Ġint ellect","ĠW H","od ing","rib utes","up id","ĠT it","Ġf inger","or ia","Ġe lev","ĠF ield","Ġcon clusion","; ;","Ġfeel ings","Ġext ensive","Ġm ixed","Ġne uro","v y","Ġhar ass","ĠC irc","ou ch","Ġterrit ory","Ġsuccess fully","M ar","Ġing red","Ġoverw hel","Ġl ayer","V iew","Ġall ies","ill ance","ĠTh ree","Ġb unch","Ġnorm ally","Ġnet works","Ġsac r","ĠC IA","b les","Ġch ose","Ġopp onents","Ġregard less","Ġfr anch","Ġpre f","ĠP o","Ġbr idge","ann a","ĠSil ver","Ġw age","p age","ri or","Ġrad ical","ĠL ittle","Ġman ip","Ġsecret ary","Ġg ang","D R","F A","Ġdec ent","ĠSp irit","Ġun cle","ĠDevelop ment","Ġinvest ors","Ġwall s","Ġpub lish","Ġgener ate","iss ions","c ar","Ġprom ote","Ġcut ting","Ġche st","Ġdrink ing","Ġcollect ed","Ġ7 2","Ġhop ing","Ġem br","gor ith","Ġwar ned","Ġinstruct ions","O G","ĠD id","ĠAg ency","Ġg ear","Ġcritic ism","ĠF urther","Ġut il","ann y","R ed","Ġcoun sel","ĠAs ian","Ġredu ction","p ool","Ġteach ing","Ġdeep ly","i y","Ġestim ates","Ġcho ices","Ġperman ent","in em","ke l","Ġf asc","p se","f ile","ĠL ow","ĠP erson","Ġt ournament","st al","Ġm el","U ST","ĠR ay","az i","V al","Ġcont ained","ĠH olly","Ġw ake","Ġreve al","Ġprocess es","ĠIS IS","Ġ0 9","Ġbl ind","Ġste el","ĠB ad","Ġcare fully","app y","ro it","Ġg aming","Ġhous es","ĠC oll","Ġtr uck","er m","Ġsc ored","Ġocc as","ret urn","b ound","v ar","Ġsh arp","Ġaf raid","ĠE X","am ber","c ific","Ġsche me","N C","ĠPol it","Ġdecl ine","Ġ199 8","Ġpus hing","Ġposs ession","Ġpriv ile","Ġteacher s","Ġy ield","H A","ĠDav is","it led","#### ####","Ġr ig","ĠD aniel","ac on","Ġh ide","ut en","Ġcolle agues","Ġprin ciples","Ġl oud","Ġs in","ĠDem on","Ġst one","Ġ0 2","Ġt aught","Ġter rible","Ġst uck","ĠPol icy","te en","Ġimplement ation","ĠB BC","ĠAP I","Ġwhe el","all as","Ġch ampions","ol ars","play er","Ġrepeated ly","ĠSt ill","Ġlik es","ast y","es ter","ĠCath olic","R L","Ġb ath","Ġno ise","t itle","Ġn orthern","P art","Ġmag n","Ġf ab","ĠAs h","Ġdis pl","Ġtick et","Ġm urd","Ġalong side","ĠMus ic","Ġr iver","ĠSte el","ĠC L","ĠPl ayer","ĠM ult","ow ing","re p","s ize","Ġt ur","ĠGeorg ia","isc al","ra ction","Ġc able","Ġ5 9","Ġw ins","Ġup coming","Ġsurv ive","Ġins pired","ĠEduc ation","Ġstat istics","ĠF oot","iam i","Ġy ellow","ĠP age",". -","ĠH as","Ġur ban","Ġa x","es sel","\\ \"","Ġquarter back","Ġreg ister","ĠLab or","Ġab ilities","ĠF amily","Ġvar iable","ĠPr ice","Ġcont em","Ġth in","ĠE qu","d ata","Ġg otten","Ġconst it","Ġas ks","Ġt ail","Ġexc iting","ĠE ffect","ĠSp anish","Ġencour age","ins on","ĠA h","Ġcommit ment","C S","Ġr ally","Ġ: :","Ġsubs id","Ġsp in","Ġcapt ured","201 8","Ġinn oc","Ġalleged ly","ĠC ome","Ġart ists","ĠN umber","Ġelect ronic","Ġreg ional","ap es","Ġw ra","Ġmy th","pr ise","ĠM iller","ĠC reat","ĠEp isode","b ell","Ġdirect ed","Ġext ract","Ġs orry","Ġv ice","ag ger","ĠSu pport","Ġ6 6","ĠI ron","Ġwonder ful","Ġg ra","N et","ion e","E ng","Ġsh ips","ik es","ĠK evin","it ar","Ġactiv ists","tr ue","ĠAri zona","ent h","ĠDes pite","ĠS E","Ġha bit","ern el","Ġin qu","Ġab ortion","Ġv oid","Ġexpl icit","Ġeng aged","Ġang ry","Ġr ating","Ġfr ag","b ro","ick ing","d ev","Ġwor ried","Ġob ser","Ġap artment","ĠG T","Ġest ate","ĠConst itution","em on","ĠS now","Ġcount y","Ġdis ag","ĠStep hen","Ġimm igrants","w ind","ĠN ations","Ġfol ks","O ut","Ġg all","Ġtarget ed","Ġst ead","ĠB on","ĠL ib","Ġinform ed","Ġ12 0","ch ain","idel ines","or ough","Ġdri ven","Ġregular ly","Ġbas ket","Ġprinc iple","oc ument","Ġst un","ib ilities","ĠRom an","ĠAb out","Ġal ert","Ġdemocr acy","Ġrepresent ed","H S","c ers","p arent","Ar t","p ack","Ġdi plom","re ts","ĠN O","Ġcapt ure","ĠAd v","Ħ ¢","Ġannounce ment","ĠL ear","Ġh ook","Ġpur s","ĠS uch","ĠC amer","Ġrefuge es","ĠV e","P ol","Ġrecogn ized","l ib","Ġhad n","A ss","Ġpil ot","us hing","Ġreturn ing","Ġtra il","ĠSt one","Ġrout ine","Ġcour ts","Ġdes per","Ġfriend ly","ĠIt aly","Ġpl ed","Ġbreat h","Ġstud io","N S","Ġimp ressive","ĠAfghan istan","Ġf ing","Ġd ownt","ink ing","ĠR og","i ary","col or","se x","ar on","Ġf ault","ĠN ick","D own","ĠR ose","ĠS outhern","X X","is odes","L ist","6 00","Ġout come","er r","Ġelse where","Ġret ire","Ġp ounds","ĠGl obal","Pe ople","Ġcommun ications","Ġlo an","Ġrat io","ĠEm pire","Ġg onna","Ġinv ent","D F","Ġ19 70","ĠComm on","p at","Ġprom ised","Ġd inner","ĠH om","Ġcreat es","Ġoper ate","ver ty","ĠJ ordan","et ime","Ġsust ain","R eg","Ġincred ible","im a","Ġwar rant","Ġm m","A tt","Ġlaw suit","Ġreview s","it ure","ĠS ource","l ights","ĠF ord","Ġ6 3","g roup","st ore","Ġfeat ured","Ġfore ver","Ġpo verty","ĠP op","ĠC NN","az z","ab is","ach ing","Ġl aid","ĠSu pp","Ġfil ter","en a","ĠCommun ity","Ġcreat ures","u ction","ĠR oyal","Ġassoci ation","ĠCon nect","ĠBr ad","âĸ Ī","l ers","the re","ĠG i","Ġval uable","AC K","ĠT aylor","Ġl iquid","ĠAtt orney","ĠCar l","ĠF inal","ag a","ĠWil son","B ecause","ĠProf essor","ak a","Ġincred ibly","r ance","! )","R ef","s k","Ġsol utions","Ġatmosp here","Ġbl ame","um es","ĠN ob","C A","um ps","r ical","ĠPut in","ĠD est","or ic","ĠP A","Ġrespect ively","w an","Ġfif th","â Ħ¢","ĠC ry","Ġgovern or","res ident","Ġpurch ased","Ġh ack","Ġint ense","ob s","Ġorig in","Ġdef ine","Ġcare ful","** *","Ġshould er","Cl ick","Ġt ied","Ġdest ruction","ou red","Ġno body","Ġh o","ĠEx per","Ġt ip","\" ;","Ġtechn ique","Ġj ur","ĠP ok","b ow","Ġleg end","Ġacc ord","Ġbus y","ĠInt el","Ġh ang","ak i",". ]","âĢĶâĢĶ âĢĶâĢĶ","Ġsur gery","Ġrep rodu","Ġun iform","Ġscen es","c ode","Ġ6 2","l isher","ĠH ave","ph ia","Ġcry pt","Ġrec on","Ġsc ream","Ġadop ted","Ġsc ores","N e","ĠIt alian","in cluding","B O","Ġindic ated","Ġent ertain","G u","T ext","i el","Ġtw enty","Ġeng age","off s","ĠPac ific","Ġsm ile","Ġperson nel","Ġto ler","Ġdo ors","Ġt one","Ġmach ines","Ġent ering","ten ance","C O","ĠJer sey","Ġfore st","Ġhor se","Ġcompl aint","ĠSpr ing","y o","ĠPl us","ed ing","ĠRet urn","qu arters","ial s","c ow","Ġacad emic","Ġf ruit","Ġ199 6","og ether","Ġw ine","Ġpur su","ĠSte ven","Ġlic ens","Wh o","Ġclot hes","re ction","Ġsqu ad","Ġst able","Ġr aw","z ens","St ar","ut ies","anc er","Ġke ys","ĠM u","Ġcompl icated","ig er","ĠTe xt","Ġabs or","Ġ6 8","Ġfun ny","Ġrel ief","ĠL ew","ĠC ook","Ġch art","Ġdraw ing","G E","Ġmod ule","ĠB ull","I LL","Ġs alt","0000 0000","il le","Ġres ource","aw ay","adel phia","ĠB ru","Ġ6 7","Ġsome body","Ġparticip ate","Ġro se","we red","Ġmus cle","Ġcons ent","Ġcontin uing","ĠGuard ian","ĠOr der","reg on","Ġre ar","Ġprov ision","Ġlik ed","ri ent","Ġb ra","Tr ans","Ġmeet ings","Ġto x","Ġcon vent","Ġaut o","Ġrec ording","ĠSo ft","00 1","ĠR oll","Ġprogram ming","Ġp ic","Ġprov ed","Ġst ab","ĠA st","Ġca ption","ul ating","ĠAtt ack","Ġnew ly","Ġ199 7","f r","Ġdis cipl","ĠGree k","Ġed ition","ĠDo es","ĠB ox","if le","ack et","Ġpass es","Ġgu est","Ġac celer","it als","U D","Ġaut hent","ĠR est","ov al","t a","u ine","Ġarm or","ĠT own","Ġcomp at","Ġinc hes","Des pite","Ġass ign","he rent","Ġprep are","ĠM eg","oc key","Ġdep ends","Ġtrack s","w atch","Ġl ists","ĠN orthern","Ġal ter","re c","ĠE astern","Ġcond em","Ġevery where","? '","Ġaff ili","Ġf ought","\": {\"","Ġm ac","it arian","Ġsc ope","ĠA L","aw s","ar ms","Ġqu e","Ġenjoy ed","nes ota","Ġagg ressive","ĠSt ory","ĠI V","Ġrec ipe","Ġrare ly","ĠMed ical","val ue","ang el","ay ing","omet hing","Ġsub section","Ġs outhern","Ġfrequ ency","re te","roll ed","ult s","ĠN ic","Ġbeh alf","Ġsequ ence","ab et","Ġcontrovers ial","Ġcomp rom","Ġwork er","Ġmain ly","Ġal gorith","ĠM ajor","or ce","g ender","Ġorgan ized","Ġf ake","Ġconclud ed","ĠE D","ĠEx ec","r age","Ġch ances","ber ry","ĠTr ad","Ġconfig uration","Ġwithd raw","Ġf ro","ud es","ĠBro ther","ĠB rian","Ġtri es","Ġsam ples","Ġb id","ĠGold en","Ġphot ograph","if est","ĠD O","ĠPar liament","******** ********","R em","Ġcont est","Ġsign ing","p x","ĠZ eal","âĶĢ âĶĢ","E ar","Ġex it","Be fore","ĠCor por","n ull","mon th","Ġrac ial","ott ed","ĠV eg","ĠRe uters","Ġsw ord","ps on","ĠRom ney","a ed","Ġt rib","Ġin ner","Ġprot ocol","ĠB i","ĠM iami","ever al","p ress","Ġsh ipping","ĠAm endment","ĠHow ard","con nect","ĠD isc","ĠJ ac","iam ond","ĠThere fore","s es","ĠPrin cess","ĠUS B","ĠAn th","Ġsurve illance","Ġap olog","Ġ6 1","ow a","Ġf ulf","j s","Ġl uck","ust ed","Ġ §","n i","Ġant icip","em an","Ġwin ner","Ġsil ver","ll a","ic ity","Ġunus ual","Ġcr ack","Ġt ies","e z","Ġpract ical","Ġprov ince","ĠPl ace","Ġprior ity","IC E","Ġdescrib es","Ġbr anch","F orm","ask a","miss ions","b i","Ġp orn","ĠTur k","Ġent hus","Ġf ighters","Ġ0 8","ĠDet roit","Ġfound ation","av id","A re","Ġjud gment","cl ing","Ġsol ve","ĠDes ign","W here","hes is","ĠT ro","a fter","Ġne utral","ĠPalestin ian","ĠHolly wood","Ġadv is","ĠN on","y es","ol is","Ġrep utation","Ġsm ell","Ġb read","ĠB ul","ĠBe ach","Ġclaim ing","Ġgen etic","Ġtechn ologies","Ġupgr ade","row s","Ġdevelop er","ĠJ osh","ĠDis ney","erv ed","ip al","Ġun ex","Ġbare ly","t hen","ĠP ub","Ġill ness","et ary","ĠB al","Ġp atch","Ġbut t","Ġst upid","ĠD og","ĠD allas","f ront","ie ce","Ġprot ests","Ġch at","oen ix","Ġw ing","Ġpar liament","Ġ7 7","ose xual","Ġre nder","pt ions","ĠCo ast","os a","ĠG reg","h op","ĠMan agement","Ġbit coin","Ġrec over","Ġincor por","or ne","ĠUs ing","Ġpre ced","Ġthreat ened","Ġspirit ual","ĠE vent","ĠF red","Ġadvert ising","Ġimprove ments","ĠC ustom","Ġer rors","Ġsens itive","ĠN avy","Ġcre am","L ook","Ġex clusive","Ġcomp rehens","Ġde leg","Ġcon ce","Ġrem em","Ġstruct ures","Ġst ored","N D","Ġ1 000","U P","ĠB udd","A F","w oman","ĠAcad emy","ð Ł","se a","Ġtem porary","Ab out","es ters","Ġtick ets","Ġposs ess","in ch","o z","Ġl a","Ġcontract s","Ġun p","Ġc ig","ĠK at","ult ural","as m","Ġmount ain","ĠCapt ain","St ep","m aking","ĠSp ain","Ġequ ally","Ġl ands","at ers","Ġreject ed","er a","im m","ri x","C D","Ġtrans action","g ener","less ly","Ġ| |","Ġc os","ĠHen ry","Ġprov isions","Ġg ained","Ġdirect ory","Ġra ising","ĠS ep","ol en","ond er","Ġcon sole","in st","Ġb om","Ġunc ertain","1 50","ock ing","Ġmeas ured","Ġpl ain","Ġse ats","Ġd ict","S L","af e","Ġest imate","iz on","at hered","Ġcontribut ed","Ġep isodes","omm od","G r","AN T","Ġ6 9","G ener","Ġ2 50","vious ly","rog en","Ġterror ism","Ġmove ments","ent le","oun ce","ĠS oul","Ġpre v","ĠT able","act s","ri ors","t ab","Ġsuff er","Ġn erv","Ġmain stream","ĠW olf","Ġfranch ise","b at","Ġdem ands","Ġag enda","Ġdo zen","Ġclin ical","iz ard","ĠO p","t d","Ġvis ited","ĠPer haps","Ġact or","Ġde lic","Ġcont ribute","Ġin ject","ĠE s","ac co","Ġlist ening","Ġcon gress","epend ent","Ġprem ium","Ġ7 6","ĠIr ish","Ġass igned","ĠPh ys","Ġworld wide","Ġnarr ative","ot ype","m ont","b ase","ĠB owl","ĠAdminist ration","Ġrel ation","ĠE V","C P","Ġco vers","Ġ7 8","Ġcert ific","Ġgr ass","Ġ0 4","pir acy","ir a","Ġengine ering","ĠM ars","Ġun employ","ĠFore ign","st ract","Ġv en","Ġst eal","Ġrepl ied","Ġult imate","Ġtit les","d ated","Ġj oy","a us","Ġhy per","ak u","Ġoffic ially","ĠPro duct","Ġdifficult y","per or","Ġresult ed","rib ed","l ink","wh o","~~ ~~","ĠSpe ed","ĠV iet","W ind","ĠBar ack","Ġrestrict ions","ĠSh are","Ġ199 5","ition ally","Ġbeaut y","op t","Ġm aps","ĠC R","ĠN ation","ĠCru z","W ill","Ġelectric ity","Ġor g","Ġb urd","Ġviol ation","Ġus age","Ġper mit","ĠCh ron","ĠF ant","Ġn aturally","Ġ0 7","Ġth rown","ĠAw oken","Ġal ien","ĠHer o","ĠK ent","ĠR ick","ri ke","Ġp ace","}, {\"","G L","Ġpo ison","ĠT ower","Ġform al","al ysis","Ġgen uine","Ġk il","a ver","Ġproced ure","ĠPro p","intend o","ĠM ain","as ant","Ġtr ained","G ame","ĠL oad","ĠM A","Ġcru cial","Ġle ts","ĠF R","Ġch ampion","1 01","ĠCon ference","Ġwrit ers","Ġconnect ions","Ġo kay","ir ms","ĠR and","Ġenc ounter","ĠB uff","Ġachie ved","Ġche cks","isc ons","Ġassist ant","Ġwhen ever","ĠA ccess","ĠU r","b in","Ġcl ock","is p","op her","Ġb orrow","Ġm ad","Ġperson ality","on ly","IS T","ab ama","Ġg ains","Ġcommon ly","Ġter r","Ġhyp ot","Ġre ly","Ġt iss","iscons in","Ġrid ic","f unction","ĠO regon","Ġun com","r ating","el and","ĠN C","Ġm oon","ann on","Ġvulner able","ut ive","³³ ³³","ĠRad io","Ġw estern","se ct","ĠT ony","Ġocc urs","ĠO s","ĠH on","à Ń","Ġv essel","ĠScot land","Ġdiscrim ination","Ġsubsequ ent","st ring","Ġfant asy","ĠSh adow","Ġtest im","W E","it i","r as","Ġbo at","Ġmar ks","Ġord inary","Ġre n","Ġrepresent ative","Ġpet ition","Ġ7 3","Ġad venture","Ġign ore","ĠPhil adelphia","ĠS av","V P","Ġfact ory","Ġt asks","Ġdep ression","z ed","................ ................","ĠSt orm","Ġc ogn","Ġelig ible","Ġredu cing","v ia","Ġ0 5","Ġstri king","Ġdoll ar","h o","O V","Ġinstr ument","Ġphilosoph y","ĠMo ore","ĠA venue","Ġrul ed","ĠFr ont","IN E","ĠM ah","Ġscen ario","ĠNAS A","Ġen orm","Ġdeb ut","Ġte a","T oday","Ġabs ence","S im","Ġh am","le ep","Ġt ables","ĠHe art","M I","K e","re qu","V D","m ap","Ġchair man","Ġp ump","Ġrapid ly","v i","Ġsubstant ial","E P","d es","ch ant","ili pp","ĠS anta","ri ers","anche ster","L oad","ĠC ase","Ġsa ving","Ġ7 4","ĠA FP","er ning","oun ced","ĠMin nesota","ĠW as","Ġrec ru","Ġassess ment","ĠB ron","U E","Ġdynam ic","Ġf urn","ul ator","Ġprop ag","h igh","Ġacc ommod","Ġst ack","ĠS us","w rit","Ġre ven","ĠGod d","ĠZeal and","ab s","Ġbr ut","Ġper pet","h ot","Ġhard ly","ĠB urn","ãĤ ¹","Ġst y","Ġtrans actions","Ġg ate","Ġsc reens","Ġsub mitted","Ġ1 01","Ġlangu ages","ugh t","em en","Ġfall s","Ġc oc","Ĥ ¬","Ġstri kes","p a","Ġdel iber","ĠI M","Ġrel ax","ann els","ĠSen ator","Ġext rem","Ġ} ,","ĠDe b","Ġbe ll","Ġdis order","c ut","Ġi OS","Ġl ocked","Ġem issions","Ġshort ly","\" ]","ĠJud ge","ĠS ometimes","Ġr ival","Ġd ust","Ġreach ing","F ile","¯¯ ¯¯","ino is","ĠJ ason","Ġs atell","are t","Ġst ations","Ġag ric","ĠTechn ology","com es","ĠUn fortunately","ĠChild ren","Ġappl ies","ast ed","Ġan ger","ail ability","ĠDam age","Ġcomp are","ĠStand ard","Ġaim ed","ĠB a","angu age","Ġreg ulation","Ġj ury","Ġair port","Ġse ctions","ĠPr ince","em ed","Ġmedic ine","Ġh itting","Ġsp ark","ol ves","Ġad s","St ate","Ġfood s","Ġrepl acement","Ġch icken","Ġlow est","Ġmind s","Ġinvol ves","u i","Ġarr ang","Ġproced ures","ĠWh ich","ivers ary","Ġb ills","Ġimprove ment","Ġin ev","Ġexpect ations","Ġintellect ual","Ġsp aces","Ġmechan ism","2 50","bre ak","ĠZ e","ĠT enn","ĠB alt","Ġbar rel","Ġstat ic","man n","Pol ice","Ġt ips","Ġhand ling","c us","od ed","il ton","ir y","Ġjournal ists","our se","Ġcom ic","Ġnom ine","IT Y","Ġvers us","Ġlo op","Ġsur f","ĠInd ust","ĠHun ter","Ġbelief s","is an","Ġset up","Ġbre w","im age","Ġcomput ers","f ol","} ,\"","ĠMed al","Ġtax p","Ġdisplay ed","Ġg rav","Ġf iscal","M on","ĠMos cow","ĠK ong","ĠCent re","Ġcamer as","ĠMr s","ĠH ay","Ġa ver","ĠK elly","p y","Ġrequire ment","Ġent itled","omb ie","Ġsh adow","ag ic","ĠA k","Ġel ite","Ġdiv ided","Ġhead ing","Ġcop ies","Ġloss es","Ġv it","k ed","ĠB ry","Ġan s","ĠSte am","Ġrep orter","he im","ĠIt em","Ġsuper ior","d on","ere nt","à ¶","Ġtherap y","Ġpe ak","ĠMod el","Ġl ying","Ġg am","z er","r itten","Ġrespons es","Ġconsider ation","ĠB ible","Ġl oyal","Ġinst ant","Ġp m","ĠFore st","à ¼","Ġext end","Ġconv icted","Ġfound er","Ġconv in","ĠO ak","che ck","Ġsch olars","p ed","Ġover se","T op","c ount","ĠAr k"," ·","Ġ0 6","ĠL A","m d","ĠLat in","im ental","ĠC PU","Ġsubst ance","Ġminor ity","Ġmanufact uring","E r","ocol ate","Ġatt ended","ĠMan ager","r ations","Ġappreci ate","om y","GB T","id ency","B L","Ġguarant ee","pos ition","Ġo cean","clud e","Ġhead ed","Ġt ape","Ġlo ose","Ġlog ic","Ġpro ven","Ġsp ir","Ġad mit","is a","Ġinvestig ate","Ġ199 4","sy lv","ĠL ost","c est","Ġ7 1","Ġrequest ed","Ġwind ows","ĠPok é","ĠWith out","M et","Ġbehavi our","Ġread er","Ġh ung","ĠKe ep","Ġro les","Ġimplement ed","Ġbl ank","Ġserv es","ĠJ ay","Ġc ited","ĠF riend","prof it","ap on","Ġrep air","it em","arr ass","Ġcrit ics","ad i","ĠF ather","Ġsh out","Ġf ool","Ġ8 8","Ġprodu cing","Ġl ib","Ġround s","Ġcirc le","Ġpre par","Ġsub mit","Ġn ic","mor row","ãĥ «","U nder","Ġv ital","ater n","Ġpass word","Ġpublic ation","Ġprom inent","Ġspeak s","Ġb ars","Ġde eper","ĠM ill","port ed","Ġw id","Ġbut ter","Ġsm oking","Ġindic ates","K ey","rop ri","ĠF ile","all ing","ast ing","ĠR us","Ġad j","Ġ7 9","av al","Ġpres um","bur gh","on ic","Ġf ur","Ġpoll s","ik a","Ġsecond ary","Ġmon ster","ig s","ĠCur rent","E vent","Ġowners hip","end ar","Ġarri ve","ĠT ax","Ġn ull","ĠPri v","Ġth ro","Ġk iss","c at","Ġup set","ang le","it ches","ect or","olog ists","ĠGal axy","Ġcor ruption","Ġh int","ent er","ĠH ospital","Ġgreat ly","Ġbeg un","es y","Ġso il","ĠAnt on","Ġmain tenance","ãĥ ©","Ġdo zens","Ġhuman ity","ĠAl abama","Ġr om","w orth","ap ing","sylv ania","l ah","Ġg athered","G A","Ġattack ing","f ound","ĠSqu are","Ġar bit","ict ions","ĠW isconsin","Ġd ance","ĠS aint","arch y","Ġbase ball","Ġcontribut ions","Ġliter ature","Ġex ha","per ty","t est","Ġb ab","Ġcontain er","let ter","Ġfall en","Ġwebs ites","Ġbott le","ĠS ac","Ġbre ast","ĠP L","Ġveter an","Ġinterview s","ĠA le","Ġb anned","eng ers","ĠRev olution","in th","Ġconc erning","IV E","Ġexp enses","ĠMatt hew","ĠColumb ia","d s","ist ance","Ġent ity",".. .\"","Ġrel iable","Ġpar alle","ĠChrist ians","Ġopin ions","Ġin du","l ow","Ġcompet e","Ġth orough","Ġemploy ed","Ġestablish ment","ig en","ĠC ro","Ġlawy ers","ĠSt ation","T E","ĠL ind","ĠP ur","it ary","Ġeffic iency","âĢ IJ","ĠL y","Ġm ask","Ġdis aster","Ġag es","ER E","es is","ĠH old","Ġcas ual","b led","Ġen abled","ĠEn vironment","ĠInt elligence","i per","ĠM ap","ĠB E","Ġemer ged","is dom","Ġc abin","Ġregist ration","Ġfing ers","Ġro ster","Ġfram ework","ĠDo ctor","et ts","Ġtransport ation","Ġaware ness","H er","Ġattempt ing","O ff","ĠSt ore","ÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤ","ĠK now","Ġdef ence","Ġsc an","ĠT en","ĠCh air","ĠP H","ĠAtl anta","Ġfuck ing","Ġans wered","b n","ĠK ar","Ġcateg ories","Ġr ational","Ġc ust","Ġrob ot","Ġcorrect ly","Ġg if","Ġgraph ics","m ic","Ġground s","ĠO pp","i ate","Ġdist ributed","Ġsan ctions","Ġchalleng ing","ut o","Ġingred ients","Ġinv ited","Ġfound ed","ĠRe qu","d ed","Ġb owl","Ġbrother s","ĠH a","I O","Ġw ages","im ore","oc ial","Ġse ed","ative ly","Ġaddress es","ĠI owa","ab eth","Ġatt itude","is d","ch ild","Ġm ole","Ġdisco very","y ard","B r","Ġ8 2","Ġsuppl ies","ell ing","Ġdist ingu","C R","Ġre cept","Ġ vert","Ġsw im","b ec","d oor","ĠY eah","Ġg al","Ġinter act","ĠE SP","ĠC S","amp s","Ġconvin ced","Ġobject ive","Ġdis h","ĠPhot os","l ad","Ġdownt own","o il","in ction","Ġto morrow","ĠC OM","Ġsurv ival","sh ot","Ġsett lement","C ons","ĠX box","int erest","ĠS M","arg o","en ess","Ġeth nic","b ered","M in","ĠT ok","Ġinc ent","ĠComm and","Ġmain tained","Ġbreak s","br idge","at ar","ag g","ĠF inally","un icip","ĠO nt","le ft","Ġrecogn ition","Ġ* /","ĠP ers","Ġwe lf","Ġaddress ed","ĠK ansas","Ġvir us","Ġwhere as","Ġp apers","ram s","ĠMin istry","Ġple asure","Ġacqu ired","Ġd uration","j pg","Ġcal m","ĠN HL","Ġburn ing","Ġfold er","ick ed","ĠP y","ĠIll inois","Cl ass","ĠGodd ess","Ġperform ing","Ġwelf are","j ar","In ter","Ġl in","Ġenh ance","Ġnot ion","f are","yp es","ĠAre a","Ġcann abis","ĠDie go","f s","ĠM anchester","com m","in ite","Ġcover ing","ĠS ound","Ġ19 60","Ġ8 4","e lect","z ing","Ġcitiz en","Ġph ones","Ġr aid","Ġign ored","ĠOb ject","Ġu pload","c ard","Ġmod ified","Ġroom s","ia h","r ange","he ast","ach us","Ġsuggest ing","âĢ ĭ","gr ade","E l","Ġclot hing","Ġr h","ĠH an","un ity","en cing","ĠAust in","sec ution","t ra","d em","ĠQ ual","Ġhe aven","Ġst ages","Ġw edd","pl us","ific ial","ĠIm m","ĠH o","iet ies","Ġphr ase","Ġbr ill","act ory","Ġprov iders","Ġsil ence","Ġa er","ĠA I","ĠAd venture","Ġplatform s","Ġdemonstr ated","Ġinter f","ing ton","Ġr aces","Ġgr ade","ult ane","ĠTh rough","f alse","Ġb ow","ĠA B","Ġfl avor","Ġhistor ic","g ov","Ġcol our","Ġview ed","ĠEm ail","el come","Ġinter vention","Ġd iversity","Ġperiod s","Ġre verse","ĠV ery","Ġqu ote","ĠLe ft","th rough","Ġsc rew","Ġland ing","Ġp ill","Ġw et","Ġprot esters","Ġrepe at","av ed","er k","Ġsal ary","ĠPenn sylvania","St ill","Ġmay or","Ġkit chen","Ġfeat uring","ĠM useum","ĠT ournament","ĠF al","Ġser vers","U C","Ġany body","im g","ĠTr ade","ixt ure","the less","Ġfin ance","Ġcl osing","ĠPat ri","i ac","ab el","Ġ> >","or ous","Ġf irms","sc reen","un a","Ġemb arrass","ul se","Ġlet ting","Ġth rew","ile y","Ġch annels","l an","ĠVeg as","Ġse ar","Ġfant astic","ar re","uzz le","ĠD er","Th ose","Ġsw ing","Ġshe et","ind ex","co ver","og an","Ġvari ables","ĠTe ch","Ġsp oken","ac hel","ĠD a","ĠMount ain","Ġload ed","Ġfoot age","vers ion","Ġun l","ĠPh oenix","Ġthrow ing","Ġf iring","Ġtrack ing","Ġw idth","Ġstrugg ling","ro oms","ot ion","Ġmonth ly","ĠSer ver","Ġegg s","op en","M C","Ġ199 3","Ġh ired","Ġstay ed","ĠAll en","Ġst ro","Ġ9 8","st ep","ĠTurk ish","Ġfab ric","ist ing","ĠD om","Ġd ates","Ġpr on","Ġbasket ball","Ġl ucky","ĠArab ia","Ġassum ed","est y","Ġaff airs","Ġgl ad","ĠInd eed","ĠF A","ĠW ord","Ġjo ining","if ice","p read","ir ts","ĠSe lect","Ġpop ulations","aw are","Ġn ose","Ġcompl aints","st art","Ġsc oring","Th anks","Ġmin ing","Ġvisit ors","S H","Ġdam aged","Ġcharacter istics","ĠP ent","D C","Ġ8 3","ĠS ix","r ates","Ġfl ags","ĠB rew","d og","M ark","// //","Ġexec ution","Ġj oke","ph ones","Ġtestim ony","Ġob st","Q L","ĠC ut","Ġstud ied","ĠN intendo","ick et","ĠN BC","Ġl ad","ĠB ra","ĠM oh","Ġk ernel","Ġoverwhel ming","Ġag ed","Ġapplic able","ĠC ond","Ġroad s","ĠBl ock","m ade","od ge","Ġcomm ands","Ġoff ices","vel and","Ġt ut","Ġrece iver","ĠF ro","Ġsho pping","Ġi P","ĠSt re","ĠA BC","Ġentertain ment","ĠB ow","ort ed","M c","Ġread s","gr ad","ĠCol lect","Ġâ ĪĴ","ĠCap ital","eder ation","Ġemploy er","Ġinvolve ment","Ġanx iety","al ia","Ġro of","ĠAm ong","ĠDemocr at","Ġstat s","ĠV ill","Ġconst itutional","Ġrefer ring","itt y","Ġtack le","out ube","Ġback ed","ĠH ong","ĠBro ad","Ġe le","ĠO tt","Ġ199 2","h our","achus etts","C al","Ġdefe ated","Ġ8 1","es p","Ġseem ingly","w as","ĠJ enn","ĠK urd","Ġg ene","Ġdisc ount","R et","EC T","( );","Ġclub s","Ġs id","ĠM arsh","Che ck","Ġp p","ĠE ag","ides pread","Ġbe ings","F T","Ġintrodu ction","ĠCh ange","AR D","Ġ1 10","ad ows","ier ce","Ġme al","a uthor","ĠB ang","lah oma","Ġr anks","201 1","?? ??","m ax","Ġcoll apse","Ġop ens","Ġe cho","Ġs oph","Ġrac ist","Ġenorm ous","Ġw aves","Ġt ap","Ġcomprehens ive",". --","ĠR oy","Ġfarm ers","Rel ated","a ired","ron es","ĠC rim","Ġproport ion","Ġdesign s","Ġnegoti ations","Ġvirt ually","ĠBat man","Ġwar n","Ġlegit imate","m ate","Ġcon vention",", ,","net ic","ĠS D","Ġconsist ently","Ġcompens ation","Ġpunish ment","Ġy e","Ġt ie","ĠB ureau","ir lf","ĠB u","ĠA ren","ĠPh ilipp","Ġkn ife","Ġmem ories","ĠR oss","Ġang le","Ġ8 6","ĠTh under","Ġre nd","ĠT our","Ġcount s","s ung","ĠIm p","Ġeduc ational","Ġaccess ible","C OM","Ġd rew","y er","G l","am ine","OR T","O B","I B","m aster","Ġtri als","og y","h ar","ĠTr ust","Ġprefer red","irlf riend","ĠN ev","Ġb in","Ġc ow","P age","Ġsign ature","ĠB L","7 00","Ġret ired","Ġby tes","Ġneigh b","ĠLeg end","Ġdev ast","Ġsuspect ed","is ons","ĠPoké mon","sc ale","Ġcap abilities","Ġre vel","Ġche ese","d y","igr ant","Ġfail ing","b its","ĠHer oes","ĠG host","ĠS cient","Ġappoint ed","ur i","Ġinst itution","Ġexpand ed","g reg","Ġmonitor ing","Ġp odcast","Ġcoal ition","Ġ9 6","J o","Ġst olen","ĠS ab","Ġstop s","Ġhol iday","Ġint r","C ar","Bl ack","ĠL GBT","Ġwar ming","ĠAnd erson","Ġ8 9","Ġprodu cer","M ed","Ġaccur acy","ĠMar vel","iz abeth","ĠPat rick","m ony","Ġmin i","ac les","Ġover t","the y","Ġmembers hip","ĠV en","Ġex ch","Ġrem oval","ĠD ave","T Y","m ad","ĠF ind","Ġad equ","Ġe c","Ġte eth","Ġemot ion","Ġper m","Ġsole ly","d b","Ġextra ord","IG HT","c al","Ġgu idelines","Ġd ying","Ġsusp ended","ĠPrem ier","ĠAnth ony","el ve","Ġd ad","ĠE th","ĠFoot ball","Ġabandon ed","Ġ< <","Ġm arch","Ġhor ror","â̦ \"","Ġchild hood","Ġcampaign s","Ġl unch","ĠAl bert","bl ock","âĸĪ âĸĪ","ound ing","Ġb one","or gan","ad ers","ĠFl ash","ĠDri ve","Ġton ight","Ġw ars","ĠF L","Ġform ation","con st","New s","Ġcom pe","or ious","ĠSt aff","Ġdiscuss ions","ĠProt ection","ĠJ am","Ġcrit eria","Ġinstall ation","Ġaccompl ish","iz za","Ġpub lisher","Ġresc ue","ĠT ry","U LL","ĠS om","ĠH op","ore t","th s","ord on","Ġp ocket","ĠIn v","Down load","ĠCr ime","Ġb ene","ĠGu ide","ĠAs sembly","Ġparam eters","I E","ĠAlex ander","Ġconc ert","ĠSc he","Ġsh oes","Ġvis iting","Ġrec all","Ġb ub","Ġr ural","Ġconc rete","ĠR os","N ext","R uss","Ġlo ans","ĠSh ield","Ġtre m","hem at","k g","ĠHar ris","is ition","ĠM ove","ĠF C","Ġf ate","ĠCh o","Ġt ired","Ġprinc ipal","h ist","ien ces","ath y","Ġse vent","Ġm ood","Ġstrateg ic","Ġdise ases","Ġfor um","Ġtem por","Ġhead quarters","P ar","ig e","fl ix","Ġgu itar","Ġ9 4","On ly","Ġrele ases","ro ph","================ ================","Ġ6 00","ĠContin ue","ig ate","ĠC rit","sy stem","Ġdis abled","Ġunex pected","ith ub","Ġuncle ar","ĠE st","Ġcontr ad","Ġstrateg ies","vent ures","Ġpass age","AM E","Ġimpro ving","Ġreve als","Ġdecre ase","ov a","Ġann oy","ĠSh ort","ĠL ibrary","Ġcy ber","n ell","ĠH ur","ĠC B","Ġphot ograp","U I","Ġs ed","G e","Ġ8 7","Ġd iverse","Ġencour aged","Ġcons piracy","Ġbird s","Ġoper ator","Ġhand ful","Ġclass ified","? )","Ġdram atic","Ġinvestig ators","it o","Ġw idespread","ĠR oom","-------------------------------- --------------------------------","Ġcollect ive","Ġjournal ist","St ring","Ġtemper atures","il a","Ġgu id","Ġins pect","Ġmiss ile","ĠMay or","Ġman ual","Ġsim ultane","Ġrat ings","Ġsu ck","Ġ9 7","Ġunivers al","Ġph arm","Ġdis rupt","ian o","A V","Ġf t","Ġstat ist","old s","ĠWalk er","ph p","Ġunder t","ĠL as","ish op","nt il","res hold","ĠWhe ther","M s","Ġden y","ĠCl oud","Ġprov ider","Ġsurv iv","ĠUp date","h as","Ġmist akes","ch arge","pl ed","r ity","Ġn ode","ĠMass achusetts","ool s","lic ation","Ġf ails","em ale","or i","back s","Ġsh irt","Ġ' '","ĠN AT","Ġwat ers","els on","Ġe ase","Ġsc ar","Ġcont ents","m ind","Ġcont ribution","Ġsh r","Ġhand ed","Ġst ability","Ġtra ve","E m","Ġmir ror","12 3","Ġwe igh","Ġf iction","ou ver","ist ant","r ition","ĠF ed","Ġphys ically","Ġst ake","ĠArt icle","ĠAr c","ĠLew is","ĠM ind","Ġdemonstr ate","Ġprof its","v ision","om ic","ol id","Ġbatt les","Ġdri ves","Ġeas tern","ĠS ony","!! !","ar ation","v ard","ĠG L","port ation","Ġ9 2","Ġlaw makers","Ġprotect ing","ĠE PA","Ġy eah","Ġsh ame","ol ph","e ven","x it","Ġatt ach","Ġrepresent ing","Ġob s","ĠUt ah","iff s","ĠFre edom","à ³","A K","Ġinc idents","it age","Ġview ers","c d","Ġm ouse","Ġcl ar","Ġaccord ance","Ġb ot","c or","ĠSum mer","he ld","Ġinnoc ent","Ġiniti ative","ol s","________________ ________________","Ġsp ots","p ace","Ġconvent ional","Ġcorpor ations","Ġblock ed","H D","at tered","Ġref ers","Ġbu ck","ĠDig ital","12 0","Ġtop ics","T F","Ä ģ","br id","re ement","Ġunder lying","ĠM ember","Ġinvestig ating","Ġpregn ancy","Ġtouch down","ĠB and","ĠCall er","Ġinst ances","P P","w a","G ood","Ġ199 1","ĠC old","Ġfear s","Ġrem arks","Ĩ Ĵ","at al","Ġm it","Ġexper iments","i pt","Col or","ind u","Up date","Ġ9 3","A g","Ġ å","anc ouver","B oth","Ġjud ges","Ob ject","Ġst ere","umb n","Ġparticip ation","ĠSt ars","ĠJ ere","Ġweek ly","ĠB an","Ġconvers ations","ĠP itt","u z","ĠIndian a","ĠK ick","Ġinf ection","Ġhero es","Ġsett led","Ġstri p","Ġh al","Ġd ump","ĠS ci","Ġl es","Ġref erences","ĠU RL","ĠBr idge","Ġwant ing","For ce","Ġex clus","Me anwhile","m n","Ġg entle","m aker","sen al","ĠG ro","ou ri","ĠR ain","ĠAll iance","Ġl ift","el a","S D","ĠCle veland","Ġrank ed","Ġst adium","Ġdead ly","ä ¸","Ġr iding","ar ia","ĠAr mor","Ġdocument ation","ĠGree ce","ree k","Ġl ens","ĠS a","Ġg ross","ĠE mer","ag ers","ĠD ub","ĠR h","ĠAM D","Ġarri val","Ġdes ert","Ġsupp lement","ĠRes p","Ġkn ee","Ġmarg in","f ont","og g","201 0","ĠP ir","ĠP rom","iv als","Ġint ake","Ġdifferent ly","ug s","Ġb its","clud ed","Ġsearch ing","ĠD u","um ble","Ġfunction al","ĠBalt imore","ĠC ould","Ġdes ired","Ġcirc uit","ĠL yn","ĠG O","ĠF alse","re pre","' :","alt ies","Ġmin im","Ġdro ve","ĠSh ould","Ġh ip","Ġpro s","Ġut ility","ĠN ature","ĠM ode","P resident","o pp","r at","form ance","Ġconcent ration","Ġf ont","ĠB ud","Ġam id","Ġre vers","ĠM L","B ar","Ġinter action","Ġjur isd","Ġspell s","d ep","f il","Ġcivil ians","ut ter","ĠCo oper","ĠBel ow","Ġent rance","Ġcon vert","Ġcontrovers y","ow ered","Ġcontr ary","Ġar c","ĠExec utive","ĠOffic er","Ġpack ages","Ġprog ressive","w idth","Ġreserv ed","v ol","ĠSam sung","Ġprint ed","Ġcent ers","Ġintrodu ce","ĠKenn edy","Ġodd s","Ġsure ly","Ġindepend ence","Ġpass engers","repre ne","ĠBe h","Ġl oves","ĠESP N","Ġfac ilit","Ġident ical","Ġdo ct","Ġpartners hip","con f","ĠH ide","Ġconf used","ĠC ow","M en","Ġw rest","ĠIraq i","Ġh oles","ĠStud ies","Ġpregn ant","h ard","Ġsign als","I X","Ġpull ing","Ġgrad uate","Ġnomine e","D ate","Ġper mitted","Ġâ Ĥ¬","ĠOk lahoma","St art","Ġauthor ized","Ġal arm","ĠC os","v an","Ġgener ations","c ular","Ġdr agon","ĠSoft ware","ĠEd ward","Ġcontro ller","S en","ge red","ĠV ik","Ġappro ached","Th ank","Ġcan ce","Ġform ula","ĠSm all","Ġweak ness","Ġr amp","it udes","j ud","Ġbrill iant","Ġacc us","s ource","Ġ8 00","ĠE vil","S w","Ġhom eless","we ek","i ens","r ics","ĠTh ird","T O","Ġorgan ic","Ġpresent ation","ag h","ĠDown load","v ation","Ġas sembly","or able","hold ers","ĠBern ie","ĠHel p","Ġt ong","ĠF ight","Ġbe ach","B ook","ĠL ic","Ġr ush","ĠR ound","ou p","ĠMar x","Ġcalcul ated","ĠDe vil","ĠSar ah","Ġoccasion ally","Ġbul let","Av ailable","g ate","Ġ9 1","Ġh osp","Ġprom ises","ĠH IV","ĠSt adium","ĠSt ock","ĠCorpor ation","g age","N G","ĠC redit","Ġs ne","ib l","Ġacc um","s uch","Ġterror ists","Ġconscious ness","ĠZ h","Ġdram a","ool a","pir ation","Ġlab our","ĠN in","Ġut ter","Ġdemocr atic","Ġass ass","il ation","Ġg est","Ġab road","Ġmet ab","Ġs orts","Ġfl av","U B","Ġm g","ĠNot hing","ĠO d","Ġmus ical","200 9","Ġdro ps","oc ated","ater al","0000 00","Ġg re","Ġequ ality","Ġburd en","Ġv ig","ĠLe ader","-------- ----","Ġcere mony","Ġf ighter","Ġact ors","Ġ æ","am an","F i","Ġal ign","put er","Ġe lder","ĠN SA","Ġrepresent ation","ĠOnt ario","IT H","usal em","Ġharass ment","itz er","Ġsy mp","Ġbox es","ĠD R","Ġman ifest","at re","Ġ ^","Ġd ies","le ton","Ġmiss ions","et he","Ġres olve","Ġfollow ers","Ġas c","Ġk m","l ord","am med","Ġsil ent","ĠAssoci ated","Ġtim ing","Ġprison ers","ĠK ings","ĠF ive","Ġtow er","Ġappro aches","Ġprecise ly","Ġb ureau","ĠM other","ĠI ss","Ġkey board","it ual","Ġfund ed","Ġstay ing","Ġpsych ological","Ġm ile","ĠLe on","ĠBar b","w ill","Ġw ider","ĠAtl antic","Ġt ill","ĠR ome","ro t","Ġaccomp an","Ġfl our","ac o","W orld","ĠExp ress","ĠY u","C or","Ġple ased","part y","Ġpoint ing","Ġinf lation","Ġro y","Ġ ),","ain er","Ġwedd ing","orm on","Ġrequ iring","Ġqual ified","Ġse gment","EN D","Ġs izes","e als","Ġcor rupt","ass ador","Ġcele b","Ġdream s","ĠM ess","Ġcheck ing","ĠV ersion","Ġprep aring","Ġact ively","ĠD iff","Ġl ux","ĠW inter","act eria","ĠN E","Ġdep uty","Ġtrans gender","Ġsum mary","Ġin her","er ies","ch ar","ĠY an","Ġkn ock","ĠP ath","Ġl ip","roll er","Ġimp ression","Ġcelebr ate","Ġsl ide","Ġgu ests","Ġcl ip","F S","Ġsav ings","Ġcapt ain","Ġleg acy","ĠDen ver","Ġw ounded","tab oola","AC T","Ġpurs ue","Ġo xy","Ġ q","Ġsem i","ĠN eed","ĠAff airs","Ġob sc","Ġcheck ed","Ġd ual","C ode","ĠM D","le m","ult y","Ġ ©","ĠEl izabeth","Ġcent uries","ard ed","s rc","Ġev ident","enn is","at in","Ġunemploy ment","ĠMar io","Ġint im","Ch rist","Ġbi ological","Ġsold ier","ĠAdd ed","Ġm ath","ĠG il","Ġbi as","Ġd ating","ĠO cean","Ġm ice","M us","h ire","ĠT es","Ser ver","lim ited","S ize","Ġmet ers","Ġrock et","es see","Ġcertific ate","ĠIran ian","AS S","Ġgr id","D ec","Ġro lling","com mun","ĠSwed en","b ury","Ġtiss ue","Ġrac ism","ĠL ocal","Ġmyster y","Ġexam ine","Ġst em","Ġs its","Ġhop ed","ot ing","Ġdial ogue","Ġpers u","W atch","l ay","M AN","Ġch ronic","ĠPort land","mark et","ĠS EC","Ġparalle l","Ġsc andal","Ġcar ries","Ġphenomen on","h uman","ack er","ĠO x","Ġretire ment","tain ment","ov ie","ĠG ear","Ġd uties","Ġdo se","Ġsc roll","M B","in f","Ġsa uce","Ġland scape","red dit","ĠChampions hip","ĠRed dit","al id","Ġco in","Ġover s","Ġpost ing","ab out","Ġf el","and y","Ġb old","Ġfocus ing","e ffect","G R","Ġde emed","Ġrecommend ations","Ġste pped","Ġvot er","ĠDe ep","ĠInst agram","Ġmoder ate","ĠMary land","Ġrestrict ed","ĠM B","ĠCh all","Ġto b","Ġc ir","ĠO cc","ĠE ver","Ġcoll aps","IN FO","= -","ĠP ict","ĠAcc ount","n c","Ġo ught","Ġex port","Ġdr unk","( '","Ġw ise","ĠM ort","ne cess","Ġan cest","ĠInc re","Ġfrequ ent","m ir","Ġinterpret ation","Ġdepend ent","Ġco ins","ĠB ol","V ideo","ĠJust in","Ġfat al","Ġcook ing","Ġconf usion","ip her","Ġcust ody","ĠMor gan","om ach","ĠGovern or","Ġrestaur ants","el ing","Ġacknowled ged","Ġthe r","Ġgen es","ch ing","He y","Ġtact ics","ĠMex ican","Ġv end","Ġhe s","qu er","Ġnot ing","ĠCamer on","Ġtarget ing","ro ck","Ġcred its","Ġemot ions","Ġrepresent atives","new s","Ġlegisl ative","Ġrem oving","Ġtweet ed","ĠCar ter","ĠF ixed","Ġfor cing","Ġspeak er","Ġm ales","ĠViet nam","l ined","Ġconcept s","Ġvo ices","o ir","ĠT rib","W he","ĠJer usalem","ĠS ant","Ġc ul","Ġl ady","ĠHaw ai","Ġar ts","ĠIn n","ĠMach ine","ĠEm peror","Ġsl ot","g ly","ĠPro cess","II I","Ġathlet es","ĠTem ple","ĠRep resent","Ġpres c","Ġt ons","Ġgold en","Ġp unch","ĠG R","iver pool","Ġen act","Ġlob by","Ġm os","Ġpick ing","Ġlif etime","Ġcogn itive","E ach","z o","Ġd ub","Ġcons ists","ol n","Ġf estival","am ous","Ġint ellig","w ords","ĠSm art","Ġde le","Ġl apt","Ġmag ical","ĠS in","b us","ur ities","igh th","ĠRub y","ĠS ure","ol ving","Ġj un","O ST","Ġimp osed","Ġast ron","Ġcor rel","ĠN S","ĠK it","ĠF uture","b urn","Ġimm une","oc us","Ġcour ses","ĠSt ring","Ġle an","Ġg host","Ġout comes","Ġexp ense","Ġevery day","Ġaccept able","A h","Ġequ ipped","Ġor ange","F R","ĠD utch","Th ough","ĠR ank","Q U","ĠRober ts","wh at","re nd","Ġdisapp ear","Ġsp awn","ĠL am","o is","Ġdes erve","Ġmin imal","Ġnerv ous","ĠW ould","Ġro ok","ĠV ancouver","Ġres ign","sh ire","ĠW orks","ĠB uild","Ġafford able","ĠG ary","ĠAren a","Ġh anging","Ġimpl ications","ĠS ong","Ġmain taining","Ġgu ards","C ON","Ġder ived","Ġexecut ed","Ġthe ories","Ġqu oted","ĠAnd re","og a","sel ess","in fo","ĠBel g","Ġt ears","ĠSur v","Ġbirth day","ig ious","im mer","Ġspect rum","Ġarchitect ure","Ġrec ruit","arm a","T able","Ġmon sters","ĠG ov","Ġdest ination","Ġattract ive","Ġf oss","ĠMore over","Ġpres ents","TH E","Ġrep ly","pt on","Ġc um","Ġdel ight","Ġaffect s","Ġdon ations","ĠT oy","ĠH im","M ENT","Ġover come","it ched","ĠFant asy","ĠH at","ĠBe ast","b ott","Ġinvestig ations","R un","Ġhun ting","d i","f und","Ġs essions","est yle","Ġport ray","oid s","Y eah","Ġcommun icate","Ġcom edy","ĠY ang","Ġbel t","ĠMar ine","Ġpredict ed","Pl ay","Ġimportant ly","Ġremark able","Ġelim inate","D avid","Ġb ind","V ID","Ġadvoc ates","ĠG aza","im p","D B","ĠN a","ĠSim ilar","I ES","Ġchar ity","v as","m ath","Ġâ ĸ","ok er","nd um","Ġcap s","ĠH al","2 000","e an","Ġfle et","Ġrec re","R ight","Ġsleep ing","ij ing","k ind","Ġdesign ated","à ¤","Ġanim ation","ke e","ĠInt rodu","Ġ/ >","Ġdelay ed","Ġtrem end","Ġcur ious","U se","Ġle ct","d am","Ġinnov ation","ĠPoint s","Ġload ing","Ġdisp ute","ct ic","ird s","ĠB Y","Ġn urs","ĠVal ue","ION S","ĠH um","Ġtem plate","m ers","Ġappear ances","ĠEnter tainment","Ġtransl ation","Ġsa ke","Ġbene ath","Ġin hib","Ġe uro","abet es","Ġstud ying","ĠM as","Ġper ceived","Ġexam ined","Ġe ager","Ġco aches","Ġim per","ch i","Ġprodu ces","\" ).","ĠEvery one","Ġm unicip","Ġg irlfriend","Ġh ire","ĠV ice","Ġsu itable","op y","Ġin equ","ĠD uke","f ish","f irst","ĠO bs","Ġinter ior","ĠBru ce","ĠR y","Ġanal ys","Ġconsider able","Ġfore cast","Ġf ert","ors hip","ĠD rug","ĠA LL",": \"","th ur","ĠM ail","Ġball ot","Ġinst antly","ĠCh annel","Ġp icks","Ġ198 9","Ġt ent","ol i","Ġcivil ian","b ling","ell o","b u","Ġin ch","Ġlog o","Ġcooper ation","Ġwal ks","Ġinvest ments","Ġimp rison","ĠF estival","ĠK y","Ġleg ally","Ġg ri","ch arg","S l","Ġthreat ening","du ction","fl ow","Ġdismiss ed","ibr aries","c ap","e le","ĠMc G","ĠHar vard","ĠConserv ative","ĠC BS","p ng","Ġro ots","ĠH aving","umb led","ĠF un","\\ /","ĠS earch","ple x","Ġdiscuss ing","Ġcontin u","ĠT ai","ĠW ik","F ree","f it","Ġref use","Ġmanag ing","Ġsy nd","ip edia","w alk","Ġprofession als","Ġguid ance","Ġunivers ities","Ġas semb","unt u","F inally","AS E","ĠAut o","ĠH ad","Ġann iversary","L D","ĠD ur","ĠUlt imate","ih ad","pro duct","Ġtrans it","Ġrest ore","Ġexpl aining","Ġass et","Ġtransfer red","Ġbur st","ap olis","ĠMag azine","ĠC ra","ĠB R","gg ed","ĠH E","M ich","b et","ĠL ady","yl um","erv es","Ġme ets","wh ite","L og","Ġcorrespond ing","Ġins isted","G G","Ġsurround ed","Ġt ens","Ġl ane","Ġco inc","h ome","Ġexist ed","ect ed","ĠDou ble","lam m","Ġske pt","ex p","Ġper ception","ie v","ĠBe ing","o ft","Ġadop t",". :","] ;","Wind ows","Ġsatell ite","AS H","Ġinf ant","d escription","ĠMe anwhile","c m","oc a","ĠT reat","act or","Ġtob acco","ĠN orm","em ption","Ġfl esh","Ġj e","o op","ĠHe aven","Ġbe ating","an im","Ġgather ing","Ġcult iv","G O","ab e","ĠJon athan","ĠSaf ety","Ġbad ly","pro t","Ġcho osing","Ġcontact ed","Ġqu it","Ġdist ur","Ġst ir","Ġto ken","D et","ĠP a","Ġfunction ality","00 3","s ome","Ġlimit ations","Ġmet h","b uild","con fig","N T","re ll","ble m","ĠM om","Ġveter ans","ĠH u","Ġtrend s","are r","ĠG iven","ĠCa ption","m ay","AS T","Ġwond ering","ĠCl ark","n ormal","Ġsepar ated","Ġdes p","st ic","b rew","Ġrel ating","ĠN ik","ĠF arm","Ġenthus i","g ood","d eb","Ġactiv ist","Ġm art","Ġexplos ion","ĠEconom ic","L ink","Ġins ight","Ġconven ient","Ġcounter part","su pport","ĠV irt","ag en","ĠTenn essee","ĠSim on","ĠA ward","OC K","ĠF igure","Ġoverse as","Ġpr ide","ĠC as","n ote","m g","C urrent","Ġdispl ays","cont ent","Ġtravel ing","Ġhosp itals","ĠFin ancial","ĠP ast","Ġdefend ant","Ġstream ing","m ble","ĠBer lin","uk i","Ġdist ribut","Ġant ib","Ġch ocolate","ĠCast le","Ġinter rupt","ĠR ow","Ġconvers ion","Ġbug s","ĠR ather","li est","L Y","ĠJe an","com mon","ak h","Ġ1 30","ot ton","ĠDe an","Ġam endment","Ġgame play","ĠWar ren","od a","Ġhigh lights","Ġir re","ĠNAT O","Ġball s","Ġdemand ing","U RE","ĠL uke","F igure","st op","on ia","z one","iz ers","ĠW R","Ġaward ed","Ġregul atory","ĠH art","ĠS N","pl ing","Ġs our","ĠP ixel","us ive","Ġf et","ĠS ent","Ġautom atic","Ġf er","vern ment","ĠKh an","T ON","f ather","Ġextraord inary","th rop","ĠP ython","ĠG PU","Ġsex ually","Ġdesk top","it ivity","ĠAnton io","Ġo rient","Ġe ars","ob by","ous es","vertis ements","Ġmanufacture rs","ic ient","min ute","Ġconv iction","Ġg arden","p ublic","Ġsatisf ied","f old","O K","Ġin hab","ĠTh ink","Ġprogram me","Ġst omach","Ġcoord in","Ġh oly","Ġth reshold","Ġr het","Ġser ial","Ġemploy ers","ĠEvery thing","ra h","Ġb other","Ġbr ands","Val ue","ĠT ed","ĠPlan et","Ġp ink","ĠFurther more","s a","P E","re ck","ĠUS D","ot te","Ġ& &","Ġland ed","g ets","Ġprodu cers","Ġhealth care","Ġdomin ant","Ġdest ro","Ġam ended","ch ron","Ġf its","ĠSy d","ĠAuthor ity","AT CH","Ġfight s","ĠL LC","Ġ-- -","ĠCor p","Ġtox ic","spe cific","ĠC orn","ĠChe l","Ġtele phone","ĠP ant","Ġmyster ious","aun ch","od ox","med ia","Ġwitness es","ag u","Ġquestion ed","ĠBre xit","ĠRem ember","ene z","Ġend orse","iat ric","ĠId ent","Ġridic ulous","1 10","Ġpr ayer","Ġscient ist","Ġ19 50","ĠA qu","Ġunder ground","ĠU FC","m are","ĠL ater","w ich","Ġsubsc rib","Ġhost s","Ġer r","Ġgr ants","ant om","Ġsum mon","ear ly","ĠC lear","ĠPr im","Ġsusp ension","Ġguarant eed","app er","Ġr ice","ĠSe an","ĠSh in","Ġrefere ndum","Ġfl ed","r ust","Ġ3 60","ter y","Ġsh ocked","B R","ĠO il","ĠAll ah","Ġpart ly","Ġign or","Ġtrans mission","Ġhom osexual","ivers al","Ġhop efully","ãĤ ¤","Ġless on","L eg","Ġ ..","Y et","t able","app ropri","re tt","Ġbo ards","Ġincor rect","Ġb acteria","ar u","am ac","Ġsn ap",".' \"","Ġpar ad","t em","he art","Ġav ailability","Ġw isdom","Ġ( +","Ġpri est","ĠÂł ĠÂł","O pen","Ġsp an","Ġparam eter","Ġconv ince","Ġ( %)","r ac","Ġf o","Ġsafe ly","Ġconver ted","ĠOlymp ic","Ġres erve","Ġhe aling","ĠM ine","M ax","Ġin herent","ĠGra ham","Ġinteg rated","D em","Ġpip eline","Ġapp lying","Ġem bed","ĠCharl ie","Ġc ave","200 8","Ġcons ensus","Ġre wards","P al","ĠHT ML","Ġpopular ity","look ing","ĠSw ord","ĠAr ts","' )","Ġelect ron","clus ions","Ġinteg rity","Ġexclus ively","Ġgr ace","Ġtort ure","Ġburn ed","tw o","Ġ18 0","P rodu","Ġent reprene","raph ics","Ġg ym","ric ane","ĠT am","Ġadministr ative","Ġmanufacture r","Ġ vel","ĠN i","Ġisol ated","ĠMedic ine","Ġback up","Ġpromot ing","Ġcommand er","Ġfle e","ĠRus sell","Ġforg otten","ĠMiss ouri","Ġres idence","m ons","Ġrese mb","Ġw and","Ġmeaning ful","P T","Ġb ol","Ġhe lic","Ġwealth y","Ġr ifle","str ong","row ing","pl an","as ury","â̦ .","Ġexpand ing","ĠHam ilton","Ġrece ives","S I","eat ures","ĠAn im","RE E","P ut","Ġbrief ly","ri ve","Ġstim ul","Ġ`` (","Ġ __","Ġch ip","Ġha z","Ġpri ze","ĠTh ings","AC E","ul in","d ict","ok u","Ġassoci ate","ock ets","y outube","St ory","ateg ory","Ġm ild","ail ing","ĠY e","O rig","ĠK a","or ig","Ġpropag anda","Ġan onymous","Ġstrugg led","Ġout rage","AT ED","ĠBe ijing","r ary","Ġle ather","Ġworld s","Ġbroad er","12 5","id al","ĠBet ter","Ġt ear","E xt","Ġpropos als","Ġit er","ĠSqu ad","Ġvol unt","m i","D id","ĠP u","p in","Ġspeak ers","Ġb orders","Ġfig ured","= '","Ġsimultane ously","aed a","Ġcharg ing","Ġur ged","Ġcon j","25 6","ĠG ordon","mer ce","Ġdocument ary","Sh are","it ol","ON E","ĠG arden","h att","ĠThom pson","ane ous","ap ore","Ġt anks","Ġless ons","tr ack","Ġout standing","Ġvolunte ers","Ġsp ray","Ġmanag ers","l arge","Ġcamp s","Ġart ificial","ĠR u","Ġb ags","th al","Ġcompat ible","ĠBl ade","Ġf ed","Ġarg ues","F I","Ġunf air","Ġcor n","Ġoff set","Ġdirect ions","Ġdisappoint ed","ĠCon vention","Ġview ing","M E","oc ity","Ġtown s","Ġlay ers","Ġro lled","Ġjump ed","Ġatt ribute","Ġun necess","inc oln","Ġsupp ose","ĠNet her","ch a","Ġbur ied","Ġsix th","B en","ress ing","OU R","Ġw ound","Ġcy cl","Ġmechan isms","Ġcongress ional","ĠE lement","Ġagre ements","Ġdec or","Ġclos est","ĠM it","Go ogle","} }","Ġm ixture","Ġflu id","S ign","ĠSch olar","Ġp ist","ask et","ab ling","Ġrac ing","he ro","ri el","ass y","Ġche aper","b en","Ġvert ical","amac are","ĠRead ing","g ments","Ġhelic op","Ġsacr ifice","ay a","p aren","V A","ĠL es","ĠStud io","Ġviol ations","ĠAn na","ac er","é ¾","ĠR at","ĠBe ck","ĠD ick","ĠA CT","Ġcomp osition","Ġtext ure","ĠO wn","Ġsmart phone","ĠN A","Ġfor b","im port","Ġdef ending","il st","re r","Ġo h","ĠJere my","Ġbank ing","cept ions","Ġrespect ive","/ .","Ġdr inks","ĠW i","Ġb ands","ĠL iverpool","Ġg rip","ĠB uy","Ġopen ly","Ġreview ed","per t","Ġver ify","ĠCo le","ĠW ales","M O","Ġun pre","Ġshel ter","ĠIm perial","Ġgu i","ĠD ak","Ġsuggest ions","Ġexplicit ly","Ġsl ave","Ġblock chain","Ġcompet ing","Ġprom ising","S ON","Ġsoc cer","Ġconst itution","4 29","Ġdist ract","ĠU ser","es ides","ĠMet hod","ĠTok yo","Ġaccompan ied","Cl ient","s ur","al og","Ġident ification","Ġinv asion","as ma","Ġindust ries","pp ers","Ġsub tle","ĠUn it","n atural","Ġsurv ived","Ġfl aw","ĺ ħ","ĠH oll","Ġdef icit","Ġtut orial","ĠCh ance","Ġarg uing","Ġcontem porary","Ġinteg ration","for ward","Ġt um","it is","Ġh iding","ĠD omin","ĠT an","ĠB uilding","ĠV in","Ġspokes person","ĠNot es","Ġemer ging","Ġprepar ation","Ġpro st","Ġsuspect s","Ġaut onom","D escription","Ġdeal t","ĠP ear","Ġstead y","Ġdecre ased","Ġso vere","ĠCl in","Ġgrad ually","ors es","ĠW AR","S erv","ãĤ ¢","h r","Ġd irty","ĠB arn","ĠB C","Ġd il","Ġcal endar","Ġcompl iance","Ġch amber","b b","Ġpass enger","ate ful","ĠT itle","ĠSyd ney","ĠG ot","Ġdark ness","Ġdef ect","Ġpack ed","ass ion","Ġgod s","Ġh arsh","IC K","le ans","Ġalgorith m","Ġoxy gen","Ġvis its","Ġbl ade","Ġkil omet","ĠKent ucky","Ġkill er","P ack","enn y","Ġdiv ine","Ġnom ination","be ing","Ġeng ines","Ġc ats","Ġbuff er","ĠPh ill","Ġtra ff","AG E","Ġtong ue","Ġrad iation","ere r","m em","ĠExpl icit","é¾ į","Ġcou ples","Ġphys ics","ĠMc K","Ġpolit ically","aw ks","ĠBl oom","Ġwor ship","e ger","ut er","ĠF O","Ġmat hemat","Ġsent enced","Ġdis k","ĠM arg","Ġ/ *","P I","Ġoption al","Ġbab ies","Ġse eds","ĠScott ish","Ġth y","] ]","ĠHit ler","P H","ng th","Ġrec overed","ing e","Ġpow der","Ġl ips","Ġdesign er","Ġdis orders","Ġcour age","Ġch aos","\" },{\"","Ġcar rier","b ably","H igh","ĠR T","es ity","l en","Ġrout es","u ating","F il","N OT","w all","s burgh","Ġeng aging","ĠJava Script","ore r","li hood","Ġun ions","ĠF ederation","ĠTes la","Ġcomple tion","ĠT a","Ġprivile ge","ĠOr ange","Ġne ur","paren cy","Ġb ones","Ġtit led","Ġprosecut ors","ĠM E","Ġengine er","ĠUn iverse","ĠH ig","n ie","o ard","Ġheart s","ĠG re","uss ion","Ġmin istry","Ġpen et","ĠN ut","ĠO w","ĠX P","in stein","Ġbul k","S ystem","ic ism","ĠMarket able","Ġpre val","Ġpost er","Ġatt ending","ur able","Ġlicens ed","ĠG h","et ry","ĠTrad able","Ġbl ast","à ¤","ĠTit an","ell ed","d ie","H ave","ĠFl ame","Ġprof ound","Ġparticip ating","Ġan ime","ĠE ss","Ġspec ify","Ġregard ed","ĠSpe ll","Ġs ons","own ed","Ġm erc","Ġexper imental","land o","h s","ĠDun geon","in os","Ġcomp ly","ĠSystem s","ar th","Ġse ized","l ocal","ĠGirl s","ud o","on ed","ĠF le","Ġconstruct ed","Ġhost ed","Ġsc ared","act ic","ĠIs lands","ĠM ORE","Ġbl ess","Ġblock ing","Ġch ips","Ġev ac","P s","Ġcorpor ation","Ġo x","Ġlight ing","Ġneighb ors","ĠU b","ar o","Ġbe ef","ĠU ber","F acebook","ar med","it ate","ĠR ating","ĠQu ick","Ġoccup ied","Ġaim s","ĠAdd itionally","ĠInt erest","Ġdram atically","Ġhe al","Ġpain ting","Ġengine ers","M M","ĠM ust","Ġquant ity","P aul","Ġearn ings","ĠPost s","st ra","ãĥ¼ ãĥ","Ġst ance","Ġdro pping","sc ript","Ġd ressed","M ake","Ġjust ify","ĠL td","Ġprompt ed","Ġscr ut","Ġspeed s","ĠGi ants","om er","ĠEd itor","Ġdescrib ing","ĠL ie","ment ed","Ġnow here","oc aly","Ġinst ruction","fort able","Ġent ities","Ġc m","ĠN atural","Ġinqu iry","Ġpress ed","iz ont","for ced","Ġra ises","ĠNet flix","ĠS ide","Ġout er","Ġamong st","im s","ows ki","Ġclim b","ne ver","Ġcomb ine","d ing","Ġcomp r","Ġsignific ance","Ġremem bered","ĠNev ada","ĠT el","ĠSc ar","ĠWar riors","ĠJ ane","Ġcou p","b as","Ġtermin al",", -","O H","Ġt ension","Ġw ings","ĠMy ster","�� ��","ĠUn like","val id","viron ments","ĠAl i","Ġn aked","book s","ĠM un","ĠG ulf","Ġd ensity","Ġdim in","Ġdesper ate","Ġpres idency","Ġ198 6","h y","IN D","Ġun lock","im ens","Ġhand led","ĠE b","Ġdisapp eared","Ġgen re","Ġ198 8","Ġdetermin ation","St ream","ik o","ap ters","Ġacknow ledge","J an","Ġcapital ism","P at","Ġ20 20","Ġpain ful","Ġcur ve","Ġbom bs","st orm","ĠMet al","en cer","ĠF ig","ĠA aron","anc hes","Ġins piration","Ġexha ust","t ains","ash i","Ġdesc ript","Ġr itual","ĠChel sea","Ġpromot ion","ĠH ung","ĠW ard","iv a","ĠE T","Ġto ss","all ow","ĠFranc is","D ep","Ġhapp iness","ĠGl ass","Ġbet a","Ġstreng then","N E","o a","Ġbutt ons","ĠMur ray","Ġkick ed","Qu est","ĠT alk","ĠS everal","ĠZ ero","Ġdr one","ul k","Ġc am","ĠM obile","Ġprevent ing","Ġret ro","ĠA x","Ġcru el","Ġflo at",". ),","Ġfil ing","ĠGr ant","ĠB or","Ġr ib","Ġchampions hip","ĠM erc","Ġsty les","Ġc ake","Ġbuild s","ĠS elf","io x","Ġep ic","oy d","B el","ĠSt ew",". (","ah u","ĠBe yond","Ġout s","Ġsol o","ĠT ree","Ġpres erve","Ġt ub","AR E","ro c","ĠIm pro","ĠW right","Ġbu nd","Ġtr aged","Ġoccas ional","b ian","Sec ond","r ons","Ġinter actions","form ed","s ing","Ġown s","Ġh ockey","Gener al","Ġlog ical","Ġexp end","Ġesc al","ĠGr iff","ĠC rown","ĠRes erve","Ġsto pping","Ġexc use","sec ond","Ġoper ated","Ġre aches","ĠMal ays","Ġpoll ution","ĠBrook lyn","Ġde lete","Ġhas h","Bl ock","ah a","âĢ ³","Ġsh orter","p iece","> >>","ĠM ormon","t or","Ġpartic les","ĠB art","ry ption","Ġad min","Ġsqu ee","VID IA","Ġcreat or","iam eter","ic ular","N BC","Ġgrab bed","Ġn odd","Ġr ated","Ġrot ation","Ġgr asp","Ġexcess ive","ĠE C","ĠWh it","Ġinvent ory","ault s","ĠF B","Ġe cosystem","Ġbill ions","Ġvent ure","n amed","Ġdef ender","out e","Inst ead","ir able","W ar","Ġassum ption","Ġb ite","Ġearth qu","t ail","sp ace","Ġgif ts","boy s","Ġinev itable","Ġstruct ural","Ġbenef icial","Ġcompe lling","h ole","erv ation","Ġco at","o j","inc arn","ĠY ears","Ġdetermin ing","Ġrhet oric","Ġbound aries","Ġwh ites","A nt","add y",") -","ra ham","eter min","Ġhar vest","ĠCon c","Ġlapt op","ĠM atch","Ġenjoy ing","cc a","oll ar","Ġtri ps","Ġadd iction","ĠS ak","Ġpow ered","Ġc ous","ĠRuss ians","ie re","Ġret rie","qu ality","Ġdiff er","Ġking dom","ĠL aur","ĠCap itol","Ġcon clusions","ĠAl tern","ĠN av","Ġtrans parent","B ER","G roup","ĠCom plete","Ġinf er","Ġint rig","Ġins ane","R O","oph ob","is en","qu al","Mich ael","Ġm useum","ĠP ope","Ġres et","r ative","f ive","Ġagg reg","itte es","osit ory","Ġcar b","ĠRec ord","Ġdec ides","ĠF ix","Ġexcept ions","ĠCommission er","un s","ĠEnvironment al","Ġlegend ary","ist ence","Ġtun nel","k m","Ġins ult","Ġt roll","Ġsh ake","Ġdet ention","qu es","ĠCh rome","ĠF iles","Ġsub t","Ġprospect s","Ġpro l","re nder","pro of","Ġperform ances","St r","Ġh ref","ern ame","Ġachieve ment","Ġf ut","F ull","ĠLe ban","go ogle","ãĥ Ī","amp a","May be","Ġproject ed","ĠE mb","Ġcol leg","Ġa wards","Ġâ Ķ","G old","ĠBl ake","ĠR aj","if ting","Ġp ending","Ġinst inct","Ġdevelop ments","Con nect","ĠM and","ĠW ITH","ĠPhilipp ines","prof ile","Ġalt ogether","ĠB und","ĠT D","oo oo","amp ed","ip h","Ġste am","Ġold est","Ġdet ection","ul pt","Ġ ç","ĠWay ne","200 6","f a","Ġcir cles","ĠF u","Ġdon ors","appropri ate","ĠDak ota","j amin","Ġmotiv ated","Ġpurch ases","ĠLouis iana","ĠS pl","Ġgl obe","Ġ10 5","z ip","c all","Ġdepart ments","Ġsustain able","10 5","ĠO P","if iers","Ġprevent ed","Ġinc omp","ĠComm ander","Ġdom inated","Ġ »","Ġinvest ed","Ġcomplex ity","Ġin cl","Ġens uring","Ġreal m","yn c","ĠInd ependent","r ained","ĠJ en","ĠFl ight","Ġat he","Ġspec ulation","ĠT E","oc ate","t ic","Ġpl aint","her ry","Ġto y","Ġ1 11","Ġpl ates","st atus","ĠIs a","Ġdev oted","C op","ĠE S","25 5","ur rency","M ain","Ġsl aves","Ġpe pper","Ġqu otes","Ġce iling","ĠF ish","Ġtrans formation","Ġfra ction","Ġadvant ages","Ġto ile","Ġstun ning","Ġmo ist","bre aking","s i","ĠL ocation","ĠMed ium","Ġtext s","Ġu gly","Ġb io",". âĢĶ","ĠB ased","Ġtr ains","ĠW ing","ĠAn cient","ĠRec ords","ĠH ope","Spe cial","ades h","ob i","[ /","Ġtempor arily","V er","h u","os er","Ġover night","Ġm amm","ĠTre asury","ĠV enezuel","ĠMeg a","Ġt ar","Ġexpect s","bl ack","or ph","\\\\ \\\\","Ġaccept ance","Ġrad ar","s is","Ġjun ior","Ġfram es","Ġobserv ation","ac ies","P ower","ĠAdv anced","M ag","olog ically","ĠMe chan","Ġsent ences","Ġanaly sts","augh ters","force ment","Ġv ague","Ġcl ause","Ġdirect ors","Ġeval uate","Ġcabin et","M att","ĠClass ic","A ng","Ġcl er","ĠB uck","Ġresear cher","Ġ16 0","Ġpoor ly","Ġexperien cing","ĠP ed","ĠMan hattan","Ġfre ed","Ġthem es","ad vant","Ġn in","Ġpra ise","10 4","ĠLib ya","b est","Ġtrust ed","Ġce ase","Ġd ign","D irect","Ġbomb ing","Ġm igration","ĠSci ences","Ġmunicip al","ĠA verage","Ġgl ory","Ġreve aling","Ġare na","Ġuncertain ty","Ġbattle field","ia o","G od","Ġc inem","ra pe","el le","ap ons","Ġlist ing","Ġwa ited","Ġsp otted","ke ley","ĠAud io","e or","ard ing","idd ing","ig ma","ĠN eg","Ġl one","Ġ ----","ex e","d eg","Ġtrans f","Ġwas h","Ġsl avery","Ġexpl oring","ĠW W","ats on","Ġen cl","l ies","ĠC reek","Ġwood en","Man ager","ĠBr and","um my","ĠAr thur","Ġbureau cr","Ġbl end","ar ians","F urther","Ġsupposed ly","Ġwind s","Ġ19 79","Ġgrav ity","Ġanalys es","ĠTra vel","ĠV eter","Ġd umb","Ġaltern ate","g al","Ġconsum ed","Ġeffect iveness",".' '","Ġpath s","ond a","L A","ĠStr ong","Ġen ables","Ġesc aped","Ġ\" \"","Ġ1 12","Ġ198 3","Ġsm iled","Ġtend ency","F ire","Ġp ars","ĠR oc","Ġl ake","Ġf itness","ĠA th","ĠH orn","Ġh ier","Ġimp ose","m other","Ġp ension","ic ut","bor ne","ic iary",". _","ĠS U","Ġpol ar","is y","eng u","itial ized","AT A","w rite","Ġexerc ises","ĠD iamond","ot ypes","Ġharm ful","on z","Ġprint ing","st ory","Ġexpert ise","ĠG er","Ġtraged y","ĠF ly","Ġd ivid","amp ire","st ock","M em","Ġre ign","Ġun ve","Ġam end","ĠProp het","Ġmut ual","ĠF ac","Ġrepl acing","H ar","ĠCirc uit","Ġthro at","ĠSh ot","Ġbatter ies","Ġto ll","Ġaddress ing","ĠMedic aid","Ġp upp","ĠN ar","ol k","Ġequ ity","M R","ĠHis pan","ĠL arge","m id","D ev","Ġexp ed","Ġdem o","ĠMarsh all","erg us","Ġf iber","Ġdiv orce","ĠCre ate","Ġsl ower","ĠPark er","ĠStud ent","ĠTr aining","Ret urn","ĠT ru","Ġc ub","ĠRe ached","Ġpan ic","Ġqu arters","Ġre ct","Ġtreat ing","Ġr ats","ĠChristian ity","ol er","Ġsac red","Ġdecl are","ul ative","et ing","Ġdeliver ing","est one","Ġt el","ĠL arry","Ġmet a","ac cept","art z","ĠRog er","hand ed","Ġhead er","Ġtra pped","ĠCent ury","Ġkn ocked","ĠOx ford","Ġsurviv ors","b ot","Ġdemon stration","Ġd irt","Ġass ists","OM E","ĠD raft","ortun ate","fol io","pe red","ust ers","g t","ĠL ock","Ġjud icial","ver ted","Ġsec ured","out ing","ĠBook s","Ġhost ing","Ġlif ted","l ength","Ġj er","Ġwhe els","ĠR ange","umbn ails","Ġdiagn osis","te ch","ĠStew art","ĠP ract","Ġnation wide","Ġde ar","Ġoblig ations","Ġgrow s","Ġmand atory","Ġsusp icious","! '","A pr","G reat","Ġmort gage","Ġprosecut or","Ġeditor ial","ĠK r","Ġprocess ed","ung le","Ġflex ibility","Ear lier","ĠC art","ĠS ug","Ġfoc uses","Ġstart up","Ġbre ach","ĠT ob","cy cle","ãĢ Į","ro se","Ġb izarre","ãĢ į","Ġveget ables","$ $","Ġret reat","osh i","ĠSh op","ĠG round","ĠSt op","ĠHawai i","ĠA y","Per haps","ĠBe aut","uff er","enn a","Ġproduct ivity","F ixed","cont rol","Ġabs ent","ĠCamp aign","G reen","Ġident ifying","Ġreg ret","Ġpromot ed","ĠSe ven","Ġer u","ne ath","aug hed","ĠP in","ĠL iving","C ost","om atic","me ga","ĠN ig","oc y","Ġin box","Ġem pire","Ġhor izont","Ġbr anches","Ġmet aph","Act ive","ed i","ĠFil m","ĠS omething","Ġmod s","inc ial","ĠOrig inal","G en","Ġspir its","Ġear ning","H ist","Ġr iders","Ġsacr ific","M T","ĠV A","ĠS alt","Ġoccup ation","ĠM i","Ġdis g","lic t","Ġn it","Ġn odes","e em","ĠP ier","Ġhat red","ps y","ãĥ ī","Ġthe ater","Ġsophistic ated","Ġdef ended","Ġbes ides","Ġthorough ly","ĠMedic are","Ġbl amed","arent ly","Ġcry ing","F OR","pri v","Ġsing ing","ĠI l","Ġc ute","o ided","olit ical","ĠNe uro","å ¤","Ġdon ation","ĠEag les","ĠG ive","T om","Ġsubstant ially","ĠLic ense","ĠJ a","Ġg rey","ĠAn imal","ĠE R","ĠU nd","Ġke en","Ġconclud e","ĠMississ ippi","Eng ine","ĠStud ios","P ress","o vers","ll ers","Ġ3 50","ĠR angers","Ġr ou","ert o","E p","iss a","iv an","Ġse al","ĠReg ist","dis play","Ġwe aken","u um","ĠComm ons","ĠS ay","Ġcult ures","Ġl aughed","Ġsl ip","Ġtreat ments","iz able","m art","ĠR ice","Ġbe ast","Ġob esity","ĠLa ure","ig a","Wh ich","hold er","Ġelder ly","Ġp ays","Ġcompl ained","Ġc rop","Ġpro c","Ġexplos ive","ĠF an","ĠAr senal","A uthor","ef ul","Ġme als","Ġ( -","id ays","Ġimag ination","Ġann ually","Ġm s","as ures","H ead","ik h","m atic","Ġboy friend","ĠCom puter","Ġb ump","Ġsur ge","ĠCra ig","ĠKir k","D el","medi ate","Ġscen arios","ĠM ut","ĠSt ream","Ġcompet itors","Ù Ħ","ĠStan ford","ĠRes ources","az ed","b age","Ġorgan is","ĠRe lease","Ġsepar ately","Ġha bits","Ġmeasure ments","ĠCl ose","Ġaccomp any","Ġg ly","Ġt ang","ĠR ou","Ġplug in","Ġcon vey","ĠChall enge","oot s","j an","Ġcur s","ĠRel ations","ke eper","Ġapproach ing","p ing","Spe aking","Ġarrang ement","ĠV I","are ttes","Ġaffect ing","Ġperm its","b ecause","Ġu seless","ĠH us","!! !!","Ġdestro ying","Un fortunately","Ġfasc inating","S em","Ġelect oral","Ġtrans parency","ĠCh aos","Ġvolunte er","Ġstatist ical","Ġactiv ated","ro x","We b","H E","ĠHamp shire","is ive","M ap","Ġtr ash","ĠLaw rence","st ick","C r","Ġr ings","EX T","Ġoper ational","op es","D oes","ĠEv ans","Ġwitness ed","P ort","Ġlaunch ing","ec onom","w ear","ĠPart icip","um m","cul es","ĠR AM","ĠT un","Ġass ured","Ġb inary","Ġbet ray","Ġexpl oration","ĠF el","Ġad mission","it ated","S y","Ġav oided","ĠSim ulator","Ġcelebr ated","ĠElect ric","¥ ŀ","Ġcl uster","itzer land","he alth","L ine","ĠN ash","at on","Ġsp are","Ġenter prise","ĠD IS","clud es","Ġfl ights","Ġreg ards","Ġà Ĺ","h alf","Ġtr ucks","Ġcontact s","Ġunc ons","ĠCl imate","Ġimm ense","N EW","oc c","ect ive","Ġemb od","Ġpat rol","Ġbes ide","Ġv iable","Ġcre ep","Ġtrig gered","ver ning","Ġcompar able","q l","Ġg aining","ass es","Ġ( );","ĠG rey","ĠM LS","s ized","Ġpros per","\" ?","Ġpoll ing","Ġsh ar","ĠR C","Ġfire arm","or ient","Ġf ence","Ġvari ations","g iving","ĠP i","osp el","Ġpled ge","Ġc ure","Ġsp y","Ġviol ated","Ġr ushed","Ġstro ke","ĠBl og","sel s","ĠE c",",' '","Ġp ale","ĠColl ins","ter ror","ĠCanad ians","Ġt une","Ġlabor atory","Ġn ons","t arian","Ġdis ability","ĠG am","Ġsing er","al g","ĠSen ior","Ġtrad ed","ĠWar rior","Ġinf ring","ĠFrank lin","Ġstr ain","ĠSwed ish","Ġsevent h","ĠB enn","ĠT ell","Ġsynd rome","Ġwond ered","id en","++ ++","ig o","Ġpur ple","Ġjournal ism","Ġreb el","Ġf u","bl og","Ġinv ite","ren cies","ĠCont act","Is rael","ĠCont ent","Ġche er","Ġbed room","ĠEngine ering","ĠQue ens","Ġd well","ĠPlay Station","ĠD im","ĠCol on","l r","Ġoper ates","Ġmotiv ation","US A","ast ered","C ore","ĠTr uth","ol o","OS E","ĠMem ory","Ġpred ec","Ġan arch","Ġ19 20","ĠY am","à ¨","b id","Ġgr ateful","Ġexc itement","Ġtre asure","Ġlong est","ct ive","Ġdes erves","Ġreserv es","Ġcop s","ĠOtt awa","ĠEgypt ian","ank ed","Ġart if","Ġhypot hesis",": /","Ġpurch asing","Ġlove ly","H P","Ġdiv ide","Ġstrict ly","Ġquestion ing","Ġtaxp ayers","ĠJ oy","Ġroll s","ĠHe avy","Ġp orts","Ġmag netic","Ġinf lamm","Ġbr ush","t ics","â ĪĴ","Ġbott les","pp y","Ġp add","ãĤ ¯","m illion","Ġdevast ating","Ġcomp iled","Ġmed ication","Ġtw elve","ĠPer ry","Sp ace","im b","y our","Ġle aked","ĠT ar","Ġun ity","Ġinfect ed","Ġtravel ed","ID E","ĠMc Donald","t xt","ĠPr inc","Ġinter ven","ĠTai wan","ĠP ow","Ġbe aring","ĠTh read","Ġz ones","iz ards","un ks","Ch apter","ll or","Ġ ·","Ġw ounds","Ġdisc retion","Ġsucceed ed","ik ing","Ġicon ic","C all","Ġscreen ing","ĠM is","ict s","Ġmin isters","Ġsepar ation","Pl ayer","Ġb ip","Ġbel oved","Ġcount ing","ĠE ye","ar ound","ing ing","Ġtable t","Ġoff ence","in ance","h ave","ĠInf o","ĠNin ja","Ġprotect ive","ĠC ass","M ac","ĠQual ity","N orth","Ġ ic","ĠCub a","ĠChron icle","ĠPro perty","Ġfast est","ot os","ĠG erm","OW N","Ġbo om","ĠStan ley","ergus on","Ġcle ver","Ġent ers","m ode","ter ior","ĠS ens","Ġlin ear","AR K","Ġcomp aring","Ġpure ly","Ġsaf er","ĠPot ter","Ġc ups","R T","Ġgl uc","Ġatt ributed","Ġdu pl","ĠP ap","Ġprec ious","Ġp a","iction ary","ĠT ig","ĠTo o","ol utions","st an","Ġrob ots","Ġlob b","Ġstat ute","Ġprevent ion","w estern","16 0","ĠAct ive","ĠMar ia","h al","N one","ell ar","ĠK B","ĠPart ners","ĠSing le","ĠFollow ing","ang o","ac ious","Ġth ou","Ġk g","Ġinflu ential","ĠFriend s","S ur","ain ted","Ġfor ums","Ġst arter","Ġcitizens hip","ĠE lection","on ge","ot ation","os ph",";; ;;","ut ical","p ur","ere n","Ġaccus ations","bit ious","ab bit","ĠOr d","Post ed","ir k","Ġsens itivity","ic he","ĠAm y","ĠF ab","Ġsum mit","Ġped est","Ġrub ber","Ġagric ultural","Ġcan cel","A E","Ġin aug","Ġcont am","Ġfirm ly","i w","st age","ĠK an","Ġt ier","Ġinv ention","Ġtransl ated","ĠR ules","B ox","Tw itter","ID S","Ġp izza","Ġdeb ug","ĠD rop","v s","Ġh orses","b ig","Ġb oring","Ġh ood","ĠMcC ain","at ched","ĠBro s","Ġsk ip","Ġess ay","st at","ĠLeg ends","Ġam munition","au c","Ġshoot er","Ġun h","Ġsuppl ied","Ġgener ic","ĠS K","ib an","yr ics","Ġ25 5","Ġclim bing","Form er","Ġfl ip","Ġjump ing","Ġfrust ration","ĠTer ry","Ġneighborhood s","Ġmed ian","be an","Ġbr ains","Follow ing","Ġsh aped","Ġdraw s","Ġal tered","J ack","Ġrecip es","Ġsk illed","we alth","ach i","e lection","Ġbehavi ors","de als","ĠU ntil","F e","Ġdecl aration","mar ks","ĠBet ween","cel ona","Ġres on","Ġbub ble","Am ong","Ġim perial","G S","Ġfemin ist","200 5","ĠK yle","Ġaccount ing","ĠTe le","ĠT yr","Ġconnect ing","Ġre hab","ĠP red","s im","Ġmeant ime","Ġphys ician","M W","ĠCamp bell","ĠBr andon","Ġcontribut ing","ĠR ule","ĠWe ight","ĠN ap","Ġinter active","Ġv ag","Ġhel met","ĠCom b","f our","Ġsh ipped","Ġcomple ting","ĠP D","PD ATE","Ġspread ing","Ġsc ary","erv ing","ĠG as","Ġfr ank","s chool","Ġrom antic","Ġstab il","R ob","Ġaccur ately","Ġac ute","ĠH ann","Ġsymbol s","Ġcivil ization","ĠA W","Ġlight ning","Ġcons iders","Ġven ue","Ġ ×","Ġo ven","ĠS F","h is","Ġn u","ĠLear n","Ġpe oples","Ġst d","Ġsle e","Ġs lic","ĠStat istics","Ġcor ners","ĠB aker","Ġ: )","ment ation","ol ver","Ġlaugh ing","ĠT odd","ond e","ĠH ills","Ġn uts","ĠW oman","pl ane","Ġl iver","ĠIn side","S orry","Ġagre es","Ġfund ament","ĠF isher","Ġa uction","Ġthread s","gl as","ĠBas ic","ĠN at","Ġlack ing","Ġceleb ration","j u","Ġs illy","E uro","Ġt att","ight y","cont rolled","T est","ĠSing h","Ġr age","Ġrh yth","o ffic","ĠPh antom","Ġhead lines","Ġrespond ing","ĠMor ning","Ġvit amin","Ġboot s","ĠS ite","al in","p i","Ġvir al","ĠU C","D ER","ĠSe x","Ġst ocks","c urrent","Ġch urches","ĠR are","ĠMur phy","Ġden ial","ĠG aming","Ġtou g","Ġn ick","Ġm akers","ĠRon ald","Ġgener ous","ĠD oc","ĠMor ris","Ġtransform ed","ĠN ormal","Ġ10 4","ĠKick starter","ĠUp on","On line","ĠI RS","Ġw rap","Ġl oving","Ġarri ves","ĠD ue","Ġhe ter","ĠM ade","Ġrent al","Ġbelong s","Ġatt orneys","Ġcro ps","Ġmat ched","ul um","ol ine","10 9","Ġdis par","Ġbuy ers","ĠCam bridge","Ġeth ics","rou ps","Ġjust ified","Ġmarg inal","Ġrespect ed","win ning","Ġnodd ed","ĠSer ge","ĠForm er","C raft","######## ########","ĠWar ner","Ġd ash","et e","Ġent ert","ĠE scape","out heast","Ġkn ees","ĠB omb","Ġr ug","P ass","Ġatt itudes","go vernment","ĠPri or","Ġqual ities","Ġnot ification","ĠPh one","l ie","Ġanticip ated","ĠCom bat","ĠBar ry","Ġ198 2","Us ers","on er","Ġcomput ing","ĠConnect icut","Ġless er","Ġpe ers","ĠC u","Ġtechn ically","Ġsub mission","ĠUn iversal","Ġman ually","our ge","Ġrespond ents","ĠB TC","ĠH ost","Ġf are","ĠB ird","Ġrece ipt","al so","Ġj ack","Ġagric ulture","Ġsk ull","Ġ! =","Ġpass ive","ĠC I","Ġsoc ieties","Ġremind ed","Ġinter ference","B uy","Ġâ ľ","g on","Ġscrut iny","ĠW itch","Ġconduct ing","Ġ ãĥ","Ġexch anges","ĠMit chell","Ġinhab it","Ġtw ist","B D","Ġwhere ver","group on","Ġj okes","ĠBen jamin","ĠR andom","fr ame","ĠL ions","Ġhighlight ed","ĠArk ansas","E nt","Ġp ile","Ġpre lim","g s","mind ed","Ġfel ony","ĠG A","ĠL uck","Ġpract ically","ĠB os","Ġact ress","D am","ĠB ou","Ġvis a","Ġembed ded","Ġhy brid","Ġear liest","Ġsoon er","s ocial","ĠH A","Ġste ep","Ġdis advant","Ġexplo it","ĠE gg","ĠUlt ra","Ġnecess ity","L ocal","ie ge","Ġd ated","Ġmass es","Ġsubsc ription","pl ess","Ġan onym","Ġpresum ably","Bl ue","The ir","asket ball","ĠPhil ip","Ġcom ed","load ed","r ane","Ġref lection","Ch ina","Ġext ends","Ġform ing","Ġund ers","200 1","Ġgr at","Ġconcent rations","Ġins ulin","Ġsec ular","Ġwh ilst","Ġwin ners","Ad vertisements","Ġdeliber ately","ĠWork ing","Ġs ink","et ics","d ale","Ġmand ate","Ġg ram","Ġvac ation","Ġwarn ings","ri pp","ĠTH AT","Ġcomment ary","Ġint u","Ġa est","Ġreason ing","Ġbreak down","ĠZ ombie","Ġ-- >","ĠPolit ical","c ott","Ġthr ust","Ġtechn ological","Ġdec iding","Ġtraff icking","L ong","W elcome","pr ising","ĠCommun ications","Ġend ors","Ġsw ift","Ġmetab ol","co ins","res a","ĠHT TP","Ġen roll","ĠH appy","us r","int age","Ġ[ \"","u ably","ĠM aterial","Ġrepe al","Se pt","k h","ĠMod i","Ġunder neath","ĠI L","sh ore","Ġdiagn osed","ace utical","Ġsh ower","au x","ĠSw itch","ĠStre ngth","Ġj ihad","n ational","Ġtra uma","uss y","on i","Ġcons olid","Ġcal ories","ĠF lynn","ag ged","16 8","ĠP ink","Ġfulf ill","Ġch ains","Ġnot ably","ĠA V","L ife","ĠCh uck","m us","ĠUr ban","ĠH end","Ġdep osit","ĠS ad","Ġaff air","OR K","ie val","ĠF DA","Ġt rop","ĠOver all","Ġvirt ue","Ġsatisf action","au nd","Ġl un","ĠSw itzerland","ĠOper ation","pro cess","Ġsh ook","Ġcount ies","le ased","ĠCharl otte","1 12","Ġtrans cript","Ġre dd","p ush","ĠHe y","ĠAn alysis","[ \"","Ġaltern atives","ard less","Ġele ph","Ġpre jud","ĠLe af","H aving","ĠH ub","Ġexpress ions","ĠVol ume","Ġshock ing","ĠRed s","Ġread ily","Ġplan ets","ad ata","Ġcollaps ed","ĠMad rid","Ġir rit","i pper","ĠEn c","ĠW ire","Ġbu zz","ĠG P","ash a","Ġaccident ally","ur u","Ġfrust rated","ĠS A","Ġhung ry","ĠH uff","Ġlab els","ant o","ĠE P","Ġbar riers",") |","ĠBer keley","ĠJ ets","Ġp airs","ĠL an","J ames","ĠB ear","Ġhum or","ĠLiber ty","Ġmagn itude","Ġag ing","ĠM ason","Ġfriends hip","umb ling","Ġemer ge","Ġnewsp apers","Ġam bitious","ĠRich ards","atern al","Ġ198 1","Ġcook ies","Ġsc ulpt","Ġpur suit","L ocation","Ġscript s","p c","Ġarrang ements","Ġd iameter","Ġl oses","am ation","Ġl iqu","ĠJ ake","aret te","Ġunderstand s","ĠZ en","v m","Ġappro ve","Ġw ip","Ġult ra","Ġint end","ĠD I","asc ular","Ġst ays","ĠK or","ĠK l","Ġinvest ing","L a","Ġbelie ving","b ad","m outh","Ġtaxp ayer","ãĥ ĥ","ĠQue bec","Ġl ap","ĠSw iss","d rop","Ġdr ain","ir i","et c","ft en","ĠN ex","Ġst raw","Ġscream ing","Ġcount ed","Ġdam aging","Ġamb assador","cent ury","Ġpro x","Ġarrest s","u v","il ateral","ĠCh arg","Ġpresc ribed","Ġindepend ently","Ġf ierce","ĠB aby","Ġb rave","Ġsu its","= >","Ġbas eline","ĠR ate","Ġis lands","Ġ( (","g reen","ix els","Ġname ly","ĠVill age","th an","am y","V ersion","g mail","ential s","ĠS ud","ĠMel bourne","Ġarri ving","Ġquant um","e ff","rop olitan","T ri","Ġfun eral","ĠI R","ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ","ĠC ob","it ably","Ġt urb","Ġcomb o","Re view","Ġdeploy ment","u ity","ĠB ott","Ġinv isible","Ġrender ing","Ġunl ocked","Ġa qu","ĠVlad imir","Ġp ad","ĠBr ain","ĠLeg acy","dr agon","ĠKurd ish","Ġsound ed","Ġdet ained","ĠD M","g ary","Ġd aughters","Ġdistur bing","uk a","ĠPar ad","Ġt ast","Ġunf ortunate","Ġu l","em in","Ġattend ance","tr l","Ġpar ks","ĠMem orial","ĠAl ice","oth y","gu ard","ĠD ise","ĠSh an","ĠFor um","R ich","Ġshif ted","ue z","Ġl ighter","ĠMag n","Ġc od","S ch","ham mad","P ub","3 50","ĠP okemon","Ġprot otype","Ġun re","B ase","ĠStud ents","ĠRep ly","ĠCommun ist","Ġg au","ĠTy ler","I Z","Ġparticip ated","Ġsup rem","ĠDet ails","Ġvessel s","ro d","Ġt ribe","ke ep","Ġassum ptions","Ġp ound","Ġcr ude","ĠAv ailable","Ġswim ming","Ġin clusion","Ġadv ances","c ulation","Ġconserv ation","Ġover d","ĠBuff alo","Art icle","ed ge","Ġaw a","ĠMad ison","Ġsid ew","Ġcat ast","ĠK rist","uc le","ĠHigh way","ĠTer ror","Ġactiv ation","Ġuncons cious","ĠSat an","ĠSus an","ill ery","Ġarr anged","i op","Ġrum ors","ur ring","th ink","ĠKe ith","ĠK ind","Ġavoid ing","by n","n ut","ĠSpe aker","r us","n ames","Ġgu ilt","ĠOlymp ics","Ġsa il","ĠM es","lev ant","ĠColumb us","a ft","C ity","S outh","ĠHar vey","ĠP un","S everal","Ġment ally","Ġimp ress","m ount","ĠUb untu","âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ","ĠSuper man","ĠMP s","Ġintent ions","ĠR acing","Ġlike lihood","Ġ2 40","T otal","Ġto ys","ĠW atson","Ġur ge","L ear","ĠP aper","Ġoccur ring","ĠB eng","ĠC ert","Ġst ones","T im","ĠTw in","z b","ĠD ynam","Ġpolit ician","k ens","ĠEnter prise","UT ERS","Ġab ol","Ġref resh","Ġarbit rary","pe ction","Ġtrou bles","Ġ} );","t v","Ġpil ots","Ġdist ribute","Ġaud it","Ġp ause","orig inal","Ġr ivals"," £","F ig","T L","ab il","ry ing","L in","ion ed","l on","Ġf ancy","Ġcr ashed","Ġt ract","Ġshe d","Ġcons ume","B ased","down load","in it","Ġvolt age","Int rodu","Ġcondem ned","ĠFin ance","res pect","Ġex cluded","Ġestablish ing","her ic","Ġher itage","Ġspect acular","Ġun st","ĠSnow den","ĠL ane","S an","Ġprotect ions","st ruction","inc inn","Ġmac ro","C ustom","ios ity","Ġes p","Ġfunction ing","Ġm ush","Ġp uzzle","Ġeth ical","M al","Ġgo verning","ĠF erguson","Ġrest ored","Ġst ressed","ĠCoun ter","ĠK as","cl ip","AN S","Ġse iz","U K","by ss","old own","ap i","Ġperman ently","oun ters","W est","Th rough","L ight","at oes","Ġne at","Ġc ord","ure r","Ġsevere ly","ĠA ven","Ġinter rog","Ġtri ple","G iven","N umber","Ġar ise","Ġs her","pl ant","Ġfl ower","ĠC ou","Ġat e","Ġnew er","b ul","Ġmean while","ĠL air","Ġadjust ment","ĠCop yright","Ġd ivers","i ological","Ġgam ers","o at","Ġhistor ically","Ġanal og","Ġlong time","Ġpres cription","ĠM ist","ĠHy per","ĠM aine","ĠDe ity","Ġmulti pl","ĠRe incarn","ĠH yd","ĠP ic","S il","r ants","ĠC ris",". ;","( {","epend ence","Ġrec y","ate ur","Ġqu ad","Ġgl ob","Ġcon ced","te am","Ġcapital ist","ĠL ot","Ġroy al","ĠCy ber","Ġblack s","met ic","ri v","ĠD anny","Ġsp o","ĠR O","Ġanim ated","rypt ed","ĠDep uty","Ġrend ered","F E","Ġstre ak","Ġcloud s","ĠDou g","~~~~ ~~~~","Ġdisc our","ĠVe h","Ġpsych ology","ĠJ ourney","Ġcry stal","ĠFro st","Ġsuspic ion","Ġrel ate","or us","ĠC rypt","ĠN VIDIA","com ed","ut ing","incinn ati","Ġvulner ability","ost ic","Ġisol ation","Ġcool ing","ĠCoal ition","Ġ1 19","F our","ĠDe al","Ġâ ī","se mble","ram ent","ĠBar celona","Ġ10 2","Ġcoc aine","ocaly pse","F eb","ogen ic","Ġmut ation","Ġcrypt oc","ĠK el","ĠG it","a is","Ġs isters","AN K","Ġactiv ate","T er","Ġd read","yl on","Ġprop ri","A ust","ĠDef ault","Ġout door","Ġshe er","ce ive","Ġg ently","Ð ¾","Pro gram","Ġâ ĨĴ","Ġve gan","ĠCr us","Ġrespons ibilities","ĠH R","OL D","Ġprev ents","Ġst iff","ĠW ere","Ġathlet ic","ĠSc ore","Ġ) :","Ġcolumn s","ĠL oc","av ailable","ĠF ram","ĠS essions","Ġcompan ion","Ġpack s","14 0","ĠKn ights","Ġf art","Ġstream s","Ġsh ore","Ġapp eals","ĠPer formance","h aul","ĠSt ra","ĠN ag","10 3","ĠTrans portation","B B","E v","z an","P ublic","Ġtw in","uls ion","M ult","Ġelect ro","Ġstat ue","ation ally","ĠN ort","Ġins pection","/ *","ig ue","Ġcomp assion","ĠT ales","ĠSte in","ĠSc reen","ĠB ug","ĠL ion","g irl","Ġwithdraw al","Ġobject ives","Ġblood y","Ġprelim inary","Ġj acket","Ġdim ensions","ĠC ool","ĠOcc up","Ġw reck","Ġdoub led","ank ing","Ġ19 75","Ġglass es","ĠW ang","pro v","P ath","connect ed","ĠMult i","ĠNor way","agon ist","Ġfe ared","Ġtouch ing","Ġarg uably","¯¯¯¯ ¯¯¯¯","ĠNC AA","che m","Ġsp at","ĠW WE","ĠC el","ig ger","Ġattack er","ĠJo in","ob ject","ett a","Ġelim inated","d et","Ġdest ruct","ĠLuc as","ct uary","18 0","ĠBr ady","ĠBl ues","B ay","au kee","Ġtim eline","Ġdeleg ates","w ritten","uff icient","Ġsh apes","Cop yright","ou ble","serv ice","Ġp ione","Ġcolleg es","Ġrow s","Ġsp ite","Ġassess ed","3 60","Ġle ase","Ġconfident ial","ck er","ĠMan ning","ĠV oice","Ġse aled","Ġcalcul ate","N O","ĠAss istant","Ġteen ager","ul ent","ather ine","Ġm ock","Ġd iamond","Ġf est","Ġsw itched","Ġres ume","ĠPu erto","Ġl anes","ir ation","ĠSimilar ly","Ġro d","ĠS el","ĠPal ace","ĠLim ited","e ous","Ġvar iant","Ġw ard","Ġ) )","Sh ow","OO K","A lex","ĠN ep","br is","ĠWik ipedia","Ġexcept ional","Ġman ages","ĠD raw","Ag ain","Ġco pper","ut t","Ġex ports","Ġport folio","Ġelev ated","R ated","ĠOther wise","ĠT act","ĠShe l","ĠT X","\" âĢĶ","Ġres ur","ĠW a","ven ant","Ġmon etary","pe ople","E mail","Ġfif ty","ĠS weet","ĠMalays ia","Ġconf using","ĠR io","ud a","uten ant","\" );","Ġpra ised","Ġvol umes","t urn","Ġm ature","Ġnon profit","Ġpassion ate","ĠPriv ate","Ġ10 3","Ġdesc end","ç ¥ŀ","uff y","head ed","Whe ther","ri en","ze ch","be it","Ġch rom","ĠMc M","Ġd ancing","Ġe leg","ĠNot iced","11 5","Ġadvoc acy","ENT S","amb ling","ĠMin or","ĠF inn","Ġprior ities","Ġthere of","ĠSt age","ĠRog ers","Ġsubst itute","ĠJ ar","ĠJeff erson","Ġlight ly","10 2","ĠL isa","u its","ys ical","Ġshif ts","Ġd rones","Ġwork place","Ġres id","ens ed","ah n","Ġpref erences","ser ver","Ġdeb ates","d oc","ĠGod s","Ġhelicop ter","Ġhon our","Ġconsider ably","ed ed","ĠF emale","ĠAn ne","Ġre un","ĠF ace","ĠHall ow","ĠBud get","Ġcondem n","Ġt ender","Pro f","ocr atic","ĠTurn er","ĠAg ric","Ġ19 76","Ġa pt","d isc","ĠF ighter","ĠA ur","Ġgar bage","in put","ĠK arl","ĠOl iver","ĠL anguage","k n","N on","ĠCl ar","Ġtrad itions","Ġad vertisement","ĠS or","Ġarch ive","Ġvill ages","7 50","Ġimplement ing","w aukee","Ġdiet ary","Ġswitch ing","Rep ublic","Ġvel ocity","Ġc it","ĠA wards","Ġfin ancing","Ġlast ed",") ]","Ġrem inder","P erson","Ġprec ision","Ġdesign ers","ĠF ried","ĠB order","Ġtr agic","Ġw ield","Ġiniti atives","ĠT ank","w er","Ġjo ins","R o","in ery","Ġar row","Ġgener ating","found er","Ġsear ches","Ġrandom ly","A ccess","Ġb atch","Ġp osed","l at","Ġpursu ing","as a","Ġtest ified","form ing","ĠSh ar","w iki","ĠE ither","S ometimes","Ġsen ators","ĠJohn ny","ĠTal iban","ĠG PS","\":\" /","ãģ® å","Ġanaly zed","ĠRub io","ĠMove ment","op ard","ii i","St and","f ight","Ġign oring","i ang","ĠG N","so ever","ĠST AT","Ġref using","Ġswe at","Ġb ay","P ORT","ir med","ak y","Ġdis pro","Ġlabel ed","Ġ10 8","H ello","Ġple asant","ab a","Ġtri umph","Ġab oard","Ġinc om","ĠC row","le tt","Ġfol k","Ġch ase","` `","ĠBr us","Ġte ens","c ue","Ġter rain","h yd","il ight","OR Y","Su pport","ew s","ll i","rain ts","ĠC and","Ġab used","ach ment","l arg","B as","ĠC ancer","Ġ19 78","Ġsupp orter","ac cess","ĠTer min","ĠT ampa","ĠAN Y","Ġnew est","ĠCrim inal","ed u","Ġ19 30","Ġadm its","Ġend e","Ġfail ures","ur ate","ful ness","cy cl","ĠSub ject","Ġinf inite","th ree","W A","p it","ĠInst all","R ad","ili ation","G M","Ġcontin ent","Ġaccommod ate","ĠCl ay","Ġp up","ĠF unction","Ġham mer","ĠAlbert a","Ġrev ised","Ġminor ities","Ġmeasure ment","Con nell","Ġdis able","ĠM ix","In cre","Ġfor k","ĠR osen","Ġimpl ies","umb lr","AN G","Ġprote ins","Ġagg ression","Ġfacilit ate","S N","Ġilleg ally","u er","Ġacad em","Ġp uzz","ĠSh ift","p ay","oll o","Ġaud iences","B uild","Ġno ble","Ġsynt ax","â ĺħ","Ġbe am","ĠB ed","ĠA ld","Ġorig ins","v ideo","Ġ19 77","ĠAss ault","Ġgar age","Te am","Ġver dict","Ġd war","ĠVirt ual","e vent","Ke ep","Ġsent iment","Ġwild life","sh irt","Ġb urg","Ġrecommend ation","rep resent","Ġgall ery","own ers","Ġsch olar","Ġconven ience","ĠSw ift","Ġconv inc","C ap","Ġwar fare","ĠVis ual","Ġconst itute","Ġab ort","ĠWe ather","ĠLook ing","ĠH em","Ġmart ial","Ġinc oming","et ition","Ġtoler ance","ĠCre ated","Ġfl ows","ĠE lder","Ġsoul s","Ġf oul","ĠP ain","ĠC AN","Ġ2 20","b c","he nd","Ġgen ius","R eal","ĠW r","omet er","p ad","Ġlim iting","ĠS i","ĠL ore","ĠAd ventures","Ġvar ied","D isc","f in","ĠPerson al","Ch ris","Ġinv ented","Ġd ive","ĠR ise","Ġo z","ĠCom ics","Ġexp ose","ĠRe b","let ters","s ite","im ated","Ġh acking","Ġeduc ated","ĠNob ody","Ġdep ri","Ġincent ive","ãĤ ·","Ġovers ight","Ġtrib es","ĠBelg ium","Ġlicens ing","our t","Produ ct","ah l","ĠG em","Ġspecial ist","Ġc ra","ann ers","ĠCor byn","Ġ19 73","RE AD","Ġsum mar","Ġover look","ĠApp lication","Ġin appropriate","Ġdownload ed","Q ue","ĠB ears","Ġth umb","ĠChar acter","ĠReincarn ated","ĠS id","Ġdemonstr ates","s ky","ĠBloom berg","ĠAr ray","ĠRes ults","ĠFour th","ĠED T","ĠO scar","c end","Ġ10 6","ĠN ULL","ĠH ERE","m atch","ĠBr un","Ġgluc ose","ie g","eg u","Ġcert ified","Ġrel ie","Ġhuman itarian","Ġpr ayers","K ing","Ġn an","h ou","10 8","ul u","Ġrenew able","Ġdistingu ish","Ġd ense","ĠV ent","ĠPack age","ĠB oss","Ġedit ors","Ġm igr","T ra","ĠPet ers","ĠAr ctic","200 4","ĠC ape","Ġloc ally","Ġlast ing","Ġhand y",". ).","P an","ĠR ES","Ind ex","Ġt ensions","Ġformer ly","Ġide ological","Ġsens ors","Ġdeal ers","Ġdef ines","S k","Ġproceed s","Ġpro xy","az ines","ĠB ash","ĠP ad","ĠC raft","eal ous","Ġshe ets","omet ry","J une","cl ock","T T","ĠThe atre","ĠB uzz","Ġch apters","Ġmill enn","Ġd ough","ĠCongress ional","Ġimag ined","av ior","Ġclin ic","Ġ19 45","Ġhold er","ro ot","oles ter","Ġrest art","B N","ĠHam as","ĠJ ob","Ġor b","Ġr am","Ġdiscl ose","Ġtransl ate","Ġimm igrant","Ġannoy ing","Ġtreat y","an ium","ĠTe a","ĠLeg ion","Ġcrowd s","ĠB ec","ĠA er","oh yd","B ro","Look ing","Ġl bs","Ġagg ress","Ġse am","Ġinter cept","ĠM I","mer cial","act iv","ĠC it","Ġdim ension","Ġconsist ency","Ġr ushing","ĠDou glas","Ġtr im","Inst all","ick er","Ġsh y","10 6","Ġment ions","pe lled","ĠT ak","c ost","Ġclass room","Ġfort une","dri ven","Ġun le","ĠWhe el","Ġinvest or","ĠM asters","k it","Ġassoci ations","ĠEv olution","op ing","us cript","Ġprov incial","ĠWal ter","av i","S O","Ġun limited","Eng lish","ĠC ards","ĠEb ola","ne red","Ġreven ge","Ġout right","um per","Ġf itting","ĠSol id","Ġform ally","Ġproblem atic","Ġhaz ard","Ġenc ryption","Ġstraight forward","ĠA K","Ġp se","ĠOr b","ĠCh amber","ĠM ak","Cont ents","Ġloyal ty","Ġl yrics","ĠSy m","Ġwel comed","Ġcook ed","Ġmon op","Ġn urse","Ġmis leading","Ġe ternal","Ġshif ting","Ġ+ =","V is","Ġinst itutional","ill ary","Ġp ant","VER T","ĠA CC","ĠEn h","Ġinc on","ĠRE UTERS","Ġdon ated","â̦â̦ â̦â̦","In tern","Ġexhib it","Ġt ire","ĠR ic","ĠCh ampion","ĠMu hammad","N ING","ĠSoc cer","Ġmob ility","Ġvary ing","ĠM ovie","Ġl ord","o ak","F ield","Ġve ctor","us ions","Ġsc rap","Ġen abling","m ake","T or",". *","| |","ĠWe bsite","ĠN PC","Ġsocial ist","ĠBill y","ĠAdd itional","Ġc argo","Ġfar ms","ĠSo on","ĠPri ze","Ġmid night","Ġ9 00","se en","ĠSp ot","Ġshe ep","Ġspons ored","ĠH i","ĠJ ump","Ġ19 67","Micro soft","ĠAg ent","Ġch arts","d ir","Ġadj acent","Ġtr icks","Ġman ga","Ġex agger","/ >","foot ball","ĠF CC","G C","ĠT ier","and ra","OU ND","% ),","Ġfru its","V C","ĠA A","R ober","Ġmid st","â Ĺ","ank a","Ġlegisl ature","ĠNe il","Ġtour ists","\" \"","ĠWar ning","ĠNever theless","ĠOffic ial","ĠWh atever","Ġm old","Ġdraft ed","Ġsubst ances","Ġbre ed","Ġt ags","ĠT ask","Ġver b","Ġmanufact ured","com ments","ĠPol ish","Pro v","Ġdetermin es","Ob ama","k ers","Ġutter ly","Ġse ct","sc he","ĠG ates","ĠCh ap","Ġal uminum","Ġz ombie","ĠT ouch","ĠU P","Ġsatisf y","Ġpred omin","asc ript","Ġelabor ate","Ġ19 68","Ġmeas uring","ĠV ari","any ahu","Ġs ir","ul ates","id ges","ick ets","ĠSp encer","T M","oub ted","Ġpre y","Ġinstall ing","ĠC ab","re ed","re ated","Su pp","Ġwr ist","ĠK erry","10 7","ĠK le","ĠR achel","Ġc otton","ĠA RE","ĠE le","Cont rol","Ġload s","ĠD od","an as","b one","Ġclass ical","ĠReg ional","ĠInt eg","V M","Ġdes ires","Ġaut ism","support ed","ĠM essage","Ġcomp act","writ er","Ġ10 9","ĠHur ricane","c ision","Ġcy cles","Ġdr ill","Ġcolle ague","Ġm aker","G erman","Ġmist aken","S un","ĠG ay","Ġwhat soever","Ġsell s","ĠA irl","l iv","ĠO ption","Ġsol ved","Ġse ctors","Ġhorizont al","Ġequ ation","ĠSk ill","ĠB io","g ement","ĠSn ap","ĠLeg al","Ġtradem ark","Ġmake up","Ġassemb led","Ġsa ves","ĠHallow een","ĠVer mont","ĠFR OM","Ġfar ming","ĠP odcast","accept able","ĠHig her","Ġas leep","ull ivan","Ġrefere n","ĠLe v","Ġbul lets","ok o","H C","Ġst airs","Ġmain tains","ĠL ower","ĠV i","Ġmar ine","Ġac res","Ġcoordin ator","ĠJ oh","Ġcounterpart s","ĠBrother s","Ġind ict","b ra","Ġch unk","Ġc ents","H ome","ĠMon th","Ġaccording ly","if les","ĠGerm ans","ĠSy n","H ub","Ġey eb","âĶĢâĶĢ âĶĢâĶĢ","Ġr anges","ĠHoll and","ĠRob ot","f c","M ike","Ġpl asma","Ġsw ap","Ġath lete","ĠR ams",",' \"","Ġinfect ions","Ġcor rid","Ġv ib","Ġpat ches","Ġtradition ally","Ġrevel ation","Ġswe ep","Ġgl ance","Ġin ex","200 3","ĠR aw","work ing","os ures","ĠD at","ĠLyn ch","Ġle verage","ĠRe id","Ġcorrel ation","ian ces","av ascript","Ġrep ository","ret ty","Ġ19 72","24 0","Ġo un","p ol","ĠRe ed","Ġtact ical","is ite","App le","ĠQu inn","Ġrap ed","ill o","Euro pe","Ġalgorith ms","ĠRod rig","i u","Ġill um","Ġf ame","Ġintrodu cing","Ġdel ays","ĠRaid ers","Ġwh istle","Ġnovel s","ĠRe ally","Ġder iv","Ġpublic ations","ĠNe ither","ĠCom merce","Ġa ston","l anguage","Not es","ĠR oth","ĠF ear","Ġm ate","Ġpar ade","ĠQ B","Ġman eu","ĠC incinnati","m itting","Ġwa ist","ĠR ew","Ġdisc ont","Ð °","Ġst aring","Ġal ias","Ġsec urities","Ġtoile t","ĠJ edi","Ġun law","v ised","//// ////","] (","ĠWe iss","Ġpre st","ĠComp an","Ġmem o","ĠGr ace","J uly","ĠEl ite","cent er","ĠSt ay","Ġgal axy","Ġto oth","ĠS ettings","Ġsubject ed","ãĤ ¦","Ġline back","Ġretail ers","ĠW ant","Ġd angers","A ir","Ġvolunt ary","ew ay","Ġinterpret ed","ot ine","à §","Ġp el","Serv ice","ĠEvent ually","Ġcare ers","Ġthreat en","Ġmem or","ĠBrad ley","anc ies","s n","ĠUn known","N ational","Ġsh adows","ail and","ĠD ash","Every one","izz ard","M arch","= (","Ġpull s","Ġstr anger","Ġback wards","ĠBern ard","imens ional","Ġch ron","Ġtheoret ical","k top","Ġw are","ĠInvest ig","ĠIn iti","ĠOper ations","o ven","oc ide","* /","Ġfl ames","ĠC ash","sh it","Ġc ab","ĠAn aly","ĠSe ah","Ġdefin ing","Ġorder ing","Ġimm un","Ġpers istent","AC H","Russ ian","m ans","Ġh ind","Ġphot ography"," ©","Ġh ug","Ġ10 7","ĠH ence","i ots","ude au","Ġsubsid ies","Ġroutine ly","ĠDev ice","it ic","Ġdisg ust","land er","Ġ19 40","Ġassign ment","ĠB esides","w ick","ĠD ust","us c","struct ed","11 1","de velop","Ġf ond","Ġinter section","Ġdign ity","Ġcommission er","With out","re ach","Ġcart oon","Ġsc ales","ãĥ Ń","F IG","Ġsurve ys","ĠIndones ia","Ġart work","Ġun ch","Ġcy cling","un ct","au er","or ate","ĠOb viously","Ġcharacter ized","fe ld","Ġaff irm","Ġinn ings","Ġ é","Ġal iens","Ġcl oth","et ooth","ĠC ertain"," §","Ġdig est","k now","ĠX L","Ġpredict ions","Ġd in","W AR","Ġafter math","Ex ample","ĠSu ccess","ĠTh r","IG N","Ġmin er","B us","Ġcl arity","heim er","ĠO UT","ĠS end","ĠCirc le","ĠD iet","Ġpron ounced","Ġcreat ors","Ġearthqu ake","atter y","ge ons","Ġo d","Ġlay ing","or p","U lt","pro ject","Ġunder min","Ġsequ el","S am","ĠDark ness","Ġre ception","b ull","Y S","ĠV ir","Ġsequ ences","ĠCo in","Ġout fit","ĠW ait","1 19","Ġdel ivers",".... ..","Ġbl own","ĠE sc","ĠM ath","per m","ĠU l","Ġgl im","Ġfac ial","Ġgreen house","Ġto kens","/ -","ĠAnn ual","ĠON E","Ġteen age","ĠPhys ical","ĠL ang","ĠC elt","Ġsu ed","ivid ually","Ġpat ience","ch air","reg ular","Ġa ug","in v","ex cept","ĠL il","Ġn est","f d","s um","ĠCh ase","Russ ia","ĠJenn ifer","Ġoff season","Over all","F ore","Ġr iot","A ud","form er","Ġdefend ers","ĠC T","iot ic","rib ly","Ġautom ated","Ġpen is","Ġins ist","Ġdi agram","ĠS QL","ĠG arc","Ġw itch","cl ient","ier ra","am bers","Ġrec ount","f ar","V ery","oster one","Ġappreci ated","ĠPer fect","S ection","Ġd oses","oca ust","Ġcost ly","Ġg rams","ĠSh i","Ġwrest ling","Ġ19 71","Ġtro phy","Ġn erve","ĠK az","ĠExper ience","Ġpled ged","Ġplay back","Ġcreat ivity","by e","Ġattack ers","Ġhold ers","ĠCo ach","ĠPh D","Ġtransf ers","Ġcol ored","ĠH indu","Ġd rown","Ġlist ened","ĠW A","ias m","P O","Ġappeal ing","Ġdiscl osed","ĠCh icken","ag ging","Ġple aded","Ġnav igation","ĠReturn s","Ġ[ [","R OR","E A","Ġphotograp her","ĠR ider","ipp ers","Ġsl ice","Ġe rect","Ġhe d","iss ance","ĠVik ings","ur ious","Ġapp et","oubted ly","Ch ild","Ġauthent ic","o os","ĠM aking","Ġannoun cing","Ġb od","Ġmet er","ĠN ine","ĠR ogue","Ġwork force","Ġrenew ed","Ġorganis ations","ac s","P LE","Sh ort","Ġcomp ounds","ĠVis it","Ġen velop","ear th","Ġsupport ive","gg le","ĠBrus sels","ĠGu ild","Cre ate","RE L","Ġaver aged","Ġ19 69","ri ages","Ġlength y","Ġforg ot","O kay","ĠE rd","Ġdeal er","Ġrec ession","D D","Ġdesper ately","Ġhun ger","Ġst icks","Ġm ph","ĠF aith","Ġintention ally","Ġdem ol","ue ller","ĠS ale","Ġde bris","s pring","Ġle ap",">> >>","Ġcontain ers","se lling","rane an","atter ing","Ġcomment ed","ĠC M","on ut","Ġwood s","es pecially","Ġorgan ize","iv ic","ĠWood s","ang a","s qu","Ġm aj","am on","Ġax is","Ġ19 74","ĠDen mark","Ġwar rior","ĠP and","Ġout lined","ĠB O","ins ula","z illa","eb ook","Ġd are","Ġsear ched","Ġnav igate","S n","writ ing","Ġun ited","J apan","ĠHe brew","Ġfl ame","Ġrel ies","Ġcatch ing","ĠSh o","Ġimprison ment","Ġp ockets","Ġclos ure","ĠF am","t im","ade qu","Act ivity","Ġrecru iting","ĠW ATCH","ĠArgent ina","d est","Ġapolog ize","or o","Ġlack s","Ġtun ed","ĠGriff in","Ġinf amous","Ġcelebr ity","ss on","Ġ ----------------------------------------------------------------","ĠIs is","ĠDis play","Ġcred ibility","Ġeconom ies","Ġhead line","ĠCow boys","Ġind ef","Ġl ately","Ġincent ives","but ton","ĠM ob","A ut","Ġres igned","ĠO m","c amp","Ġprof iles","Ġsche mes","olph ins","ay ed","Cl inton","en h","ĠY ahoo","Ġab st","Ġan k","su its","Ġw ished","ĠMar co","udd en","Ġsp here","ĠB ishop","Ġincorpor ated","ĠPl ant","11 4","Ġh ated","p ic","Ġdon ate","Ġl ined","Ġbe ans","Ġsteal ing","Ġcost ume","Ġsher iff","Ġfor ty","Ġint act","Ġadapt ed","Ġtrave lling","b art","Ġnice ly","Ġdri ed","Ġsc al","os ity","NOT E","ĠB h","ĠBron cos","ĠI gn","Ġint imate","Ġchem istry","Ġopt imal","D eb","ĠGener ation","Ġ] ,","ich i","ĠW ii","ĠYOU R","vent ions","W rite","Ġpop ul","un ning","ĠW or","V ol","Ġqu een","head s","K K","Ġanaly ze","op ic","ear chers","Ġd ot","leg raph","ast ically","Ġupgr ades","Ġca res","Ġext ending","Ġfree ze","Ġin ability","Ġorg ans","Ġpret end","Ġout let","11 3","ol an","ĠM all","ul ing","t alk","Ġexpress ing","ĠAl ways","ĠBe gin","f iles","Ġlic enses","% %","ĠM itt","Ġfil ters","ĠMil waukee","G N","Ġunf old","M o","Ġnut rition","pp o","B o","Ġfound ing","Ġunder mine","Ġeas iest","ĠC zech","ĠM ack","Ġsexual ity","ĠN ixon","W in","ĠAr n","ĠK in","ãĤ £","ic er","Ġfort un","Ġsurf aces","agh d","Ġcar riers","ĠP ART","ĠT ib","Ġinter val","Ġfrust rating","ĠSh ip","ĠAr med","ff e","Ġbo ats","ĠAb raham","in is","Ġsu ited","th read","i ov","ab ul","ĠVenezuel a","Ġto m","su per","Ġcast le","alth ough","iox ide","ec hes","Ġevolution ary","Ġnegoti ate","Ġconfront ed","Rem ember","Ġ17 0","S uch","Ġ9 11","m ult","ĠA byss","ur ry","ke es","spe c","ĠBarb ara","Ġbelong ing","Ġvill ain","ist ani","Ġaccount able","Ġport ions","ĠDe cl","U r","ĠK ate","g re","Ġmag azines","UC K","Ġregul ate","om on","ĠAl most","Ġover view","Ġsc ram","Ġl oot","ĠF itz","Ġcharacter istic","ĠSn ake","s ay","ĠR ico","Ġtra it","ĠJo ined","au cus","Ġadapt ation","ĠAirl ines","Ġarch ae","ĠI de","Ġb ikes","Ġliter ary","Ġinflu ences","ĠUs ed","C reat","Ġple a","ĠDef ence","ĠAss ass","Ġp ond","UL T",") \"","Ġeval uated","Ġob taining","Ġdem ographic","Ġvig il","ale y","Ġsp ouse","ĠSeah awks","resp ons","ĠB elt","um atic","Ġr ises","run ner","ĠMichel le","Ġpot ent","r ace","ĠP AC","F ind","olester ol","IS S","ĠIntrodu ced","ress es","ign ment","O s","ĠT u","ĠDe x","ic ides","Ġspark ed","ĠLaur a","ĠBry ant","Ġsm iling","ĠNex us","Ġdefend ants","ĠCat al","Ġdis hes","sh aped","Ġpro long","m t","( $","ãĢ Ĥ","Ġcalcul ations","ĠS ame","Ġp iv","H H","Ġcance lled","Ġgr in","Ġterrit ories","ist ically","C ome","ĠP arent","Pro ject","Ġneg lig","ĠPriv acy","Ġam mo","LE CT","olute ly","ĠEp ic","Ġmis under","w al","Apr il","m os","path y","ĠC arson","Ġalbum s","ĠE asy","Ġpist ol","< <","Ġ\\ (","t arget","hel p","Ġinter pre","cons cious","ĠH ousing","ĠJ oint","12 7","Ġbe ers","s cience","ĠFire fox","effect ive","ĠC abin","ĠO kay","ĠApp lic","Ġspace craft","ĠS R","ve t","ĠStr ange","S B","Ġcor ps","iber al","e fficient","Ġpreval ence","Ġeconom ists","11 8","Th read","ord able","OD E","ĠC ant","=- =-","if iable","ĠA round","Ġpo le","Ġwilling ness","CL A","ĠK id","Ġcomple ment","Ġsc attered","Ġin mates","Ġble eding","e very","Ġque ue","ĠTr ain","Ġh ij","Ġme lee","ple ted","Ġdig it","Ġg em","offic ial","Ġlif ting","Ð µ","Re qu","it utes","Ġpack aging","ĠWork ers","h ran","ĠLeban on","ol esc","Ġpun ished","ĠJ uan","Ġj am","ĠD ocument","Ġm apping","ic ates","Ġinev itably","Ġvan illa","ĠT on","Ġwat ches","Ġle agues","Ġiniti ated","deg ree","port ion","Ġrec alls","Ġru in","Ġm elt","I AN","Ġhe m","Ex p","Ġb aking","ĠCol omb","at ible","Ġrad ius","pl ug","ĠI F","et ically","Ġf ict","H ER","ĠT ap","atin um","Ġin k","Ġco h","ĠW izard","b oth","te x","Ġsp ends","ĠCurrent ly","ĠP it","Ġneur ons","ig nt","Ġr all","Ġbus es","b uilding","Ġadjust ments","Ġc ried","ibl ical","att ed","ĠZ ion","ĠM atter","Ġmed itation","ĠD ennis","Ġour s","ĠT ab","Ġrank ings","ort al","Ġad vers","Ġsur render","ĠG ob","ci um","om as","im eter","Ġmulti player","Ġhero in","Ġoptim istic","Ġindic ator","ĠBr ig","Ġgro cery","Ġapplic ant","ĠRock et","v id","Ex ception","p ent","Ġorgan izing","Ġenc ounters","ĠT OD","Ġjew el","S ave","ĠChrist ie","Ġhe ating","Ġl azy","ĠC P","Ġcous in","Con fig","Ġreg ener","Ġne arest","Ġachie ving","EN S","th row","ĠRich mond","ant le","200 2","Ġan ten","b ird","13 3","Ġn arc","r aint","un ny","ĠHispan ic","ourn aments","Ġprop he","ĠTh ailand","ĠT i","Ġinject ion","Ġinher it","rav is","Ġmed i","Ġwho ever","ĠDE BUG","G P","ĠH ud","C ard","p rom","Ġp or","Ġover head","L aw","Ġviol ate","Ġhe ated","Ġdescript ions","Ġachieve ments","ĠBe er","ĠQu ant","W as","Ġe ighth","ĠI v","Ġspecial ized","U PDATE","ĠD elta","P op","J ul","ĠAs k","oph y","Ġnews letters","ĠT ool","Ġg ard","ĠConf eder","ĠGM T","ĠAb bott","Ġimm unity","ĠV M","Is lam","Ġimpl icit","w d","Ġ19 44","rav ity","omet ric","Ġsurv iving","ur ai","ĠPr ison","Ġr ust","ĠSk etch","Ġbe es","ĠThe ory","Ġmer it","T ex","ch at","Ġm im","Ġpast e","ĠK och","Ġignor ance","ĠSh oot","Ġbas ement","Un ited","ĠAd vis","he ight","Ġf oster","Ġdet ain","in formation","Ġne ural","' ;","Ġprov es","all ery","Ġinv itation","um bers","Ġc attle","Ġbicy cle","z i","Ġconsult ant","Ġap ology","ĠT iger","Ġ12 3","99 9","Ġind ividually","r t","ig ion","ĠBrazil ian","Ġdist urb","Ġentreprene urs","Ġfore sts","cer pt","pl ates","p her","clip se","Ġtw itter","Ġac ids","ograph ical","h um","ĠB ald","if ully","Ġcomp iler","ĠD A","Ġdon or","as i","Ġtrib al","l ash","ĠCon fig","Ġapplic ants","Ġsal aries","13 5","Put in","ĠF ocus","ir s","Ġmisc onduct","ĠH az","Ġeat en","M obile","Mus lim","ĠMar cus","v iol","Ġfavor able","Ġst ub","ad in","ĠH ob","Ġfaith ful","Ġelectron ics","Ġvac uum","w ait","back ed","econom ic","d ist","Ġten ure","Ġsince re","ĠT ogether","ĠW ave","Ġprog ression","Ġden ying","Ġdist ress","br aska","th ird","Ġmix ing","Ġcolon ial","Ġpriv ately","Ġun rest","atern ity","Ġprem ises","ant i","greg ation","Ġlic ence","ĠH ind","ĠSam uel","Ġconvinc ing","ĠA ce","ĠR ust","ĠNet anyahu","Ġhand les","ĠP atch","orient ed","ah o","ĠG onz","Ġhack ers","claim er","Ġcustom s","ĠGr an","f ighters","Ġl uc","Ġman uscript","aren thood","Ġdev il","Ġwar riors","Ġoff enders","Will iam","Ġhol idays","Ġnight mare","Ġle ver","iff erent","St at","Ġexhib ition","put ed","ĠP ure","Ġal pha","Ġenthus iasm","ĠRepresent atives","E AR","ĠT yp","Ġwhe at","ĠAl f","Ġcor rection","Ġev angel","AT T","M iss","Ġs oup","Ġimpl ied","par am","Ġsex y","ĠL ux","Ġrep ublic","p atch","ab lish","Ġic ons","Ġfather s","ĠG ET","ĠCar ib","Ġregul ated","ĠCo hen","ĠBob by","Ġn er","Ġb ent","vent ory","ĠAl ong","ĠE ST","ĠWall ace","Ġmurd ers","r ise","ke ll","ĠCommon wealth","Ġn asty","et a","ĠM IT","Ġadminist ered","Ġgenuine ly","Ed itor","n ick","Ġhyd ro","**************** ****************","ĠB le","Ġfin es","Ġg orge","aus ible","r h","Ġapp le","ment ioned","Ġro pe","ot yp","H R","Ġdisappoint ing","Ġc age","n ik","Ġdoub ts","ĠF REE","print s","ĠM UST","Ġvend ors","ĠIn qu","Ġliber als","Ġcontract or","Ġup side","child ren","Ġtrick y","Ġregul ators","charg ed","l iter","Ġ ***","Ġreb ell","l ang","Ġloc als","Ġphys icians","Ġhe y","ar se","t m","ĠLe x","Ġbehavior al","success ful","F X","Ġbr ick","ov ic","Ġcon form","Ġreview ing","Ġins ights","Ġbi ology","ĠRem ove","ĠExt ra","Ġcomm itting","indu ced","ignt y","ig m","Ġat omic","Comm on","ĠE M","ĠP ere","ĠIt ems","e h","Ġpres erved","ĠH ood","Ġprison er","Ġbankrupt cy","Ġg ren","us hes","Ġexplo itation","Ġsign atures","Ġfin an","] ,\"","ĠM R","Ġme g","rem lin","Ġmusic ians","Ġselect ing","Ġexam ining","IN K","l ated","H i","Ġart ic","Ġp ets","Ġimp air","ĠM AN","Ġtable ts","in clude","R ange","Ġca ut","Ġlog s","Ġmount ing","Ġun aware","Ġdynam ics","ĠPalest ine","ĠQu arter","ĠPur ple","Ġm a","ĠIm port","Ġcollect ions","ci ation","Ġsuccess or","Ġcl one","Ġaim ing","Ġposs essed","Ġstick ing","Ġsh aking","Ġloc ate","ĠH ockey","T urn","17 0","Ġfif teen","ĠHar rison","Ġcontinu ously","ĠT C","ĠVal ent","ĠRes cue","Ġby pass","am ount","Ġm ast","Ġprotect s","Ġart istic","Ġsomet ime","Ġsh oe","Ġshout ed","ific ant","et itive","ĠReg ister","ĠJ in","Ġconcent rated","ling ton","on ies","Ġgener ator","yr im","ĠAr men","Ġclear ing","id o","ĠT W","al ph","Ġlad ies","H ard","Ġdial og","Ġinput s","æ ľ","Ġpos es","Ġsl ots","ĠPrem ium","Ġle aks","Ġboss es","Ġ11 3","c ourse","A cc","ĠNew ton","ĠAust ria","ĠM age","Ġte aches","ab ad","Ġwe ars","Ġc yl","Ġcur se","ĠS ales","ĠW ings","Ġp sy","Ġg aps","ĠIce land","ĠP interest","Ġland lord","Ġdefin itions","ĠK er","Ġsufficient ly","ĠP ence","ĠArch itect","Ġsur pass","Ġ11 4","Ġsuper hero","ĠDise ase","Ġpri ests","ĠC ulture","Ġdefin itive","Ġsecret ly","ĠD ance","inst all","ch ief","ĠJess ica","W ould","Up dated","Ġlock er","ĠK ay","Ġmem orial","è ¦","f at","Ġdis gu","Ġflav ors","ĠBase ball","ĠRes istance","Ġk icks","Ġen v","Ġteen agers","D ark","ĠC AR","Ġh alt","ĠL G","ĠGab riel","Ġfe ver","Ġs atur","Ġm all","Ġaffili ate","ĠS leep","ĠSpe cific","ĠV el","Ġj ar","ĠSac red","ĠEd wards","ĠA CL","Ġret ained","ĠG iant","Ġlim itation","in ces","Ġref usal","ĠT ale","ĠBut ler","Ġacc idents","ĠC SS","Ġimport ed","ĠCop y","Î ±","ER T","z el","Ġdiv isions","h ots","ĠAl b","ĠD S","Load er","W ashington","at isf","ĠCreat ive","\\ .","ĠAut om","red ict","Ġrecept or","ĠCarl os","Met hod","ok a","Ġmal icious","Ġste pping",", [","ĠD ad","Ġatt raction","ĠEffect s","ĠPir ate","ĠC er","ĠIndust ry","ĠR ud","Ġchar ter","Ġd ining","Ġins ists","Ġconfig ure","Ġ( #","ĠSim ple","ĠSc roll","UT C","17 5","ĠK on","Ġmarket place","Ġ ãĤ","Ġref res","Ġg ates","er red","ĠP od","Ġbeh ave","Fr ank","n ode","Ġendors ed","he tt","as ive","ĠHom eland","Ġr ides","ĠLe ave","er ness","Ġflood ing","A FP","Ġris en","Ġcontin ually","Ġun anim","ĠCont ract","ĠP as","Ġgu ided","ĠCh ile","b d","Ġsu cc","pt ic","Ġcomm ittees","ĠL uther","ĠAny one","Ġs ab","12 4","Ġp ixel","ĠB ak","ĠT ag","ĠBenn ett","En ter","sm all","ĠPresident ial","Ġp ul","Ġcontr ace","arch ive","Ġcoast al","ĠK ids","19 2","âĢ ²","ick y","ING TON","Ġw olf","ĠSt alin","T ur","id get","am as","ĠUn less","Ġspons or","Ġmor ph","ĠCho ose","Ġrun ner","Ġun bel","Ġm ud","ĠMan a","Ġdub bed","Ġg odd","ure rs","wind ow","Ġrel ied","Ġcelebr ating","os c","Ġ13 5","Ġlobb ying","Ġincom plete","Ġrestrict ion","Ġinc ap","it us","Ġexpect ation","ĠAp ollo","Ġint ens","Ġsyn c","G H","Ġmanip ulation","B Y","Ġspe ar","Ġbre asts","Ġvol can","il ia","M aterial","Ġform ats","ĠB ast","Ġparliament ary","Ġsn ake","Ġserv ants","ĠTr udeau","ĠGr im","ĠArab ic","ĠSC P","ĠBoy s","st ation","Ġprospect ive","ord e","in itialized","Ġb ored","AB LE","Ġaccess ed","Ġtax i","ĠShe ll","aid en","urs ed","in ates","ĠIns urance","ĠPet e","Sept ember","6 50","Ġad ventures","ĠCo ver","Ġt ribute","Ġsk etch","Ġem power","Ġ Ø","ĠGl enn","ĠD aw","= \\\"","ĠPolit ics","Ġgu ides","Ġd ioxide","ĠG ore","ĠBr ight","ĠS ierra","Ġval ued","c ond","Ġpo inter","Se lect","Ġrisk y","Ġabsor b","im ages","Ġref uses","Ġbon uses","__ _","Ġh ilar","ĠF eatures","2 20","ĠCollect or","F oot","Ġ19 64","cul us","Ġd awn","Ġwork out","ĠL O","Ġphilosoph ical","ĠSand y","ĠYou th","Ġl iable","A f","bl ue","Ġovert urn","less ness","ĠTrib une","ĠIn g","Ġfact ories","Ġcat ches","Ġpr one","Ġmat rix","Ġlog in","Ġin acc","Ġex ert","s ys","Ġneed le","ĠQ ur","Ġnot ified","ould er","t x","Ġremind s","Ġpublisher s","Ġn ort","Ġg it","Ġfl ies","ĠEm ily","Ġflow ing","ĠAl ien","ĠStr ateg","Ġhard est","Ġmod ification","AP I","ĠM Y","Ġcr ashes","st airs","n umber","Ġur ging","ch annel","ĠFal con","Ġinhabit ants","Ġterr ifying","Ġutil ize","Ġban ner","Ġcig arettes","Ġsens es","ĠHol mes","Ġpract ition","ĠPhill ips","ott o","Ġcomp ile","Mod el","ĠK o","Ġ[ ]","Americ ans","ĠTer ms","Ġmed ications","ĠAn a","Ġfundament ally","ĠNot ice","Ġwe aker","Ġ 0000","Ġgar lic","Ġout break","Ġeconom ist","ĠB irth","Ġobst acles","ar cer","ĠOr thodox","Ġplace bo","ĠC rew","asp berry","ĠAng els","Ġdis charge","Ġdestruct ive","11 7","ĠR ising","Ġd airy","l ate","Ġcoll ision","ĠTig ers","ean or","ocument ed","ĠIn valid","Ġd ont","ĠL iter","ĠV a","Ġhyd rogen","Ġvari ants","ĠBrown s","Ġ19 65","Ġind igenous","Ġtrad es","Ġremain der","Ġswe pt","ĠImp act","Ġred ist","Ġun int","grad uate","ãĥ ķ","ĠW ILL","ãģ® ç","ĠCrit ical","Ġf isher","Ġv icious","Ġrevers ed","Y ear","ĠS ox","Ġshoot ings","Ġfil ming","Ġtouchdown s","ai res","m el","Ġgrand father","Ġaffect ion","ing le","Ġover ly","Add itional","Ġsup reme","ĠGr ad","Ġsport ing","Ġmer cy","ĠBrook s","ount y","Ġperform s","Ġtight ly","Ġdem ons","Ġkill ings","Ġfact ion","ĠNov a","aut s","Ġund oubtedly","ar in","Ġunder way","ra k","Ġl iv","ĠReg ion","Ġbrief ing","s ers","cl oud","ĠM ik","us p","Ġpred iction","az or","Ġport able","ĠG and","Ġpresent ing","Ġ10 80"," »","ush i","ĠSp ark","there um","Ġjust ification","ĠN y","Ġcontract ors","ming ham","ĠSt yle","å ħ","ĠChron icles","ĠPict ure","Ġprov ing","Ġw ives","set t","Ġmole cules","ĠFair y","Ġconsist ing","Ġp ier","al one","in ition","Ġn ucle","j son","Ġg otta","Ġmob il","Ġver bal","ar ium","Ġmon ument","uck ed","Ġ25 6","T ech","mine craft","ĠTr ack","Ġt ile","Ġcompat ibility","as is","Ġs add","Ġinstruct ed","ĠM ueller","Ġle thal","Ġhorm one","Ġor che","el se","Ġske let","Ġentert aining","Ġminim ize","ag ain","Ġunder go","Ġconst raints","Ġcig arette","ĠIslam ist","Ġtravel s","ĠPant hers","l ings","C are","Ġlaw suits","ur as","Ġcry st","Ġlow ered","Ġaer ial","Ġcomb inations","Ġha un","Ġch a","Ġv ine","Ġquant ities","Ġlink ing","b ank","Ġso y","B ill","ĠAngel a","Ġrecip ient","ĠProt est","Ġs ocket","Ġsolid arity","Ġâ Ĩ","m ill","Ġvar ies","ĠPak istani","Dr agon","Ġun e","Ġhor izon","³³³³ ³³³³","Ġprov inces","Ġfrank ly","Ġenact ed","not es","[ '","Ġ19 2","ocr acy","Ġendorse ment","Ġover time","Tr ue","L ab","lic ted","ĠD NC","Ġbe ats","ĠJam ie","15 2","ĠIN T","Cont act","Ġaccount ed","h ash","ĠPack ers","p ires","Ġles bian","Ġamend ments","Ġhop eful","ĠFin land","Ġspot light","Ġconfig ured","Ġtrou bled","Ġg aze","ĠCal gary","Ġrel iability","Ġins urg","sw er","b uy","ĠSk in","Ġp ixels","Ġhand gun","Ġpar as","Ġcateg or","ĠE L","ĠRe x","Ind eed","Ġkind a","Ġconj unction","ĠBry an","ĠMan ufact","y ang","Pl us","S QL","ish ment","Ġdom inate","Ġn ail","Ġo ath","Ġeru pt","ĠF ine","it bart","ĠCh ip","ĠAb d","ĠN am","Ġbuy er","Ġdiss ent","Le aks","Cont in","Ġr ider","ĠSome one","Ġill usion","c in","ĠBoe ing","Ġin adequ","ov ation","i ants","Ġreb uild","4 50","ĠDest iny","S W","ĠT ill","H it","ia z","ĠBang l","acher s","ĠRe form","Ġse gments","Ġsystem atic","d c","ĠConserv atives","Ġport al","h or","ĠDragon bound","Ġdrag ged","om o","Ġthe e","ad vert","ĠRep orts","ĠE t","Ġbarrel s","Aug ust","Ġcompar isons","Ġhe x","Ġan throp","\" [","bor ough","ab i","Ġpict ured","play ing","ĠAdd ress","ĠMir ror","Sm ith","Ġt ires","ĠN PR","AA AA","Ġclass ification","ĠTh an","ĠH arm","ĠR A","Ġreject ion","min ation","Ġr anged","ĠF alls","D I","H ost","ãĤ ´","ĠEx ample","list ed","th irds","Ġsaf egu","br and","Ġprob able","Can ada","IT ION","ĠQ aeda","Ġch ick","Ġimport s","h it","l oc","W W","Ġble w","Ġany time","Ġwh oles","ik ed","Ġcal culation","cre ate","ĠO ri","Ġupgr aded","Ġapp ar","ut ory","ĠM ol","B rit","ĠJ ong","IN AL","ĠStart ing","Ġd ice","urt le","Ġre lying","cl osure","Ġprof itable","Ġsl aughter","ĠMan ual","c aster","Ġ\" $","Ġfe ather","ĠSim ply","ie ves","Ġdeter ior","ĠPC I","Ġst amp","Ġfl aws","Ġsh ade","ham mer","Ġpass port","Ġcont ing","am el","Ġobser vers","Ġneg lect","ĠR B","ĠBrother hood","Ġskept ical","f amily","us k","Ġemotion ally","â Ļ","ĠBet a","ason able","id ity","ĠM ul","Ġkick ing","ĠC arm","oll ah","VERT IS","ĠAt hen","Ġlad der","ĠBul let","å £","00 01","ĠWild life","ĠM ask","ĠN an","R ev","Ġun acceptable","leg al","Ġcrowd ed","ag i","ĠC ox","j e","Ġmor ality","Ġfu els","Ġc ables","Ġman kind","ĠCarib bean","Ġanch or","Ġby te","ĠO ften","ĠO z","Ġcraft ed","Ġhistor ian","ĠW u","Ġtow ers","ĠCitiz ens","Ġhel m","Ġcred entials","Ġsing ular","ĠJes se","Ġtack les","Ġcont empt","Ġa fore","ĠSh adows","Ġn il","Ġur gent","app le","bl ood","Ġv on","Ġoff line","Ġbreat he","Ġj umps","Ġirre levant","ox ic","om al","import ant","J im","Ġgl oves","arm ing","dep th","Ġtal ents","ook ie","ĠS B","Ġpal m","uff s","est a","IG H","Ġcan on","ĠVer izon","ĠP le","Ġcou pled","vel t","Ġfundra ising","ĠGet ting","ĠD LC","Ġmathemat ical","ĠH S","ĠCard inals","te lling","Ġspons ors","Ġ Ï","ĠBull s","op tion","Ġprop ose","Ġmem orable","Ġembr aced","Ġdecl ining","He alth","ed a","Ġ} ;","Ġsp am","m ile","Ġpit cher","ĠE ight","Ġcar ing","ut ic","ro le","Ġair line","ernand ez","ĠAth let","Ġcert ification","ux e","rig er","Ġem pir","Ġsens ation","Ġdis m","Ġb olt","Ġev olve","H ouse","Ġconsult ation","ĠD uty","Ġtou ches","ĠN athan","Ġf aint","h ad","\" (","ĠCons umer","ĠExt reme","Ġ12 7","ĠHer m","ĠSac rament","iz oph","Ġanx ious","ul ously","Ġsoc ially","ĠU TC","Ġsol ving","ĠLet ter","Hist ory","ed uc","Pr ice",") );","Ġrel oad","am ic","Ġp ork","Ġdisc ourse","Ġt ournaments","ai ro","ĠK ur","ĠCost a","Ġviol ating","Ġinterf ere","Ġrecre ational","uff le","Ġspe eches","Ġneed ing","Ġremem bers","Ġcred ited","n ia","f ocused","amer a","Ġb ru","um bs","ĠCub an","Ġpreced ing","Ġnons ense","ac ial","Ġsmart phones","ĠSt ories","S ports","ĠEmer gency","oun cing","ef ined","Ġb er","Ġconsult ing","Ġm asters","he astern",".\" [","ĠRun ning","Ġsus cept","ĠF eng","Americ a","pr ises","st itial","ĠWeek ly","ĠGreat er","mod ules","if ter","G raphics","ul er","Ġwho lly","Ġsupp ress","Ġconce aled","Ġhapp ily","Ġaccept s","ĠEn joy","Ġr ivers","ĠEx cept","2 25","ĠN HS","ĠMc Connell","Ġp ussy","fer red","ut able","Ġatt ain","Ġ> =","Ġdepos its","roph ic","Ġnot orious","ĠSh aw","il itation","Ġepid emic","all ic","Ġsmall est","ov ich","Ġaccess ories","per ties","Ġsur plus","ĠMe ch","Ġamb ig","ĠImm igration","Ġch im","ev al","Ġpract icing","ĠMyster y","Ġdom ains","ĠSil icon","app s","Ġkilomet ers","e a","ĠSm ash","Ġwarrant y","Ġn ost","s il","re v","J on","ĠDub lin","Ġtast es","Ġb out","g reat","er ror","Ġsw itches","ĠB apt","D O","ok i","Ġsour ced","pro du","Ġattach ment","ĠIss ue","ĠQuest ion","Jo in","Ġf itted","Ġunlaw ful","^ ^","ere k","Ġauthent ication","Ġst ole","Ġaccount ability","l abel","S earch","Ġal beit","atic an","fund ed","ĠAdd ing","ĠI Q","Ġsub mar","l it","a que","ĠLear ning","Ġint eger","M aster","ĠCh rom","Ġprem ier","O p","ĠLi u","Ġbl essed","ĠGl obe","ĠResp onse","Ġlegit im","ĠMer kel","Ġdispos al"," ´","Ġgau ge","pe at","Ġindu ced","Ġquestion able","arth y","ĠV it","ĠF eed","U ntil","U t","worth y","R Y","ĠH erald","ĠHam mer","Ġmed al","ĠR ivers","ĠH ack","Ġclar ify","Ġtrack ed","Ġautonom ous","Ġten ant","ĠQ atar","er ie","Ġgr im","ĠMon itor","Ġresist ant","ĠSpe c","ĠWell s","N AS","14 8","Ġmin ers","iot ics","Ġmiss es","11 6","g ian","g it","ĠE yes","p res","Ġgrad uated","Ġang el","Ġsyn chron","Ġefficient ly","Ġtrans mitted","H arry","Ġglob ally","EN CE","ĠMont ana","r aged","ĠPre vention","Ġp iss","ĠL l","Ġshe lf","ĠB JP","ĠTest ament","ĠL ate","ik er","ĠH app","ĠJul ian","h all","Ġsp ont","Ġshut down","Ġincons istent","Ġsubscrib ers","Ġske leton","ĠNe braska","Ġins pire","ĠV oid","F eed","Ġang les","ĠSpr ings","Ġbench mark","Ġvacc ines","izoph ren","se xual","uff ed","Ġsh ine","ĠK ath","Ġgest ure","ine a","Ġr ip","Ġopp ression","Ġcons cience","b t","ĠL um","Ġinc idence","ĠF a","w r","Ġmin eral","ĠSp urs","alk y","Ġth under","Ġop io","Be ing","ĠPal m","Ġwas ted","Ġl b","i aries","ĠIniti ative","Ġcur ric","Ġmark er","ĠMc L","Ġext ensions","ĠP v","ĠAr ms","Ġoffer ings","Ġdef enses","Ġvend or","Ġcontrad ict","ĠCol in","Ġredd it","Ġper ipher","12 2","Ġs ins","E dit","IC T","So ft","ĠSh ah","Ġadministr ator","ĠT rip","Ġporn ography","Ġtu ition","in ence","ĠPro gress","Ġcat alog","Ġsu ite","Ġh ike","Ġreprodu ctive","eng ine","Ġd rought","ĠNo ah","Ġ2 30","Ġd ude","Ġrelax ed","Ġpart ition","Ġparticip ant","Ġtel esc","Ġfe as","ĠF F","own er","Ġswe eping","Ġl enses","Ġmatch up","ĠRe pl","ourn als","Ġcred ible","Ġgrand mother","Ġther mal","Ġsubscrib ing","Ġident ities","col m","U CT","Ġreluct ant","us ers","ĠC ort","Ġassist ed","OS S","ATION S","IS H","Ġpharm aceutical","ic able","ad ian","ĠSon ic","ĠF ury","ĠM ong","A H","ĠPsych ology","Ġph osph","Ġtreat s","Ń Ķ","Ġstead ily","ĠHell o","Ġrel ates","Ġcl ue","Ex pl","a uth","Ġrev ision","Ġe ld","os ion","Ġbr on","14 4","ri kes","Ġmin es","Ġblank et","ĠF ail","el ed","ĠIm agine","ĠPl anned","a ic","Re quest","M ad","ĠHor se","ĠEag le","Ġcap ac","15 7","Ġl ing","ĠN ice","ĠP arenthood","min ster","og s","ens itive","Not hing","Ġcar n","F in","ĠP E","Ġr ifles","ĠL P","S and","Ġgui Active","Ġtour ist","C NN","Ġunve iled","Ġpredec essor","} {","u ber","Ġoff shore","Ġopt ical","ĠR ot","ĠPear l","et on","Ġst ared","Ġfart her","at ility","cont in","ĠG y","ĠF oster","ĠC oc","ri ents","Ġdesign ing","ĠEconom y","ON G","W omen","ĠN ancy","er ver","Ġmas cul","Ġcasual ties","Ġ2 25","ĠS ullivan","ĠCh oice","Ġa ster","w s","Ġhot els","Ġconsider ations","Ġcou ch","ĠSt rip","ĠG n","Ġmanip ulate","l ied","Ġsynt hetic","Ġassault ed","Ġoff enses","ĠDra ke","Ġim pe","Oct ober","ĠHer itage","h l","ĠBl air","Un like","Ġg rief","Ġ4 50","Ġopt ed","Ġresign ation","il o","Ġver se","ĠT omb","Ġu pt","Ġa ired","ĠH ook","ĠML B","Ġassum es","out ed","ĠV ers","Ġinfer ior","Ġbund le","ĠD NS","ograp her","Ġmult ip","ĠSoul s","Ġillust rated","Ġtact ic","Ġdress ing","Ġdu o","Con f","Ġrel ent","Ġc ant","Ġscar ce","Ġcand y","ĠC F","Ġaffili ated","Ġspr int","yl an","ĠGarc ia","Ġj unk","Pr int","ex ec","C rit","Ġport rait","ir ies","ĠOF F","Ġdisp utes","W R","L ove","ãģ Ħ","ĠRe yn","Ġh ipp","op ath","Ġflo ors","ĠFe el","Ġwor ries","Ġsett lements","ĠP os","Ġmos que","Ġfin als","Ġcr ushed","ĠPro bably","ĠB ot","ĠM ans","ĠPer iod","Ġsovere ignty","Ġsell er","Ġap ost","Ġam ateur","Ġd orm","Ġconsum ing","Ġarm our","ĠRo ose","Ġint ensive","Ġelim inating","ĠSun ni","ĠAle ppo","j in","Ġadv ise","p al","ĠH alo","Ġdes cent","Ġsimpl er","Ġbo oth","ST R","L ater","ĠC ave","== =","Ġm ol","Ġf ist","Ġshot gun","su pp","Ġrob bery","E ffect","Ġobsc ure","ĠProf essional","Ġemb assy","Ġmilit ant","Ġinc arcer","Ġgener ates","Ġlaun ches","Ġadministr ators","Ġsh aft","Ġcirc ular","Ġfresh man","ĠW es","ĠJo el","ĠD rew","ĠDun can","ĠApp arently","s ight","ĠIntern al","ĠInd ividual","ĠF E","Ġb ore","ĠM t","Ġbroad ly","ĠO ptions","ount ain","ip es","ĠV ideos","20 4","Ġh ills","Ġsim ulation","Ġdisappoint ment","it an","ĠLabor atory","Ġup ward","Ġbound ary","Ġdark er","h art","Ġdomin ance","C ong","ĠOr acle","ĠL ords","Ġscholars hip","ĠVin cent","ed e","ĠR ah","Ġencour ages","ro v","Ġqu o","Ġprem ise","ĠCris is","ĠHol ocaust","Ġrhyth m","Ġmet ric","cl ub","Ġtransport ed","Ġn od","ĠP ist","Ġancest ors","ĠFred er","th umbnails","ĠC E","ON D","Ph il","ven ge","ĠProduct s","cast le","Ġqual ifying","ĠK aren","VERTIS EMENT","Ġmight y","Ġexplan ations","Ġfix ing","D i","Ġdecl aring","Ġanonym ity","Ġju ven","ĠN ord","ĠDo om","ĠAct ually","O k","ph is","ĠDes ert","Ġ11 6","I K","ĠF M","Ġinc omes","V EL","ok ers","Ġpe cul","Ġlight weight","g ue","Ġacc ent","Ġincre ment","ĠCh an","Ġcompl aining","ĠB aghd","Ġmidfield er","Ġover haul","Pro cess","ĠH ollow","ĠTit ans","Sm all","man uel","ĠUn ity","ĠEv ents","S ty","Ġdispro portion","n esty","en es","ĠC od","Ġdemonstr ations","ĠCrim son","ĠO H","Ġen rolled","Ġc el","ĠBre tt","Ġa ide","Ġhe els","Ġbroad band","Ġmark ing","Ġw izard","ĠN J","ĠChief s","Ġingred ient","Ġd ug","ĠSh ut","urch ase","end or","Ġfar mer","ĠGold man","12 9","15 5","Or der","Ġl ion","i ably","Ġst ain","ar ray","ilit ary","ĠFA Q","Ġexpl oded","ĠMcC arthy","ĠT weet","ĠG reens","ek ing","l n","ens en","Ġmotor cycle","Ġpartic le","Ġch olesterol","B ron","Ġst air","Ġox id","Ġdes irable","ib les","Ġthe or","for cing","Ġpromot ional","ov o","b oot","ĠBon us","raw ling","Ġshort age","ĠP sy","Ġrecru ited","Ġinf ants","Ġtest osterone","Ġded uct","Ġdistinct ive","Ġfirm ware","bu ilt","14 5","Ġexpl ored","Ġfact ions","Ġv ide","Ġtatt oo","Ġfinan cially","Ġfat igue","Ġproceed ing","const itutional","Ġmis er","Ġch airs","gg ing","ipp le","Ġd ent","Ġdis reg","ç Ķ","st ant","ll o","b ps","aken ing","Ġab normal","ĠE RA","å£ «","ĠH BO","ĠM AR","Ġcon cess","Ġserv ant","Ġas pir","l av","ĠPan el","am o","Ġprec ip","Ġrecord ings","Ġproceed ed","Ġcol ony","ĠT ang","ab lo","Ġstri pped","Le ft","to o","Ġpot atoes","Ġfin est","% ).","Ġc rap","ĠZ ach","ab ases","ĠG oth","Ġbillion aire","w olf","Ġsan ction","S K","Ġlog ged","P o","ey ed","un al","Ġcr icket","Ġarm ies","Ġunc overed","Cl oud","ó n","Ġreb ounds","Ġm es","O per","P ac","Ġnation ally","Ġinsert ed","p ict","Ġgovern ance","Ð ¸","Ġprivile ges","G ET","Ġfavor ites","im ity","Ġlo ver","the m","em pl","Ġgorge ous","An n","Ġsl ipped","Ġve to","B ob","Ġsl im","u cc","ĠF ame","udden ly","Ġden ies","ĠM aur","Ġdist ances","Ġw anna","t ar","ĠS ER","Ġâ Ī","Ġle mon","at hetic","Ġlit eral","Ġdistingu ished","Ġansw ering","G I","Ġrelig ions","ĠPhil os","ĠL ay","Ġcomp os","ire ments","ĠK os","ine z","roll ing","Ġyoung est","and ise","ĠB orn","Ġalt ar","am ina","ĠB oot","v oc","Ġdig ging","Ġpress ures","Ġl en","26 4","Ġassass ination","ĠBir mingham","ĠMy th","Ġsovere ign","ĠArt ist","ĠPhot ograph","Ġdep icted","Ġdisp ens","orth y","Ġamb ul","int eg","ĠC ele","ĠTib et","Ġhier archy","Ġc u","Ġpre season","ĠPet erson","Ġcol ours","Ġworry ing","Ġback ers","ĠPal mer","ĠÎ ¼","Ġcontribut or","Ġhear ings","Ġur ine","Ġ Ù","ourge ois","Sim ilar","ĠZ immer","s omething","ĠUS C","Ġstrength s","ĠF I","Ġlog ging","As ked","ĠTh ai","in qu","ĠW alt","Ġcrew s","it ism","3 01","Ġshar ply","um ed","Ġred irect","r ators","In f","ĠWe apons","Ġte asp","19 99","L ive","ĠEs pecially","ĠS ter","ĠVeter ans","Ġint ro","other apy","Ġmal ware","Ġbre eding","Ġmole cular","ĠR oute","ĠCom ment","oc hem","Ġa in","Se ason","Ġlineback er","Ä «","ĠEconom ics","es ar","ĠL ives","ĠEm ma","Ġk in","ĠTer rit","Ġpl anted","ot on","ĠBut ter","ĠSp ons","P ER","Ġdun geon","Ġsymb olic","Ġfil med","Ġdi ets","Ġconclud es","Ġcertain ty","ĠForm at","Ġstr angers","form at","ĠPh ase","Ġcop ied","Ġmet res","ld a","ĠUs ers","Ġdeliber ate","Ġwas hed","ĠL ance","im ation","Ġimpro per","ĠGen esis","ick r","ĠK ush","Ġreal ise","Ġembarrass ing","alk ing","b ucks","Ġver ified","Ġout line","year s","ĠIn come","20 2","Ġz ombies","F inal","ĠMill enn","Ġmod ifications","ĠV ision","ĠM oses","ver b","iter ranean","ĠJ et","Ġnav al","ĠA gg","Ġur l","Ġvict ories","Ġnon etheless","Ġinj ust","ĠF act","ç ļ","Ġins ufficient","re view","face book","Ġnegoti ating","Ġguarant ees","im en","uten berg","Ġg ambling","Ġcon gr","Load ing","Ġnever theless","Ġpres idents","ĠIndust rial","Ġ11 8","Ġp oured","ĠT ory","Ġ17 5","Ġ: =","Sc ott","ange red","T ok","Ġorgan izers","M at","ĠG rowth","Ġad ul","Ġens ures","Ġ11 7","é¾į å","Ġmass acre","Ġgr ades","be fore","AD VERTISEMENT","ĠSl ow","ĠM MA","âĢĶ \"","ĠV atican","Q aeda","Ġo we","66 66","ĠS orry","ĠGr ass","Ġbackground s","Ġexha usted","Ġcl an","Ġcomprom ised","ĠE lf","ĠIsa ac","ens on","In vest","IF A","Ġinterrupt ed","ãĥī ãĥ©","Ġtw isted","ĠDrag ons","M ode","ĠK remlin","Ġfert il","he res","ph an","ĠN ode","f ed","ĠOr c","Ġunw illing","C ent","Ġprior it","Ġgrad uates","Ġsubject ive","Ġiss uing","ĠL t","Ġview er","Ġw oke","Th us","bro ok","Ġdep ressed","Ġbr acket","ĠG or","ĠFight ing","Ġstri ker","Rep ort","ĠPortug al","Ġne o","w ed","19 9","Ġflee ing","sh adow","ident ified","US E","Ste am","Ġstret ched","Ġrevel ations","art ed","ĠD w","Ġalign ment","est on","ĠJ ared","S ep","Ġblog s","up date","g om","r isk","Ġcl ash","ĠH our","Ġrun time","Ġunw anted","Ġsc am","Ġr ack","Ġen light","on est","ĠF err","Ġconv ictions","Ġp iano","Ġcirc ulation","ĠW elcome","Ġback lash","ĠW ade","Ġrece ivers","ot ive","J eff","Ġnetwork ing","ĠPre p","ĠExpl orer","Ġlect ure","Ġupload ed","ĠMe at","B LE","ĠNaz is","ĠSy nd","st ud","ro ots","ri ans","Ġportray ed","Ġ ??","ĠBudd ha","s un","Rober t","ĠCom plex","Ġover see","Ġste alth","T itle","ĠJ obs","ĠK um","Ġappreci ation","ĠM OD","Ġbas ics","Ġcl ips","Ġnurs ing","Ġpropos ition","Ġreal ised","ĠNY C","Ġall ocated","ri um","ar an","ĠPro duction","ĠV ote","Ġsm ugg","Ġhun ter","az er","ĠCh anges","Ġfl uct","y on","Ar ray","Ġk its","W ater","Ġuncom mon","Ġrest ing","ell s","w ould","Ġpurs ued","Ġassert ion","omet own","ĠMos ul","ĠPl atform","io let","Ġshare holders","Ġtra ils","P ay","ĠEn forcement","ty pes","ĠAn onymous","Ġsatisf ying","il ogy","Ġ( '","w ave","c ity","Ste ve","Ġconfront ation","ĠE ld","C apt","ah an","ht m","ĠC trl","ON S","2 30","if a","hold ing","Ġdelic ate","Ġj aw","ĠGo ing","or um","S al","Ġd ull","ĠB eth","Ġpr isons","Ġe go","ĠEl sa","avor ite","ĠG ang","ĠN uclear","Ġsp ider","ats u","Ġsam pling","Ġabsor bed","ĠPh arm","iet h","Ġbuck et","ĠRec omm","O F","ĠF actory","AN CE","Ġb acter","H as","ĠObs erv","12 1","Ġprem iere","De velop","Ġcur rencies","C ast","Ġaccompany ing","ĠNash ville","Ġfat ty","ĠBre nd","Ġloc ks","Ġcent ered","ĠU T","augh s","or ie","ĠAff ordable","v ance","D L","em et","Ġthr one","ĠBlu etooth","Ġn aming","if ts","AD E","Ġcorrect ed","Ġprompt ly","ĠST R","Ġgen ome","Ġcop e","Ġval ley","Ġround ed","ĠK end","al ion","p ers","Ġtour ism","Ġst ark","v l","Ġblow ing","ĠSche dule","st d","Ġunh appy","Ġlit igation","ced es","Ġand roid","Ġinteg ral","ere rs","ud ed","t ax","Ġre iter","ĠMot ors","oci ated","Ġwond ers","ĠAp ost","uck ing","ĠRoose velt","f ram","Ġyield s","Ġconstit utes","aw k","Int erest","Ġinter im","Ġbreak through","ĠC her","Ġpro sec","ĠD j","ĠM T","Res p","ĠP T","Ġs perm","ed it","B T","Lin ux","count ry","le ague","Ġd ick","Ġo ct","Ġinsert ing","Ġsc ra","ĠBrew ing","Ġ19 66","Ġrun ners","Ġpl un","id y","ĠD ian","Ġdys function","Ġex clusion","Ġdis gr","Ġincorpor ate","Ġrecon c","Ġnom inated","ĠAr cher","d raw","achel or","Ġwrit ings","Ġshall ow","Ġh ast","ĠB MW","ĠR S","Ġth igh","Ġ19 63","Ġl amb","Ġfav ored","ag le","Ġcool er","ĠH ours","ĠG U","ĠOrig in","Ġglim pse","---------------- ----","L im","Ġche ek","Ġj ealous","- '","Ġhar ness","ĠPo ison","Ġdis abilities","ne apolis","Ġout look","Ġnot ify","ĠIndian apolis","Ġab rupt","ns ic","Ġenc rypted","Ġfor fe","reat h","Ġr abb","Ġfound ations","Ġcompl iment","ĠInter view","ĠS we","Ġad olesc","Ġmon itors","ĠSacrament o","Ġtime ly","Ġcontem pl","Ġposition ed","Ġpost ers","ph ies","iov ascular","v oid","ĠFif th","Ġinvestig ative","OU N","Ġinteg rate","ĠIN C","ish a","ibl ings","ĠRe quest","ĠRodrig uez","Ġsl ides","ĠD X","Ġfemin ism","Ġdat as","Ġb end","ir us","ĠNig eria","F ox","Ch ange","Ġair plane","ĠLad en","Ġpublic ity","ixt y","Ġcommit ments","Ġaggreg ate","Ġdisplay ing","ĠAr row","Ġ12 2","Ġrespect s","and roid","s ix","ĠSh a","Ġrest oration",") \\","W S","oy s","Ġillust rate","with out","12 6","ĠâĶ Ĥ","Ġpick up","n els","Ġ ....","f ood","ĠF en",") ?","Ġphenomen a","Ġcompan ions","ĠW rite","Ġsp ill","Ġbr idges","ĠUp dated","ĠF o","Ġinsect s","ASH INGTON","Ġsc are","il tr","ĠZh ang","Ġsever ity","Ġind ul","14 9","ĠCo ffee","Ġnorm s","Ġp ulse","ĠF T","Ġhorr ific","ĠDest roy","ĠJ SON","Ġo live","Ġdiscuss es","R est","E lect","ĠW inn","ĠSurv iv","ĠH ait","S ure","op ed","Ġro oted","ĠS ke","ĠBron ze","Ġl ol","Def ault","Ġcommod ity","red ited","Ġliber tarian","Ġforb idden","Ġgr an","à ¨","Ġl ag","en z","dri ve","Ġmathemat ics","Ġw ires","Ġcrit ically","Ġcarb ohyd","ĠChance llor","ĠEd die","Ġban ning","ĠF ri","Ġcompl ications","et ric","ĠBangl adesh","Ġband width","St op","ĠOrig inally","Ġhalf way","yn asty","sh ine","Ġt ales","rit ies","av ier","Ġspin ning","ĠWH O","Ġneighbour hood","b ach","Ġcommer ce","ĠS le","B U","Ġentreprene ur","Ġpecul iar","ĠCom ments","f re","3 20","IC S","Ġimag ery","ĠCan on","ĠElect ronic","sh ort","( (","D ig","Ġcomm em","u ced","Ġincl ined","ĠSum mon","Ġcl iff","ĠMed iterranean","Ġpo etry","Ġprosper ity","ĠRe ce","Ġp ills","m ember","Ġfin ale","un c","ĠG ig","ä ½","Ġl od","Ġback ward","- +","ĠFor ward","Ġth ri","s ure","Ġso ap","ĠF X","R ES","ĠSe xual","oul os","Ġfool ish","Ġright eous","Ġco ff","terror ism","ust ain","ot er","Ġab uses","ne xt","Ġab usive","Ġthere after","Ġprohib ition","ĠS UP","Ġd ip","Ġr ipped","Ġinher ited","Ġb ats","st ru","G T","Ġflaw ed","ph abet","Ġf og","do ors","Ġim aging","Ġdig its","ĠHung ary","Ġar rog","Ġteach ings","Ġprotocol s","ĠB anks","à ¸","p ound","ĠC urt",".\" )",". /","Ġex emption","end ix","ĠM ull","Ġimpro ves","ĠG amer","d imensional","I con","ĠMarg aret","St atus","d ates","Ġint ends","Ġdep ict","Ġpark ed","J oe","ĠMar ines","chn ology","! ).","Ġjud ged","Ġwe ights","R ay","Ġapart ments","he ster","Ġrein force","Ġoff ender","occ up","Ġs ore","e pt","ĠPH P","ĠB row","Ġauthor ization","ĠR isk","ĠDel aware","ĠQ U","Ġnot ifications","Ġsun light","Ġex clude","d at","Ġm esh","ĠSud an","Ġbelong ed","Ġsub way","Ġno on","ĠInter ior","ol ics","ĠL akers","Ġc oding","Dis claimer","Cal if","O ld","Ġdis l","???? ?","Ġconfir ms","Ġrecruit ment","Ġhom icide","Cons ider","ĠJeff rey","ft y","} ;","Ġobject ion","do ing","ĠLe o","W ant","Ġgl ow","ĠClar ke","ĠNorm an","Ġver ification","Ġpack et","ĠForm ula","Ġpl ag","es ville","Ġshout ing","Ġo v","ĠR EC","ĠB ub","Ġn inth","Ġener g","Ġvalid ity","Ġup s","j ack","Ġneighbor ing","ĠN ec","ew orks","ĠH ab","are z","Ġsp ine","Ġevent ual","ĠLe aders","ĠC arn","Ġprob ation","Ġrom ance","ms g","ĠMechan ical","ER Y","R ock","Ġpart isan","N ode","ass ets","min ent","Ġforeign ers","Ġtest ify","ĠUs ually","l ords","ĠG ren","ĠPow ell","BI L","Ġs r","Ġadd ict","Ġshell s","Ġs igh","ĠY ale","tern ity","Ġ7 50","E U","ĠR ifle","Ġpat ron","em a","ĠB annon","an ity","Ġtrop ical","ĠV II","c ross","Every thing","ĠIS O","Ġhum ble","ass ing","ĠF IG","Ġupd ating","ys on","Ġcal cium","Ġcompet ent","Ġste ering","Pro t","ĠS Y","ĠFin als","ĠR ug","15 9","13 7","ĠG olf","Ġ12 6","Ġaccommod ation","ĠHug hes","Ġaest hetic","art isan","ĠTw ilight","Ġpr ince","ĠAgric ulture","ĠDis co","Ġpreced ent","Ġtyp ing","author ized","O ption","ĠA ub","l ishes","ach t","m ag","P eter","ĠU FO","mont on","ĠL ith","Ġa rom","Ġsec uring","Ġconf ined","priv ate","Ġsw ords","Ġmark ers","Ġmetab olic","se lect","ĠCur se","ĠO t","g ressive","Ġinc umb","ĠS aga","Ġpr iced","Ġclear ance","Cont ent","Ġdr illing","Ġnot ices","Ġb ourgeois","Ġv est","Ġcook ie","ĠGuard ians","ry s","in yl","Ġ12 4","Ġpl ausible","on gh","ĠOd in","Ġconcept ion","ĠY uk","ĠBaghd ad","ĠFl ag","Aust ral","ĠI BM","Ġintern ationally","ĠWiki Leaks","I ED","Ġc yn","Ġcho oses","ĠP ill","Ġcomb ining","Ġrad i","ĠMoh ammed","def ense","atch ing","Sub ject","ic iency","Fr ame","Ġ{ \"","Ġche ss","Ġtim er","19 0","Ġt in","Ġord inance","emet ery","Ġacc using","Ġnotice able","Ġcent res","Ġl id","ĠM ills","img ur","Ġz oom","erg ic","Ġcomp ression","pr im","f ind","Ġsur g","Ġp and","ĠK ee","ĠCh ad","cell ence","oy le","Ġsocial ism","ĠT ravis","ĠM Hz","Ġgu ild","ALL Y","ĠSub scribe","ĠRel ated","Ġoccur rence","itch ing","Ġfict ional","Ġcr ush","ĠE A","c od","m ix","ĠTri ple","Ġretrie ve","Ġstimul us","Ġpsych iat","ĠDo or","Ġhomosexual ity","Ġelement ary","Ġcell ular","id ian","ĠL aun","Ġintrig uing","Ġfo am","ĠB ass","id i","its u","Ġass ure","Ġcongr at","Ġbusiness man","ĠBo ost","cl ose","Ġl ied","Ġsc iences","ĠO mega","ĠG raphics","Ġ< =","sp oken","Ġconnect ivity","S aturday","ĠAven gers","Ġto ggle","Ġank le","Ġnational ist","mod el","ĠP ool","ophob ia","V ar","ĠM ons","ator ies","Ġaggress ively","C lear","For ge","act ers","Ġhed ge","Ġpip es","Ġbl unt","Ġs q","Ġremote ly","W ed","as ers","Ġref riger","Ġt iles","Ġresc ued","Ġcompr ised","ins ky","Ġman if","avan augh","Ġprol ifer","Ġal igned","x ml","Ġtri v","Ġcoord ination","ĠP ER","ĠQu ote","13 4","b f","ĠS aw","Ġtermin ation","Ġ19 0","Ġadd itions","Ġtri o","Ġproject ions","Ġpositive ly","Ġin clusive","Ġmem br","19 90","old er","Ġpract iced","ink le","Ar ch","Ġstar ters","ari us","Ġinter mediate","ĠBen ef","ĠK iller","Ġinter ventions","ĠK il","ĠF lying","In v","Ġprem ature","Ġpsych iatric","Ġind ie","Ġcoll ar","ĠRain bow","af i","Ġdis ruption","ĠFO X","cast ing","Ġmis dem","c ro","Ġw ipe","ard on","Ġb ast","ĠTom my","ĠRepresent ative","Ġbell y","ĠP O","ĠBre itbart","13 2","Ġmess aging","Sh ould","Ref erences","ĠG RE","ist ical","L P","ĠC av","ĠC razy","Ġintu itive","ke eping","ĠM oss","Ġdiscont in","ĠMod ule","Ġun related","ĠPract ice","ĠTrans port","Ġstatist ically","orn s","Ġs ized","p u","Ġca f","ĠWorld s","ĠRod gers","ĠL un","ĠCom ic","l iving","Ġc ared","Ġclim bed",") {","Ġconsist ed","Ġmed ieval","fol k","Ġh acked","Ġd ire","ĠHerm ione","Ġt ended","ce ans","D aniel","w ent","Ġlegisl ators","Ġred es","g ames","Ġg n","am iliar","Ġ+ +","gg y","th reat","Ġmag net","Ġper ceive","Ġz ip","Ġindict ment","Ġcrit ique","g ard","ĠSaf e","ĠC ream","Ġad vent","ob a","Ġv owed","ous ands","Ġsk i","Ġabort ions","u art","Ġstun ned","Ġadv ancing","Ġlack ed","Ġ\\ \"","Ġsch izophren","Ġeleg ant","Ġconf erences","Ġcance led","ĠHud son","ĠHop efully","Ġtr ump","Ġfrequ encies","Ġmet eor","ĠJun ior","ĠFle et","ĠMal colm","ĠT ools","Ġ ........","Ġh obby","ĠEurope ans","Ġ15 00","ĠInt o","Ġs way","ĠApp ro","ĠCom pl","Comm unity","Ġt ide","ĠSum mit","ä »","Ġinter vals","ĠE ther","Ġhabit at","ĠSteven s","lish ing","ĠDom ain","Ġtrig gers","Ġch asing","Ġchar m","ĠFl ower","it ored","Ġbless ing","Ġtext ures","F ive","Ġliqu or","R P","F IN","Ġ19 62","C AR","Un known","Ġres il","ĠL ily","Ġabund ance","Ġpredict able","r ar","Ġbull shit","le en","che t","M or","M uch","ä ¹","Ġemphas ized","Ġcr ust","Ġprim itive","Ġenjoy able","ĠPict ures","Ġteam mate","pl er","ĠT ol","ĠK ane","Ġsummon ed","th y","ram a","ĠH onda","Ġreal izing","Ġquick er","Ġconcent rate","cle ar","Ġ2 10","ĠErd ogan","ar is","Ġrespond s","ĠB I","Ġelig ibility","Ġpus hes","ĠId aho","Ġagg rav","Ġru ins","ur ations","Ġb ans","Ġan at","sh are","Ġgr ind","h in","um en","Ġut ilities","ĠYan kees","Ġdat abases","ĠD D","Ġdispl aced","Ġdepend encies","Ġstim ulation","h un","h ouses","ĠP retty","ĠRaven s","ĠTOD AY","Ġassoci ates","Ġthe rape","cl ed","Ġde er","Ġrep airs","rent ice","Ġrecept ors","Ġrem ed","ĠC e","Ġmar riages","Ġball ots","ĠSold ier","Ġhilar ious","op l","13 8","Ġinherent ly","Ġignor ant","Ġb ounce","ĠE aster","REL ATED","ĠCur rency","E V","ãĥ ŀ","ĠLe ad","Ġdece ased","B rien","ĠMus k","J S","Ġmer ge","heart ed","c reat","m itt","m und","ĠâĢ ĭ","ĠB ag","Ġproject ion","Ġj ava","ĠStand ards","ĠLeon ard","Ġcoc onut","ĠPop ulation","Ġtra ject","Ġimp ly","Ġcur iosity","ĠD B","ĠF resh","ĠP or","Ġheav ier","ne ys","gom ery","Ġdes erved","Ġphr ases","ĠG C","Ġye ast","d esc","De ath","Ġreb oot","Ġmet adata","IC AL","Ġrep ay","ĠInd ependence","Ġsubur ban","ical s","Ġat op","Ġall ocation","gener ation","ĠG ram","Ġmoist ure","Ġp ine","ĠLiber als","Ġa ides","Ġund erest","ĠBer ry","Ġcere mon","3 70","ast rous","ĠPir ates","Ġt ense","ĠIndust ries","ĠApp eals","ĠN ear","Ġè£ı ç","Ġlo vers","ĠC AP","ĠC raw","Ġg iants","Ġeffic acy","E lement","ĠBeh avior","ĠToy ota","Ġint est","P riv","A I","Ġmaneu ver","Ġperfect ion","Ġb ang","p aper","r ill","Ge orge","b order","in ters","ĠS eth","Ġcl ues","ĠLe vi","ĠRe venue","14 7","Ġv apor","Ġfortun ate","Ġthreat ens","Ġve t","Ġdepend ency","ers ed","art icle","ĠBl izzard","Ġch lor","Ġmin us","ĠB ills","Ġcryptoc urrency","Ġmetabol ism","ter ing","Ġp estic","step s","ĠTre asure","ract ed","ĠConst ant","Ġtem p","13 9","ĠDet ective","ur ally","Ġrecover ing","Ġcort ex","Ġ14 4","cl osed","Ġprejud ice","aun ted","Ġstorm s","ĠN OW","Ġmach inery","Add ress","Ġcompe lled","27 0","Ġdesp air","b ane","Ġveget able","Ġbed s","Lear n","Ġcolor ful","Ġsp ike","Ġmarg ins","Ġsymp athy","Ġworks hop","ĠC BC","S at","Ġburn s","ĠG ender","Ġ12 9","ĠC able","Ġdeb ts","ĠThe resa","Ġreflect ing","Ġa irst","Ġr im","ram id","Ġweakness es","W rit","ogg le","t i","ĠCh arge","Ġwe ighed","Ġ( .","Ġl aughter","Ġrou ter","ĠDemocr acy","D ear","Ġhas ht","Ġd y","Ġhint s","run ning","Ġfin ishes","ar us","M ass","res ult","asc us","Ġv intage","Ġcon qu","Ġwild ly","ac ist","Ġl ingu","Ġprot agonist","st rom","te enth","ĠSol o","m ac","f illed","Ġre nown","it ives","Ġmot ive","ĠAnt ar","ĠM ann","ĠAd just","Ġrock ets","Ġtrou bling","e i","Ġorgan isms","ass is","Christ ian","Ġ14 5","ĠH ass","Ġsw all","Ġw ax","ĠSurv ival","V S","ĠM urd","v d","stand ard","Ġdrag ons","Ġacceler ation","r ational","f inal","Ġp aired","ĠE thereum","Ġinterf aces","Ġres ent","Ġartif acts","Å «","are l","Ġcompet itor","ĠNich olas","ĠSur face","c pp","ĠT ot","Ġeconom ically","Ġorgan ised","Ġen forced","in ho","Ġvar ieties","Ġab dom","ĠBa iley","id av","ĠSal v","p aid","Ġalt itude","ess ert","ĠG utenberg","are a","op oulos","Ġprofess ors","igg s","ĠF ate","he y","Ġ3 000","D ist","Ġtw ins","c ill","ĠM aps","Ġtra ps","Ġwe ed","ĠK iss","Ġy oga","Ġrecip ients","ĠWest minster","Ġpool s","ĠWal mart","18 8","ĠSchool s","att ack","ĠAR M","par agraph","W arning","j l","Ġself ish","anche z","ĠHe ights","F re","ĠS oph","Ġ --------------------------------","t ml","33 3","Ġraid s","Ġsatell ites","KE Y","Ġlast s","Ñ Ĥ","In s","ĠD ame","Ġunp redict","// /","gh ai","Ġart illery","Ġcru ise","Ġg el","ĠCabin et","Ġbl ows","ĠE sp","Ġprox imity","ot he","ĠSk ills","ĠU pper","ob o","ĠN DP","Ġenjoy s","Ġrepe ating","ĠConst ruction","ĠQuest ions","H illary","Ġu int","Ġprocess ors","ĠGib son","ĠMult iple","q a","ĠB om","ĠM iles","vent ional","Ġhur ts","s kin","ĠA IDS","Ġadvis ers","ĠR oot","Ġmethod ology","ĠD ale","Ġdet on","ĠKnow ledge","sequ ently","Ġ12 1","Ġconnect s","C y","ĠD anger","Ġcontribut ors","ĠB ent","Ġbr ass","ĠGun s","int o","ĠFort une","Ġbro ker","bal ance","Ġlength s","Ġv ic","Ġaver aging","Ġappropri ately","ĠCamer a","Ġsand wich","ĠCD C","Ġcoord inate","Ġnav ig","Ġgood ness","l aim","Ġbra ke","Ġextrem ist","ĠW ake","ĠM end","ĠT iny","ĠC OL","ĠR F","ĠD ual","ĠW ine","C ase","Ġref ined","Ġl amp","L ead","Ġb apt","ĠCar b","ĠS add","ĠMin neapolis","PD F","Ear ly","ĠH idden","I ts","ĠT IME","Ġp ap","Ġcommission ed","ĠF ew","ĠCol ts","ĠB ren","Ġbot hered","Ġlike wise","Ex per","ĠSch w","c ry","n n","ĠM itch","im on","M G","b m","UM P","r ays","Ġregist ry","Ġ2 70","ach ine","re lla","ant ing","00 000","Ġru ined","sp ot","Ġt a","Ġmaxim ize","Ġincon ven","D ead","H uman","En abled","ĠMar ie","Ġch ill","ĠParad ise","Ġstar ring","ĠLat ino","ĠProt ocol","ĠE VER","Ġsuppl iers","m essage","ĠBro ck","Ġser um","âĸĪâĸĪ âĸĪâĸĪ","Ġen comp","Ġamb ition","ues e","Ġar rows","And rew","Ġanten na","Ġ19 61","ĠB ark","Ġb ool","ãĤ ª","ĠSt orage","Ġrail way","Ġtoug her","ĠC ad","Ġwas hing","P y","' ]","em bed","ĠMem phis","ack le","Ġfam ously","ĠF ortunately","ov ies","Ġmind set","Ġsne ak","ĠD h","RA W","ĠSim pson","Ġliv est","Ġland mark","Ġc ement","L ow","Ġthr illed","ĠCour se","in el","Ġch uck","id ate","gl obal","Ġwh it","Ġ �","ad ays","s ki","ĠS V","Ġvir uses","30 6","ĠResp ons","Ġthe aters","ĠBr anch","ĠGene va","ĠM K","Ġunbel iev","Ġcommun ist","Orig inal","ĠRe ceived","ĠTrans fer","ĠAr g","In put","ĠStr ategy","Ġpal ace","the ning","D ri","Ġsent encing","umbn ail","Ġp ins","re cy","Ġs iblings","Get ting","ĠB U","ĠNorth west","Ġprolong ed","ĠSak ura","C omb","ĠB our","Ġinadequ ate","ĠK ash","Ġus ername","ĠImpro ve","Ġbatt ling","ĠM AC","Ġcurric ulum","Ġs oda","ĠC annon","Ġsens ible","sp ons","De cember","Ġw icked","ĠP engu","Ġdict ators","ĠHe arts","og yn","Ġsimilar ities","ĠSt ats","Ġh ollow","it ations","\": [","Ġh over","ĠList en","s ch","S und","Ġc ad","ĠPar ks","Ġl ur","Ġhy pe","ĠL em","N AME","is ure","Fr iday","Ġshoot s","Ġclos es","Ġd b","ĠR idge","ĠDiff erent","Ġrepl ies","ĠBroad way","op ers","Ġint oler","ĠZe us","akes pe","Ġpropri etary","Ġrequest ing","Ġcontro llers","ĠM IN","im edia","be cca","Ġexp ans","Ġoil s","B ot","ĠCh and","Ġpr inter","Ġto pped","ĠP OL","ĠEar lier","S ocial","av in","Ġdecre ases","ĠSe b","Ġspecific ations","ĠBl ast","ĠK urt","Ġfre el","B rown","Ġdil ig","ro e","ĠPro blem","ĠQu ad","Ġdecent ral","ĠV ector","an ut","Ġplug ins","ĠGreg ory","Ġfuck ed","el ines","ĠAmb assador","t ake","Ġcle ans","ong yang","An onymous","st ro","\" }","al ine","ĠO dd","ĠE ug","2 16","Ġbo il","ĠP owers","Ġnurs es","Ob viously","ĠTechn ical","Ġexceed ed","OR S","Ġextrem ists","Ġtr aces","ex pl","Ġcom r","ĠS ach",") /","Ġm asks","Ġsc i","B on","Ġreg ression","we gian","Ġadvis or","it ures","ĠV o","ex ample","ĠInst ruct","Ġs iege","Ġredu ctions","pt r","Ġstat utory","Ġrem oves","Ġp uck","red its","Ġbe e","Ġsal ad","Ġpromot ions","ĠJosh ua","with standing","ET H","ĠCh a","im us","Ġexpend iture","aun ting","Ġdelight ed","Ġ15 5","be h","Ġcar pet","ĠSp art","Ġj ungle","l ists","Ġbull ying","ĠNob el","ĠGl en","Ġreferen ced","Ġintrodu ces","se in","Ġcho pped","gl ass","ĠW rest","Ġneutral ity","Ġâ Ļ","Ġinvestig ator","Ġshel ves","Ġun constitutional","Ġreprodu ction","Ġmer chant","m ia","Ġmet rics","Ġexplos ives","ĠSon ia","Ġbod ily","Ġthick ness","Ġpredomin antly","ĠAb ility","Ġmon itored","IC H","Ġ] .","ĠMart inez","Ġvis ibility","Ġqu eries","Ġgen ocide","ĠWar fare","Qu ery","Ġstud ios","Ġemb ry","Ġcorrid or","Ġclean ed","com plete","ĠM H","Ġenroll ment","ING S","Ġimpact ed","Ġdis astrous","ĠY un","ĠCl aire","ĠBas ically","y t","uster ity","Ġindirect ly","w ik","Ġd od","ĠCar r","Ġam p","Ġprohib it","ĠIn itial","ĠR d","ij i","Ġeduc ate","c orn","i ott","ĠBeaut y","Ġdetect ive","ĠCon n","s ince","Ġst agger","Ġob ese","Ġb ree","olog ic","is se","walk er","Ġbl ades","Ġlaw ful","fun c","ĠBeh ind","Ġappet ite","Ġ( *","Ġt ennis","Ġoff spring","Ġj ets","Ġstruct ured","Ġafore mentioned","N ov","Ġsc aling","f ill","Ġst ew","Ġcur b","ĠStep han","ed In","S F","ob ic","é ŃĶ","ou g","ĠM M","Ġgen etically","ope z","13 6","Ġu mb","anc ers","Ġcoh ort","Ġmerch andise","Ġimp osing","ĠLegisl ature","ĠArch ive","iv ia","ĠN aval","Ġoff ences","Ġmir acle","Ġsn apped","Ġf oes","Ġextensive ly","ĠR af","Ġc ater","ed ience","K it","ĠB in","Ġrecomm ends","ĠC ities","Ġrig id","ĠRE AD","ĠNob le","ĠT ian","Ġcertific ates","ant is","o iler","ĠBudd hist","d id","Ġsurvey ed","Ġdown ward","Ġprint s","ĠMot ion","ron ics","ĠS ans","oss ibly","u ctions","Ġcolon ies","ĠDan ish","un it","Ġsp oil","Ġadvis ory","ber ries","Pl an","Ġspecific ation","op hers","ĠRes ource","Ġsh irts","prising ly","commun ications","Ġtriv ial","Ġmention ing","ise xual","Ġsupp lements","Ġsuper vision","B P","v or","Ġw it","Ġco oldown","Ġplaint iff","ĠReview s","ĠS ri","ĠM int","ĠSug ar","Ġafter ward","ĠPri est","ĠInvest ment","og ene","ĠT aking","Ġstretch ing","Ġinflamm ation","ĠTe hran","Ġl ining","Ġfree zing","ĠEnt ity","Ġins piring","spe cial","pr ice","Ġsu e","ĠP orter","oun ge","ET A","ĠD erek","ĠLu is","u o","ym ph","Ġex terior","ih il","ĠAsh ley","in ator","Ġnut rients","ĠTh rones","Ġfin ances","ĠIn spect","Ġspe cially","ĠRequ ired","ĠP TS","ĠViol ence","oint ed","sh ots","Ġex cerpt","co on","IN S","ĠG ri","Ġrecogn ised","We ek","You ng","Ġv om","is le","ĠCur ry","ĠBudd h","Ġnot ebook","Ġd urable","/ ?","ĠG ad","ĠP upp","Ġforg ive","p ark","Ġpersonal ities","an alysis","cl amation","Ġelev ator","Ġware house","ĠR ole","un n","Ġillust ration","ĠSc an","Ġatmosp heric","Im port","AN C","rict ed","f u","01 0","Ġar che","Ġreward ed","akespe are","Ġintern ally","ĠR BI","alk er","Ġeleph ant","ow itz","ĠP izza","Ġbip artisan","é s","Ġslow ed","ĠSt ark","Ġover ride","OU S","Ġ3 20","undred s","ĠDe ck","ĠC ensus","be e","14 6","ot or","Ġ ip","Ġu b","oc ations","ĠBut ton","r ice","Ġc ripp","ff f","Ġorig inated","Ġoverwhel med","app a","Ġfore most","âĢ ij","ĠL EG","re lease","eat ured","at ches","Ġre ps","Ġl ending","ĠRe ference","ĠCl ient","16 5","vent h","Com plete","ĠPat rol","Ġsw orn","c am","Ġshut tle","ĠR alph","Ġh ometown","- ,","on al","ĠB P","å ı","Ġpersu ade","ĠAlex and","Ġcomb ines","Ġv ivid","ĠL ag","Ġenc oding","Ġsal vation","w en","ĠRec overy","i ya","Un iversity","ĠB iden","Ġbud gets","ĠTex ans","f its","Ġhon ored","Ġp ython","T D","## #","cl one","Ġbl ink","ĠL iquid","Ġunemploy ed","Ġcl ashes","ĠCoun sel","Ġdirect ing","Ġpun ct","ĠFal cons","Ġsh ark","ĠDam ascus","Ġje ans","Ġemb ark","Ġse ize","Ġup wards","2 80","ĠE z","ĠAny thing","Ġex otic","l ower","ĠCreat or","ĠU m","Ġsubur bs","ber ger","ĠW end","Ġm int","ĠX X","ĠD ro","Ġsuff ers","Ġher b","t ree","Ġfrag ile","Ġflood ed","ĠAl cohol","ole an","ny der","ĠK O","F ram","Ġ13 6","Ġow ed","ĠMe lee","ĠH ash","Ġwh isk","Ġsu do","r r","Qu ick","app ro","Ġi i","ĠEx amples","he e","Ġpromot es","per ature","k ar","ĠHon or","Ġs odium","ĠL if","ros so","intend ent","Ġcorrespond ent","F ound","sec ret","Ġident ifies","ag ne","Ġl ou","ĠP P","Ġcoinc idence","m ove","Ġmilit ia","Ġinf iltr","ĠPrim ary","Ġpitch ing","ĠI b","ĠGO OD","ãĤ ¸","ĠW izards","ir al","ĠVen us","R R","ĠâĢ ķ","ĠCase y","Ġsad ly","Ġadm ire","Ġembarrass ed","c b","M el","Ġtub es","Ġbeaut ifully","ĠQueens land","Bel ow","re z","qu et","ple asant","Ġ «","C amp","Ġdec isive","19 98","ĠL amb","ut ton","h n","ĠJ agu","au nder","ĠC ord","Ġcl erk","Ġca ffe","Ġwip ed","Ġre im","ĠMount ains","Ġimprison ed","Ġdevelop s","ĠP ra","Ġmodel ing","Any one","ance l","ĠS it","Ġshield s","Ġl awn","Ġcard iovascular","Ġdemonstr ating","Ġpar se","ĠIsrael is","Ġeuro s","14 3","Ġgl orious","ins ki","ec d","Ġcondition ing","Ġhel pless","Ġmicro sc","ĠHar bor","Ġst akes","Ġ2 60","Ġun equ","ĠFl oyd","Ġd amp","Ġappar atus","ĠLaw s","Ġcoun ters","Ġindu ce","at able","ĠAh med","Ġsl am","N ovember","Ġpers ist","Ġim minent","á n","Ġsh red","Ġph ases","ĠEd monton","ĠArm strong","ĠMe et","ĠK itty","Ñ Ģ","c irc","ĠAd ult","Ġa rose","ĠX en","D an","g ow","Ġsuper f","ĠAd mir","Ġend ure","Ġkey word","yr us","Ġy arn","Ġpath way","ĠHop kins","mid t","Ġcens orship","d ependent","Ġinstruct or","S ources","Ġto e","Ġball oon","N ob","Ġsw ear","ĠCast ro","Ġgl oss","ĠK avanaugh","Ġremark ably","Ph otos","ĠN om","ĠS outheast","y ers","Ġvalid ation","Ġcann on","ĠVict ory","ĠPier re","Ġcaut ious","Aud io","Ġf etch","ĠG ift","ĠH yp","Ġrem edy","Z E","Ġsc ent","Ġbe ard","ĠR ut","- \"","Ġpat ents","H y","Ġun just","Ġpot ato","Ġforth coming","Ġche f","ĠR ift","aff e","ĠR OM","ĠL aunch","Ġp ads","ĠNe o","Ġon set","Ġsquee ze","s afe","Ġpref ix","ĠT M","ĠN early","ĠClin ical","ĠM ental","ot iation","ĠUn ic","ant ry","ĠC ir","Ġep it","à ¦","Ġextract ed","verse ly","ri ad","Ġstr ains","Ġto ps","Ġpo em","ĠRand y","ĠMap le","TH ER","up iter","ĠSS D","ļ é","Ġun con","per ing","Ġsle pt","in ers","Ġunder water","ĠEv idence","g one","20 5","Ġhistor ians","Ġsynt hesis","Ġf rog","b asketball","Ġvibr ant","Ġsub ord","Ġ3 65","ĠD ial","Ġcooper ate","HA HA","Ġgreet ed","15 8","Ġj azz","Ġinto x","ĠWalk ing","Ġsuper visor","ĠF usion","ĠMer cedes","s end","H am","s d","n l","Ġtour s","ĠF IFA","Ġcul p","g d","30 4","Ġple as","Ġillust rates","ĠColomb ia","Ġhighlight ing","ĠSum mary","Ġexp osing","ĠD ru","Ġir ony","r itional","ĠCar roll","ĠEll is","P ict","ĠR apt","Ġad apter","Ġun m","Ġcor pse","Ġceleb rities","D en","at um","ĠAp ocalypse","ĠW ag","lin ing","Ġhorm ones","R ub","ĠX i","ĠV aults","20 8","alky rie","inos aur","Ġfeed s","v ity","Ġdefe ating","W ait","Ġemphas ize","ĠSteel ers","yr inth","le ys","ĠWhe never","Current ly","ĠCl ock","Ġcollect ively","any on","ĠJ P","Ġment ality","Ġdownload s","Ġsurround ings","ĠBarn es","Ġflags hip","Ġindic ators","Ġgra pp","Jan uary","ĠElement al","ĠAthen a","ib al","Ġs ights","Ġcap ita","ĠTreat y","Ġvo iced","ĠG az","let te","Ġy a","Ġexp ired","Leg end","H ot","n ature","Ġunst able","Ġ2 80","à º","Com ment","AL E","Ġquest s","Ġhand ler","n is","Ġvers atile","Ġconce al","enge ance","ĠInter active","Ġobs essed","ĠDog s","Ġcr acked","S ound","s v","ĠD ylan","ro ads","f x","ĠCath olics","ĠH ag","Ġsl ammed","Ġgl owing","s ale","Ġtiss ues","ĠCh i","ne e","Ġc her","s ic","ur rection","Ġb acon","ul atory",") .\"","Ġir regular","FOR M","ass ed","Ġintention al","Ġcompens ate","ĠSpe aking","ĠS ets","15 3","Ġconvent ions","b ands","em ade","Ġe cc","ĠWin ston","ĠAssass in","ĠBelg ian","Ġdepend ence","Ġnic he","Ġb ark","ĠJ azz","Ġdisadvant age","Ġgas oline","Ġ16 5","çļ Ħ","ess a","mod ule","ang ular","O Y","ĠTreat ment","it as","ol ation","ĠArn old","Ġfe ud","ĠN est","Ġthe atre","ew ater","Ġmin ors","olic y","ĠH aven","div ision","Ġtr unk","F ar","ĠP ull","Ġcapt uring","Ġ18 00","ĠTe en","Ġex empl","Ġclin ics","ĠB urg","Ġsubst it","Ġpay load","ĠL av","ĠT roy","ĠW itness","Ġfrag ments","Ġpass words","Ġg ospel","ĠG in","Ġten ants","ol ith","S ix","Pre vious","ĠAg es","ĠDar win","Ġbl at","Ġem pathy","sm ith","b ag","ĠE cho","ĠC amb","ĠM add","ĠB oo","Ġred e","ĠBurn ing","Ġsmooth ly","ĠAd rian","ĠV ampire","ĠMon sters","ste am","Sty le","M a","re a","ĠD war","aly st","urs or","Ġelim ination","Ġcrypt o","ch t","ĠE ternal","â̦ ]","ĠS orce","I ll","N ER","Ġu h","Con clusion","w age","Ġresp ir","Ġrem inis","het ical","Ġg y","Ġutil ized","ic idal","Ġ19 00","Ġhun ters","ĠSw an","ĠRe act","Ġvis itor","ĠThanks giving","30 8","Post s","Ġh ips","19 97","om ers","Ġkn ocking","ĠVeh icle","Ġt il","Ġ13 8","Ġm i","ĠInvest igation","ĠKen ya","Ġcas ino","Ġmot ives","Ġreg ain","re x","Ġweek ends","Ġstab bed","bor o","Ġexplo ited","ĠHA VE","ĠTe levision","c ock","Ġprepar ations","Ġende av","ĠRem ote","ĠM aker","ĠPro du","ĠEv an","Ġinform ational","ĠLouis ville","15 4","ĠDream s","Ġpl ots","ĠRun ner","Ġhur ting","Ġacad emy","ĠMont gomery","n m","ĠL anc","ĠAl z","2 10","el ong","Ġretail er","Ġar ising","Ġrebell ion","Ġbl onde","play ed","Ġinstrument al","C ross","Ġret ention","Ġtherape utic","Ġse as","Ġinfant ry","ĠCl int","Ġprompt ing","Ġbit ch","Ġst ems","ĠK ra","Ġthe sis","ĠB og","ru ed","Ġk ings","Ġcl ay","ific ent","ĠY ES","ĠTh ing","ĠCub s","vey ard","els h","in arily","ĠE y","ĠRoll ing","Ġev olving","Ind ia","Ġrecogn izes","Ġgrad uation","is ers","Ġfert ility","ĠMil an","Comm and","Ġbox ing","Ġ19 43","Ġgl uten","ĠEm ir","Ġid ol","Ġcon ceived","ĠCre ation","Mer it","udd y","uss ions","ĠLie utenant","iet al","Ġunch anged","ĠSc ale","ĠCrime a","ball s","ator ial","Ġdepth s","Ġempir ical","Ġtrans m","Ġuns afe","miss ible","com fort","15 6","Ġmechan ic","00 2","l ins","Ġsm oked","P os","Ġslow ing","Ġl av","Tex as","Ġche ating","ĠMet ropolitan","eth yl","Ġdiscover ing","as se","Ġpen cil","ĠPy ongyang","Ġclos et","ĠShe et","ĠEnt ry","ou stic","Ġmy st","er ate","ari at","Ġminer als","Ġmusic ian","ĠP ul","ĠM az","24 9","Ġper missions","Ġ iv","en ary","ick ers","ĠB ing","he a","en able","Ġgri ev","Ġassert ed","ĠColon el","Ġaff idav","w o","Ġse ated","ĠR ide","Ġpaint ings","ĠP ix","Ġ13 7","ish i","umb ai","g otten","ĠEar l","Ġin ning","Ġc ensus","Ġtrave lled","ĠCons ult","18 5","b ind","Ġsimpl icity","Ġoverlook ed","ĠHelp ful","Ġmon key","Ġoverwhelming ly","Bl ood","ĠFl int","ĠJ ama","ĠPres ent","ĠR age","ĠT A","pt ive","Ġturn out","w ald","ĠD olphins","ĠV PN","Ġon ion","Ġcraft ing","m ma","ĠMerc ury","Ġarr ange","Ġalert s","ĠO T","zb ollah","Ġg ases","ĠRichards on","s al","l ar","Ġfro st","Ġlower ing","Ġacc laim","Ġstart ups","ĠG ain","ess ment","Ġguard ian","äº º","ĠP ie","ĠL inks","Ġmer its","Ġaw ake","Ġparent al","Ġexceed s","Ġid le","ĠPil ot","Ġe Bay","ĠAc cept","ipe g","C am","ĠK ot","Ġtrad ers","olit ics","unk er","ĠP ale","os i","an mar","Ġ19 47","ĠF ell","est ial","it ating","G F","ĠS r","if ted","Ġconnect or","ĠB one","ill es","2 60","h ma","Ġoverl ap","ĠGit Hub","Ġclean er","ĠBapt ist","ĠW AS","Ġlung s","Ñ ģ","ĠB UT","Ġc ite","Ġpit ched","reat ment","Ġtro phies","ĠN u","38 6","ĠPr ide","Ġattend ees","[ ]","17 9","Ġspat ial","Ġpri zes","ĠRel igion","Ġshow case","ĠC ategory","vid ia","T arget","Pro perty","? ,","Ġf usion","p ie","ĠU CLA","Ġsound track","Ġprin cess","ĠC aval","sh ould","Ġlim bs","Back ground","Ġlone ly","Ġc ores","ĠT ail","she et","Ġ13 2","R a","ãĤ «","ĠB olt","Ġbook ed","Ġadmin ister","Ġequ als","w y","Ġobserv ing","ĠBar on","ĠAd obe","Ġv irgin","ĠSocial ist","M ove","gh azi","ĠLind a","2 12","Ġbre wing","Ġmerch ants","bur se","Ġdiv or","Ġmet als","ĠN er","Ġsum s","ĠEn emy","Ġen vision","Ġgrant ing","ĠH oney","ĠSk yrim","Ġsoc io","gr aded","Ġselect ive","W ASHINGTON","Ġ19 48","ĠSir ius","ĠG ross","act ivity","ĠI van","Ġfur ious","BS D","ĠPre vious","Ġrespons ive","Ġchar itable","Ġle aning","ĠP ew","Ġviol ates","\\\\\\\\ \\\\\\\\","ĠCom ing","w ire","Ġpo et","Ġres olutions","comm and","ĠPortug uese","Ġnick name","Ġde af","Feb ruary","Ġrecogn ise","Ġentire ty","Ġseason al","pl aced","ĠTe legraph","Ġmicro phone","our ing","Ġgr ains","Ġgovern ed","Ġpost p","ĠW aters","in ement","Ġund ocumented","ĠCom cast","Ġf ox","Ġassault s","re on","man y","ĠJen kins","ĠAny way","Ġassess ments","Ġdown s","ĠM ouse","Ġsuper b","k t","ĠD ow","Ġtax ation","4 01","Ġsm iles","Ġundert aken","Ġex h","Ġenthusi astic","Ġtw ent","Ġgovernment al","Ġautonom y","ĠTechn ologies","ĠCh ain","Ġpreval ent","f b","Ġnic otine","og ram","j ob","Ġawa iting","ĠMen u","Ġdep uties","k ov","ish ops","But ton","ĠShan ghai","Ġdies el","ĠD uck","R yan","ĠPC s","N F","j ury","ent e","Ġinacc urate","edd y","Wh atever","Ġshow c","ĠN ad","od us","et r","Ġplaint iffs","ĠW OR","ĠAss ange","Ġpriv at","Ġpremium s","Ġt am","UR L","Ġel ites","ĠR anger","otten ham","ĠH off","ĠAt hens","Ġdefin ite","Ġs ighed","Ġeven ly","2 11","ĠAm ber","ak ia","Ġmail ing","Ġcr ashing","ĠConfeder ate","ru gged","W al","ĠDep ths","Ġjuven ile","Ġreact or","Introdu ction","ĠDel uxe","19 95","ĠS anchez","ĠM ead","iv able",": -","ĠPlan ning","ĠT rap","qu in","ĠProt ect","ve red","In formation","Ġkid ney","inn amon","l as","Ġpolic ing","Ġtoler ate","ĠQ i","Ġbi ased","F ort","ĠK i","s ave","Ġprivile ged","Ġbe asts","ĠGl as","ĠC inem","Ġcome back","Sund ay","Ġext inction","h ops","Ġtrans mit","Ġdoub les","ĠFl at","16 7","Ġdis puted","Ġinjust ice","f oo","V ict","role um","ĠJul ie","Con text","ĠR arity","iss ue","Comp onent","Ġcounsel ing","an ne","d ark","Ġobject ions","u ilt","Ġg ast","Ġpl ac","Ġun used","ãĥ ĩ","ĠT rial","ĠJ as","hed ral","ob b","Ġtempor al","ĠPR O","ĠN W","ĠAnn iversary","L arge","Ġther m","Ġd avid","Ġsystem ic","ĠSh ir","m ut","ĠNe pt","add ress","Ġscan ning","Ġunderstand able","Ġcan vas","C at","ĠZ oo","Ġang els","L O","ĠStat ement","ĠS ig","ov able","ĠA way","sh aring","ocr ats","st ated","Ġweigh ing","N or","w ild","B ey","Ġaston ishing","ĠReyn olds","Ġop ener","Ġtrain er","Ġsurg ical","p n","Ġadjust ing","whe el","Ġf rown","erv ative","Ġsusp end","With in","te in","Ġobst acle","Ġliber ties","ym es","Ġur anium","ans om","an ol","ub a","ĠL oss","Ġa rous","ĠHend erson","W ow","s pl","c ur","Ġ Ń","Ġtheir s","Dam age","Ġdownload ing","Ġdisc ern","ĠSt o","ĠFl a","Ġh ath","ĠA j","Ġun pleasant","Europe an","exp ensive","Ġscreens hot","ĠU V","Ġall ied","ĠPers ian","Ġmonop oly","Ġat om","ĠReds kins","\"> <","Ġcan cell","Ġcinem a","13 1","f air","ĠAlf red","Ġd uck","arg s","22 3","ĠIS I","Ġsign aling","in ar","Ġlaugh s","Ġfor wards","Ġreck less","Ġlisten ers","at ivity","Ġvast ly","n ant","L ess","ĠHun ting","ĠScient ific","IT ED","Ġkn ight","ĠH TC","us a","t mp","Ġr ude","ĠLegend ary","Ġar ises","B ad","ĠCl aim","pe g","Ġreal ities","Th ink","Ġ °","Ġro de","Ġstri ve","Ġan ecd","Ġshort s","Ġhypot hes","Ġcoord inated","ĠGand hi","ĠF PS","R ED","Ġsuscept ible","Ġshr ink","ĠCh art","Hel p","Ġ ion","de ep","rib es","ĠK ai","ĠCustom er","Sum mary","Ġc ough","w ife","Ġl end","Ġposition ing","Ġlot tery","ĠC anyon","Ġf ade","Ġbron ze","ĠKenn y","Ġbo asts","ĠEnh anced","rec ord","Ġemer gence","Ġa kin","ĠB ert","it ous","âĸ ij","Ġst ip","Ġexch anged","om ore","als h","Ġreserv oir","Ġstand point","W M","Ġiniti ate","Ġdec ay","Ġbrew ery","Ġter ribly","Ġmort al","lev ard","Ġrev is","N I","el o","Ġconf ess","ĠMS NBC","Ġsub missions","Cont roller","Ġ20 2","ĠR uth","} );","ĠAz ure","Ġ .\"","20 6","ĠMarket ing","Ġl aund","ien cies","Ġrenown ed","ĠT rou","ĠN GO","ble ms","Ġterr ified","Ġwar ns","Ġper t","Ġuns ure","4 80","ale z","ult z","ĠOut side","Ġst yl","ĠUnder ground","Ġp anc","Ġd ictionary","Ġf oe","rim inal","ĠNor wegian","Ġj ailed","Ġm aternal","é e","ĠLu cy","c op","Ch o","Ġuns igned","ĠZe lda","ĠIns ider","ĠContin ued","Ġ13 3","ĠNar uto","ĠMajor ity","16 9","ĠW o","ãĤ ĵ","Ġpast or","Ġinform al","Ð ½","an throp","jo in","ãģ Ĺ","it ational","N P","ĠWrit ing","f n","ĠB ever","19 5","Ġy elling","Ġdr astically","Ġe ject","Ġne ut","Ġth rive","ĠFre qu","ou x","Ġpossess es","ĠSen ators","ĠD ES","ĠSh akespeare","ĠFran co","ĠL B","uch i","Ġinc arn","Ġfound ers","F unction","Ġbright ness","ĠB T","Ġwh ale","ĠThe ater","m ass","ĠD oll","S omething","Ġecho ed","ĠHe x","c rit","af ia","Ġgodd ess","Ġele ven","ĠPre view","ĠAur ora","Ġ4 01","uls ive","ĠLog an","in burgh","ĠCent ers","ĠON LY","ĠA id","Ġparad ox","Ġh urd","ĠL C","D ue","c ourt","Ġoff ended","Ġeval uating","ĠMatthew s","Ġto mb","Ġpay roll","Ġextra ction","ĠH ands","if i","Ġsuper natural","ĠCOM M","] =","dog s","Ġ5 12","ĠMe eting","Rich ard","ĠMax imum","Ġide als","Th ings","m and","ĠReg ardless","Ġhum ili","b uffer","L ittle","ĠD ani","ĠN ak","Ġliber ation","ĠA be","ĠO L","Ġstuff ed","ac a","ind a","raph ic","Ġmos qu","Ġcampaign ing","Ġoccup y","S qu","r ina","ĠW el","ĠV S","Ġphys ic","Ġp uls","r int","oad ed","ET F","ĠArch ives","Ġven ues","h ner","ĠTur bo","Ġl ust","Ġappeal ed","que z","il ib","ĠTim othy","Ġo mn","d ro","Ġobs ession","ĠSav age","19 96","Gl obal","J es","2 14","Ġsl iding","Ġdisapp ro","ĠMag ical","Ġvolunt arily","g b","ane y","Ġprop het","ĠRe in","ĠJul ia","ĠW orth","aur us","Ġb ounds","ie u",")) )","Ġcro re","ĠCitiz en","S ky","Ġcolumn ist","Ġseek ers","ond o","IS A","ĠL ength","Ġnost alg","Ġnew com","Ġdet rim","ent ric","3 75","ĠG E","Ġaut op","Ġacadem ics","App Data","ĠS hen","Ġid iot","ĠTrans it","Ġteasp oon","W il","K O","ĠCom edy","> ,","Ġpop ulated","W D","Ġp igs","ĠO culus","Ġsymp athetic","Ġmar athon","19 8","Ġseiz ure","s ided","Ġd op","irt ual","L and","ĠFl oor","osa urs","... ]","Ġl os","Ġsubsid iary","E Y","ĠPart s","ĠSt ef","ĠJud iciary","Ġ13 4","Ġmir rors","Ġk et","t imes","Ġneuro log","Ġc av","ĠGu est","Ġtum or","sc ill","ĠLl oyd","E st","Ġcle arer","Ġstere otypes","Ġd ur","not hing","Red dit","Ġnegoti ated","---------------- --------","23 5","Ġfl own","ĠSe oul","ĠRes ident","ĠS CH","Ġdisappear ance","ĠV ince","g rown","Ġgrab s","r il","ĠInf inite","ĠTw enty","Ġpedest rian","Ġjer sey","ĠF ur","ĠInf inity","ĠEll iott","Ġment or","Ġmor ally","Ġob ey","sec ure","iff e","Ġantib iotics","ang led","ĠFre eman","ĠIntrodu ction","J un","Ġm arsh","ic ans","ĠEV ENTS","och ond","W all","icult y","Ġmisdem eanor","Ġl y","Th omas","ĠRes olution","Ġanim ations","ĠD ry","Ġinter course","ĠNew castle","ĠH og","ĠEqu ipment","17 7","Ġterrit orial","Ġarch ives","20 3","Fil ter","ĠMun ich","Ġcommand ed","ĠW and","Ġpit ches","ĠCro at","Ġrat ios","ĠM its","Ġaccum ulated","ĠSpecific ally","Ġgentle man","acer b","Ġp enn","Ġa ka","ĠF uk","Ġinterven e","ĠRef uge","ĠAlz heimer","Ġsuccess ion","oh an","d oes","L ord","Ġsepar at","Ġcorrespond ence","Ġsh iny","P rior","Ġs ulf","Ġmiser able","Ġded ication","( ).","Ġspecial ists","Ġdefect s","ĠC ult","ĠX ia","Ġje opard","ĠO re","Ab ility","Ġle ar","Ġamb itions","ĠB MI","ĠArab s","Ġ19 42","Ġpres ervation","ific ate","Ġash amed","l oss","ĠRest aur","Ġrese mble","Ġen rich","ĠK N","ĠCl an","fl oat","Ġplay able","IT T","Ġharm ony","arr ison","ĠWe instein","w ere","Ġpoison ing","ĠCom put","ĠWord Press","m ajor","ĠVal ve","F an","ĠTh row","ĠRom ans","ĠDep ression","ad os","Ġtort ured","Ġbal ancing","bott om","Ġacqu iring","ĠMon te","ard i","Ġa ura","Ġ# #","ĠStand ing","ĠAtl as","C F","Ġintr ins","ĠBen ghazi","Ġcamp ing","Ġt apped","bl ade","st rous","ĠR abb","ĠW ritten","t ip","ĠNe igh","ster dam","ĠAll ow","ĠHe aling","ĠR hod","n um","Ġcaffe ine","ĠPer cent","Ġbo o","Ġapp les","30 5","Ġwel coming","Ġappl aud","Ġa usterity"," ±","ĠRe ality","ef e","å ®","Ġsu cks","Ġtab s","ĠPay Pal","Ġback pack","Ġgif ted","abul ary","ĠSc out","ir teen","Ġch in","Ġo mitted","Ġnegative ly","Ġaccess ing","ĠE arn","Ġambul ance","Ġhead phones","Ġ20 5","ĠRef resh","p resident","ĠKit chen","ĠEnt ered","ĠS nyder","00 5","om ical","Ġborrow ed","ĠN em","Ġav iation","Ġst all","rim ination","Ġuniform s","it ime","ĠSim mons","ener gy","ab lished","y y","qual ified","Ġrall ies","ĠSt uart","fl ight","Ġgang s","r ag","Ġv ault","lu x","ĠCom par","Ġdesign ation","20 9","ĠJ os","d ollar","z ero","Ġwell s","30 3","Ġconstitu ents","Ġhe ck","Ġc ows","Ġcommand ers","Ġdifferent ial","ĠC atherine","29 9","Ġval ve","Ġbr ace","Ġperspect ives","c ert","f act","icular ly","ĠMc N","pl anes","Ġint ric","Ġpe as","ov an","Ġtoss ed","ret ch","ĠL opez","Ġunf amiliar","de ath","ĠA part","ĠCh ang","Ġrelie ved","rop he","Ġair ports","Ġfre ak","ut il","M ill","ĠCh in","ĠOw en","m ale","ĠBro ken","ĠWind s","ro b","r ising","Ġfire fighters","Ġauthor itarian","Ġ14 8","Bit coin","ex ternal","Ġbrow sers","iche ver","or ian","Ġun b","Ġpo ke","ĠZ ot","M id","ĠPop ular","Ġco vert","Ġcont ributes","Ġ6 50","Ġcont ention","G ate","Ġcons oles","Ġchrom os","ĠI X","Ġvis ually","ĠE isen","Ġjewel ry","Ġdeleg ation","Ġacceler ate","ĠR iley","Ġsl ope","Ġind oor","it ially","Ġhuge ly","Ġtun nels","Ġfin ed","Ġdirect ive","Ġfore head","ustom ed","Ġsk ate","Mus ic","g as","Ġrecogn izing","am bo","Ġover weight","ĠGr ade","Ù Ĭ","Ġsound ing","Ġlock ing","ĠR EM","St ore","Ġexc av","ĠLike wise","ĠL ights","Ġel bow","ĠSupp ly","w ic","Ġhands ome","19 94","C oll","Ġadequ ately","ĠAssoci ate","Ġstri ps","Ġcrack down","Ġmar vel","ĠK un","Ġpass ages","@@ @@","ĠT all","Ġthought ful","names e","Ġprost itution","bus iness","Ġball istic","person al","c ig","iz ational","R ound","ĠÂłĠÂł ĠÂłĠÂł","ĠCole man","Ġadm itting","ĠPl ug","Ġbit coins","ĠSu z","Ġfair ness","Ġsupp lier","Ġcatast rophic","ĠHel en","o qu","M arc","ĠArt icles","g ie","Ġend angered","Ġdest iny","ĠVol t","ol ia","ax is","Ġche at","Ġun ified","IC O","qu ote","30 2","ĠS ed","Ġsupp ression","Ġanaly zing","Ġsqu at","Ġfig uring","Ġcoordin ates","Ġch unks","Ġ19 46","Ġsub p","Ġw iki","ĠFor bes","ĠJ upiter","ĠE rik","im er","ĠCom mercial","\\ )","Ġlegitim acy","Ġd ental","ĠMe an","Ġdefic its","5 50","Orig inally","ĠHor ror","Ġcontam ination","ll ah","Ġconf isc","ĠCl are","T B","ĠF ailed","an ed","Ġrul er","ĠCont roller","Ġfemin ists","F ix","g ay","20 7","Ġr abbit","Th ird","ownt own","Ġgl ue","Ġvol atile","Ġsh ining","Ġf oll","Ġimp aired","Ġsup ers","æ Ī","Ġcl utch","ļé ĨĴ","Ġpro let","Ġ( !","Ġy elled","ĠK iev","ĠEr n","ĠSh ock","K B","Ġsit uated","qu ery","ĠN as","Ġan nex","char acter","ĠHol iday","Ġautom ation","ĠJ ill","ĠRem astered","Ġl inem","Ġwild erness","ĠHor izon","ĠGu inea","A Z","Ġmain land","Ġsec recy","LE ASE","Ġp unk","ĠProv ince","( ),","Spe ed","Ġhand ing","ĠSeb ast","S ir","r ase","Ġj ournals","Ġcon gest","ĠT ut","ir rel","Ġschizophren ia","Ġmis ogyn","health y","I ron","Ġreact ed","- $","25 2","Ġpl ural","Ġpl um","Ġbarg ain","Ġground ed","f inder","Ġdis se","ĠL az","O OD","Ġat roc","F actory","Ġmin ions","Ġo ri","ĠB rave","ĠP RE","ĠMy anmar","ĠH od","Ġexped ition","Ġexpl ode","ĠCo ord","Ġext r","ĠB rief","ĠAD HD","Ġhard core","feed ing","Ġd ile","ĠF ruit","Ġvacc ination","ĠM ao","osp here","Ġcont ests","- |","Ġf ren","isp here","R om","ĠSh arp","ĠTre nd","Ġdis connect","âĢ¢ âĢ¢","Ġper secution","Ear th","Ġhealth ier","38 4","Ġc ob","ĠTr inity","OW S","AN N","Ġspecial ty","Ġg ru","Ġcooper ative","wh y","Start ing","ĠIss ues","st re","ens or","Ġ18 5","Ad v","! ?","ĠRe vel","em ia","ĠH ulk","Ġcelebr ations","ĠS ou","ra ud","ĠKle in","Ġun real","con text","Ġpartners hips","Ġadop ting","t ical","Ġspl ash","ĠHe zbollah","c ategory","cycl op","xt on","ĠD ot","urd y","t z","Ġenvelop e","ĠN L","â ķ","Ġwhere in","Spe c","18 4","Ġte lev","al iation","Ġmyth s","å °","Ġrig orous","Ġcommun icating","Ġobser ver","Ġre he","ĠW ash","Ġapolog ized","ĠT in","Ġexpend itures","work ers","d ocument","Ġhes itate","ĠLen in","Ġunpredict able","Ġrenew al","cl er","ok ia","ĠCON T","Ġpost season","Tok ens","Ġex acerb","Ġbet ting","Ġ14 7","Ġelev ation","W ood","ĠSol omon","19 4","00 4","out put","Ġredu nd","ĠM umbai","Ġp H","Ġreprodu ce","ĠD uration","MA X","Ġb og","C BS","ĠBal ance","ĠS gt","ĠRec ent","Ġc d","Ġpo pped","Ġincomp et","pro p","ay an","g uy","Pac ific","Ġty r","Ġ{ {","ĠMy stic","ĠD ana","Ġmast urb","Ġge ometry","à ¢","ĠCor rect","Ġtraject ory","Ġdistract ed","Ġf oo","ĠW elsh","L uc","m ith","Ġrug by","Ġrespir atory","Ġtri angle","Ġ2 15","Ġunder graduate","ĠSuper ior","ch anging","_ -","Ġright ly","Ġrefere e","Ġluc rative","Ġun authorized","Ġresemb les","ĠGN U","ĠDer by","Ġpath ways","ĠL ed","Ġend urance","Ġst int","Ġcollect or","F ast","Ġd ots","Ġnational s","ĠSec urities","Ġwh ip","Par am","Ġlearn s","M agic","Ġdetail ing","m oon","Ġbroadcast ing","Ġb aked","26 5","hol m","ĠS ah","ĠHus sein","ĠCourt esy","17 4","Ġ14 6","Ġge ographic","pe ace","Ġjud ging","ĠS tern","B ur","Ġstory line","G un","ĠSt ick","24 5","30 7","ãĤ´ ãĥ³","ĠAdminist rator","Ġbur nt","Ġp ave","ch oes","Ex ec","Ġcamp uses","Res ult","Ġmut ations","ĠCh arter","Ġcapt ures","Ġcomp ares","Ġbad ge","S cient","Ġer ad","ier y","o i","ett es","ĠE state","Ġst rap","Ġproud ly","Ġf ried","Ġwithd rawn","ĠV oy","ph ony","It ems","ĠP ierce","b ard","Ġann otation","ant on","ill on","Im pro","... )","Ġhapp ier","---- --","ad just","Ġstaff ers","Ġactiv ism","Ġper f","Ġal right","N eed","Ġcomm ence","Ġopio id","ĠAm anda","E s","ĠP ars","ĠK aw","W orks","24 8","Ġind o","t c","end ant","ĠM oto","Ġlegal ization","OT E","Ġtask ed","Ġt sp","ĠACT IONS","16 6","Ġrefres hing","ĠN R","ĠPere z","Ġinfring ement","S Y","List en","in ning","k u","Ġrot ate","pro gram","ar ah","Des ign","Ġ( £","Ġst oring","Ġwar rants","Ġjud gement","ĠB rist","us ually","ph oto","ĠR an","ĠP ine","Ġoutrage ous","ĠValent ine","lu ence","ĠEvery body","Al tern","Ġrele vance","Ġtermin ated","Ġd essert","Ġfulf illed","Ġprosecut ed","ĠW ords","Ġm igrant","Ġcultiv ation","ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ","idel ity","ĠV ern","ĠLog in","Ġmetaph or","ĠT ip","Ġrecru its","ĠP ig","rib ing","Ġenthusi asts","ex per","Ġfright ening","ĠH air","ans on","str ate","Ġh i","He ight","Ġown ing","n one","Ġdis like","Ġkn ives","pher d","Ġloud ly","ĠAP Is","Dis play","ĠL ac","ĠUS S","ab l","ver ages","J ew","Ġ17 2","ĠHist orical","at oon","ĠPhys ics","in tern","Ġwarm th","Ġto pp","D M","Ġgun man","Ġem peror","od i","ãĥ £","in atory","ĠR ib","Ġ13 1","ĠSat urn","ĠSh ining","Ġw aking","Qu otes","Ġcomed ian","en berg"," ½","Ġbelie vers","Ġpaper work","c ustom","Ġle v","Ġl ament","Ġpour ing","22 2","p olitical","ĠSupp lement","m aid","Ġcruel ty","Ġt read","ys ics","A w","rit es","Ġmod ifier","ĠP osition","Ad am","l b","ub s","Ġimper fect","Ġcl usters","ĠEngine er","ĠC herry","Ġinaug uration","ĠS au","Ġembod iment","ĠUn cle","Ġover r","Ġexplos ions","c ule","ĠPrinc eton","ĠAndre a","Ġincorrect ly","Ġearn est","Ġpil gr","ĠS print","Ġslee ve","Ġhe ars","ĠAm azing","Ġbrow sing","ag in","Ġhom eland","Ġha w","Ġd iving","ist ered","17 8","Ġbarg aining","ĠArc ade","Ġdeleg ate","ters on","................................ ................................","ĠJackson ville","27 5","Ġst agn","Ġad am","ĠSher man","C B","Ġsub urb","ĠFood s","Ġconver ting","ĠAr ist","Ġch ambers","l ove","Ġam ino","ĠG an","Ġmad ness","m c","ĠUS E","def ined","Ġul tr","ind ust","Ġw olves","l ance","Add itionally","Ġcr acks","as ia","ĠRe ason","ĠP ump","Ġaccident al","ĠL aser","ĠR id","Ġinitial ized","ell i","Ġun named","Ġn oun","ĠPass ed","Ġhost age","ĠEth iop","sh irts","Ġun rel","ĠEmb assy","Ġ19 41","Ġat oms","Ġpur ported","16 4","ĠF i","Ġgall ons","ĠMon ica","Ġp g","en ment","Ġsort ed","ĠG ospel","Ġhe ights","Ġtr aced","Ġunder going","She ll","Ġs acks","Ġproport ions","Ġhall uc","F ont","ac et","Ġwar mer","ĠIN TER","Ġgrab bing","Pl ug","Ġreal ization","ĠBur ke","Ġen chant","AT ER","ĠSe ed","Ġabund ant","F M","Ġc ivic","V s","is i","Ġv ow","Ġre per","ĠPartners hip","Ġpenet ration","Ġax e","Ġsh attered","ĠZ ombies","Ġv inyl","ĠAl ert","e on","Ġoblig ed","ĠIll ust","ĠPl aza","ĠFront ier","Ġdavid jl","ĠSer ial","ĠH av","ĠNut rition","B i","Ġâĸ Ī","ĠJ ays","lin ux","Ġhur ry","Ġv oy","Ġhop eless","ĠSte alth","Ġ ãģ","ess ors","tt le","b org","ĠSaf ari","f ell","Ġw ary","d ue","ĠAb ove","H a","E LL","Ġnot or","ĠW on","T oo","Ġoccup ations","Ġposs essions","Ġinv iting","Ġpred ators","Ġacceler ated","Ġ15 7","uter te","ĠC ube","e ast","acc ount","G ive","Ġtrans plant","red ients","id able","Ġscreens hots","ĠG und","ĠF S","Ġtravel ers","Ġsens ory","ĠF iat","ĠRock ets","İ ĭ","_ {","F riend","Ġchar ming","AL S","Ġenjoy ment","m ph","Ġ5 000","ĠRE G","Ù Ĩ","b ia","Ġcomp ilation","ro st","ĠV P","ĠSch ne","201 9","Ġcop ying","M ORE","ĠFl ore","f alls","2 15","t otal","Ġdis ciples","d ouble","Ġexceed ing","Ġsm ashed","Ġconcept ual","ĠRom ania","ĠB rent","ĠI CE","ĠT ou","Ġg rap","Ġn ails","18 9","ãĥ ĺ","Ġproc ure","e ur","Ġconfir ming","ĠC ec","aw i","ĠEd en","Ġn g","Ġengine ered","at ics","Ġhook ed","Ġdisgust ing","ĠMur der","ãĤ ¿","L ibrary","Ġ16 8","Al most","hem atic","Men u","ĠNot re","ĠJ ur","Ġkidn apped","Ġhack er","ĠJ ade","Ġcreep y","Ġdraw ings","ĠSpons or","Ġcycl ists","ĠGob lin","Ġoptim ized","Ġst aged","ĠMc D","bet ween","A ge","en o","S ex","ĠW ide","n ings","av is","Ġincap able","ĠK ob","Ġreward ing","ĠL one","oles cent","Ġcontract ed","Ġstick y","J ose","B all","f est","ĠIn put","ĠRec ently","Ġto mat","squ are","App lication","Ġnit rogen","Ġdupl icate","ĠRec on","ĠD ear","L ondon","Ġint ra","Ġd ock","Ġout reach","ĠM illion","Ġmamm als","am pton","V AL","Ġsn aps","Ġd os","ĠWh ole","ĠRead y","T ry","ĠWinn ipeg","ear ance","Ġinc urred","ren ched","ĠNS W","il ot","rain e","Ġc ube","g ot","Ġrun way","etermin ed","ĠHaw ks","Ġsurviv or","ĠW ish","ĠD in","ĠDE F","ĠV ault","18 7","Ġmush rooms","Ġcris p","be y","ĠDisco very","Ġdevelopment al","Ġparad igm","Ġcha otic","ĠT su","Ġ3 33","b ons","Ġbacter ial","Ġcomm its","Ġcos mic","Ġme ga","oc ative","ĠP aint","ophob ic","Ġv ain","Ġcar ved","ĠTh ief","ĠG ul","ows hip","Ġc ites","ĠEd inburgh","Ġdimin ished","Ġacknowled ges","ĠK ills","Ġmic row","ĠHer a","Ġsen iors","Ġwhere by","H op","at ron","Ġun available","ĠN ate","Ġ4 80","Ġsl ated","ĠRe becca","ĠB attery","Ġgram mar","Ġhead set","Ġcurs or","Ġex cluding","any e","aunder ing","eb in","Ġfeas ible","ĠPub lishing","ĠLab s","ĠCl iff","ĠFerr ari","Ġp ac","vis ible","mark ed","pe ll","Ġpol ite","Ġstagger ing","ĠGal actic","Ġsuper st","Ġpar an","ĠOffic ers","ãĢ ģ","Ġspecific s","ul us","23 9","ĠP aste","AM P","ĠPan ama","ĠDe lete","angu ard","rest rial","Ġhero ic","ĠD y","ا ÙĦ","Ġincumb ent","Ġcr unch","t ro","Ġsc oop","Ġblog ger","Ġsell ers","ure n","Ġmedic ines","ĠC aps","ĠAnim ation","ox y","Ġout ward","Ġinqu iries","22 9","Ġpsych ologist","ĠS ask","ev il","Ġcontam inated","ãĤ ¨","he rence","Ġbrand ed","ĠAbd ul","z h","Ġparagraph s","Ġmin s","Ġcor related","er b","Ġimp art","Ġmil estone","ĠSol utions","ot le","Ġunder cover","Ġmar ched","ĠCharg ers","f ax","ĠSec rets","Ġr uth","we ather","Ġfemin ine","Ġsh am","Ġprest igious","igg ins","Ġs ung","hist ory","ett le","gg ie","Ġout dated","ol and","Ġper ceptions","ĠS ession","ĠDod gers","u j","ĠE ND","D oc","Ġdefic iency","Gr and","ĠJ oker","Ġretro spect","Ġdiagn ostic","Ġharm less","Ġro gue","ĠA val","E qu","Ġtrans c","ĠRoberts on","ĠDep ending","ĠBurn s","iv o","Ġhost ility","F eatures","ĵ ĺ","Ġdis comfort","ĠL CD","spec ified","ĠEx pect","3 40","Ġimper ative","ĠReg ular","Ch inese","Ġstate wide","Ġsy mm","Ġlo ops","Ġaut umn","N ick","Ġsh aping","Ġqu ot","Ġc herry","ĠCross ref","è¦ ļéĨĴ","Stand ard","he ed","ĠD ell","ĠViet namese","Ġo st","ĠV alkyrie","O A","Ass ad","Ġreb ound","ĠTra ffic","pl aces","æ ĺ","ĠB uc","17 2","Ġshel ters","Ġins isting","ĠCertain ly","ĠKenn eth","ĠT CP","Ġpen al","ĠRe play","he ard","Ġdial ect","iz a","ĠF Y","it cher","ĠD L","Ġspir al","Ġquarterback s","Ġh ull","Ġgo ogle","Ġto dd","ĠSter ling","ĠPl ate","Ġsp ying","mb ol","ĠReal m","ĠPro ced","ĠCr ash","Ġtermin ate","Ġprotest ing","C enter","gu ided","Ġun cover","Ġboy cott","Ġreal izes","s ound","Ġpret ending","ĠV as","19 80","Ġfram ed","Ġ13 9","Ġdesc ended","Ġrehab ilitation","Ġborrow ing","ĠB uch","Ġbl ur","R on","ĠFro zen","en za","Ch ief","ĠP oor","Ġtransl ates","M IN","Ġ2 12","J ECT","Ġerupt ed","Ġsuccess es","S EC","Ġpl ague","Ġg ems","d oms","Ġstret ches","ĠSp y","Ġstory telling","C redit","ĠP ush","Ġtra ction","Ġin effective","ĠL una","Ġt apes","Ġanaly tics","erc ise","Ġprogram mes","ĠCar bon","Ġbeh old","he avy","ĠConserv ation","ĠF IR","Ġs ack","ter min","ric ks","Ġhous ed","Ġunus ually","I ce","Ġexecut ing","ĠMor oc","ed ay","Ġed itions","Ġsm arter","ĠB A","Ġout law","Ġvan ished","ib a","AL SE","ĠSil va","23 8","C ould","Ġphilos opher","Ġevac uated","Sec ret","14 2","Ġvis as","ãĤ ¬","ĠM alt","ĠClear ly","ĠN iger","ĠC airo","ĠF ist","3 80","ĠX ML","aut o","it ant","Ġrein forced","Rec ord","ĠSurviv or","G Hz","Ġscrew s","parent s","Ġo ceans","ma res","Ġbra kes","vas ive","Ġhell o","ĠS IM","rim p","Ġo re","ĠArm our","24 7","Ġterr ific","Ġt ones","14 1","ĠMin utes","Ep isode","Ġcur ves","Ġinflamm atory","Ġbat ting","ĠBeaut iful","L ay","Ġunp op","v able","Ġr iots","ĠTact ics","b augh","ĠC ock","Ġorg asm","ĠS as","Ġconstruct or","et z","G ov","Ġant agon","Ġthe at","Ġde eds","ha o","c uts","ĠMc Cl","Ġu m","ĠScient ists","Ġgrass roots","ys sey","\"] =>","Ġsurf aced","Ġsh ades","Ġneighb ours","Ġad vertis","oy a","Ġmer ged","Up on","Ġg ad","Ġanticip ate","Any way","Ġsl ogan","Ġdis respect","I ran","ĠT B","act ed","Ġsubp oen","medi ately","OO OO","Ġwa iver","Ġvulner abilities","ott esville","ĠHuff ington","J osh","ĠD H","M onday","ĠEll en","K now","x on","it ems","22 8","Ġf ills","ĠN ike","Ġcum ulative","and als","I r","Ġ ì","Ġfr iction","ig ator","Ġsc ans","ĠVi enna","ld om","Ġperform ers","P rim","Ġb idding","M ur","Ġlean ed","ĠPri x","al ks","Ġ[ â̦]","ĠTw itch","ĠDevelop er","ĠG ir","Ġcall back","Ab stract","Ġacc ustomed","Ġfreed oms","ĠP G","ur acy","Ġl ump","is man",",, ,,","19 92","ĠR ED","Ġwor m","M atch","ĠPl atinum","I J","ĠOwn er","Tri via","com pl","Ġnew born","Ġfant as","O wn","Ġ19 59","Ġsymp ath","Ġub iqu","Ġoutput s","Ġal lev","Ġpr ag","K evin","Ġfav ors","Ġbur ial","Ġn urt","so lete","c ache","Ġ15 6","Ġunl ocks","te chn","M aking","Ġcon quer","ad ic","æ ĸ","Ġel f","Ġelect orate","ĠKurd s","ĠSt ack","ĠSam urai","Ġâ ĺħ","Ġ{ }","ĠS aid","ĠFall out","Ġkind ness","ĠCustom s","ĠBou levard","Ġhelicop ters","ot ics","ĠVe get","com ment","Ġcritic ised","Ġpol ished","ĠRem ix","ĠC ultural","Ġrec ons","Ġdo i","at em","Sc reen","Ġbar red","Com ments","ĠGener ally","Ġsl ap","7 20","V ari","p ine","Ġem pt","Ġh ats","ĠPlay ing","l ab","a verage","form s","ĠC otton","Ġcan s","ĠD ON","ĠSom alia","C rypt","ĠIncre ases","E ver","mod ern","Ġsur geon","3 000","Ġrandom ized","================================ ================================","B ern","im pl","ĠC OR","Ġpro claim","th ouse","Ġto es","Ġam ple","Ġpres erving","Ġdis bel","gr and","B esides","Ġsil k","ĠPat tern","h m","Ġenter prises","Ġaffidav it","ĠAdvis ory","Ġadvert ised","ĠRel igious","se ctions","psy ch","ĠField s","aw ays","Ġhasht ag","ĠNight mare","Ġv ampire","Ġfore nsic","rosso ver","n ar","Ġn avy","Ġvac ant","ĠD uel","Ġhall way","Ġface book","ident ally","ĠN RA","Ġm att","Ġhur ricane","ĠKir by","ĠP uzzle","Ġsk irt","ou st","du llah","Ġanal ogy","in ion","Ġtomat oes","ĠN V","ĠPe ak","ĠMe yer","Ġappoint ments","Ġm asc","Ġal ley","re hend","Ġchar ities","Ġund o","Ġdest inations","ĠTest ing","\"> \"","c ats","* .","Ġgest ures","gener al","Le ague","Ġpack ets","ĠInspect or","ĠBer g","Ġfraud ulent","Ġcritic ize","F un","Ġbl aming","nd ra","Ġsl ash","ĠE ston","Ġpropos ing","Ġwh ales","Ġtherap ist","Ġsub set","Ġle isure","EL D","ĠC VE","ĠAct ivity","Ġcul min","sh op","ĠD AY","is cher","ĠAdmir al","ĠAtt acks","Ġ19 58","Ġmem oir","Ġfold ed","Ġsex ist","Ġ15 3","ĠL I","Ġread ings","Ġembarrass ment","ĠEmploy ment","w art","ch in","Ġcontin uation","l ia","Rec ently","Ġd uel","Ġevac uation","ĠKash mir","Ġdis position","ĠR ig","Ġbol ts","Ġins urers","4 67","M ex","Ġret aliation","Ġmis ery","Ġunre asonable","r aining","I mm","ĠP U","em er","Ġgen ital","ãĤ ³","ĠC andy","Ġon ions","ĠP att","lin er","Ġconced ed","Ġf a","Ġfor c","ĠH ernandez","ĠGe off","deb ian","ĠTe ams","Ġc ries","Ġhome owners","23 7","A BC","Ġst itch","Ġstat istic","Ġhead ers","ĠBi ology","Ġmot ors","ĠG EN","ĠL ip","Ġh ates","Ġhe el","S elf","i pl","ED IT","ort ing","Ġann ot","ĠSpe ech","old emort","ĠJ avascript","ĠLe Bron","Ġfoot print","Ġf n","Ġseiz ures","n as","h ide","Ġ19 54","ĠBe e","ĠDecl aration","ĠKat ie","Ġreserv ations","N R","f emale","Ġsatur ated","Ġb iblical","Ġtroll s","Dev ice","ph otos","Ġdr ums","ãĥīãĥ© ãĤ´ãĥ³","N ight","f ighter","ĠH ak","ri ber","Ġc ush","Ġdiscipl inary","ba um","ĠG H","ĠSch midt","ilib rium","Ġs ixty","ĠKush ner","ro ts","Ġp und","ĠR ac","Ġspr ings","Ġcon ve","Bus iness","F all","Ġqual ifications","Ġvers es","Ġnarc iss","ĠK oh","ĠW ow","ĠCharl ottesville","ed o","Ġinterrog ation","ĠW ool","36 5","B rian","Ġâľ ĵ","Ġalleg es","ond s","id ation","ĠJack ie","y u","Ġl akes","Ġworth while","Ġcryst als","ĠJud a","Ġcomp rehend","Ġfl ush","Ġabsor ption","ĠO C","Ġfright ened","ĠCh ocolate","Mart in","Ġbu ys","Ġbu cks","Ġapp ell","ĠChampions hips","Ġlist ener","ĠDef ensive","Ġc z","ud s","ĠM ate","Ġre play","Ġdecor ated","Ġs unk","ĠV IP","ĠAn k","Ġ19 5","aa aa","Nob ody","ĠMil k","ĠG ur","ĠM k","ĠS ara","Ġse ating","ĠW id","Tr ack","Ġemploy s","Ġgig antic","AP P","ãĤ §","in ventory","Ġtow el","at che","l asting","ĠT L","Ġlat ency","Ġkn e","B er","me aning","Ġup held","Ġplay ground","Ġm ant","S ide","Ġstere o","Ġnorth west","Ġexception ally","Ġr ays","Ġrec urring","D rive","Ġup right","Ġab duct","ĠMar athon","Ġgood bye","Ġal phabet","h p","Ġcourt room","ring ton","ot hing","T ag","Ġdiplom ats","Ġbar bar","ĠAqu a","18 3","33 33","Ġmat urity","Ġinst ability","ĠAp ache","Ġ= ==","Ġfast ing","ĠGr id","Mod Loader","Ġ15 2","A bs","ĠOper ating","ett i","Ġacqu aint","Don nell","ĠK em","ĠFor ge","Ġarm ored","M il","Ġphilos ophers","in vest","Pl ayers","â Ī","Ġmy riad","Ġcomr ades","R ot","Ġremember ing","Ġcorrespond s","Ġprogram mers","ĠLyn n","Ġo lig","Ġco herent","yn chron","ĠChem ical","Ġj ugg","p air","post s","E ye","ĠIn ner","Ġsem ester","ott est","ĠEmir ates","ric anes","or ously","m its","ĠW is","Ġd odge","l ocation","Ġf aded","Am azon","ĠPro ceed","ĠIN FO","j ournal","ĠTru ck","T en","Ġ2 17","Ġstat utes","m obile","ĠT ypes","Rec omm","b uster","pe x","Ġleg ends","Ġhead ache","f aced","ĠWi Fi","if ty","ĠH ER","Ġcirc uits","ER ROR","22 6","ol in","Ġcyl inder","osp ace","ik ers","P rem","Qu ant","Ġconflic ting","Ġslight est","Ġfor ged","ion age","Step hen","ĠK ub","ĠOpp ortun","ĠHe al","Ġbl o","Ġrul ers","Ġh uh","Ġsubmar ine","f y","ass er","Ġallow ance","ĠKas ich","ĠT as","ĠAustral ians","Forge ModLoader","ĠâĨ ij","ĠMat rix","am ins","Ġ12 00","ĠAc qu","23 6","D ocument","ĠBre aking","19 3","ĠSub st","ĠRoll er","ĠPro perties","ĠN I","t ier","Ġcr ushing","Ġadvoc ating","Further more","keep ers","Ġsex ism","x d","Ġcall er","ĠS ense","chie ve","ĠT F","Ġfuel ed","Ġreminis cent","Ġobs ess","ur st","Ġup hold","ĠF ans","het ics","Ġâ Ĺ","ĠB ath","Ġbe verage","Ġo scill","25 4","Ġpol es","Ġgrad ual","Ġex ting","ĠS uff","ĠS uddenly","Ġlik ing","Ġ19 49","un ciation","am ination","ĠO mar","ĠL V","ĠCon sequently","Ġsynt hes","ĠG IF","Ġp ains","Ġinteract ing","u ously","inc re","Ġrum or","ĠScient ology","19 7","ĠZ ig","Ġspe lling","ĠA SS","Ġexting u","ms on","Ġg h","Ġremark ed","ĠStrateg ic","ĠM ON","å ¥","g ae","ĠWH AT","E ric","ĠCamp us","Ġmeth ane","Ġimag in","J UST","ĠAl m","X T","i q","ĠR SS","Ġwrong doing","att a","Ġbig ot","Ġdemonstr ators","ĠCal vin","ĠV illa","Ġmembr ane","ĠAw esome","Ġbenef ic","26 8","Ġmagn ificent","ĠL ots","G reg","ĠBor is","Ġdetain ees","ĠH erman","Ġwhis pered","Ġa we","Prof essor","fund ing","Ġphys iological","ĠDest ruction","Ġlim b","Ġmanip ulated","Ġbub bles","Ġpse ud","Ġhyd ra","ĠBrist ol","Ġst ellar","ĠExp ansion","ĠK ell","ĠInterest ingly","Ġm ans","Ġdrag ging","Ġec ological","ĠF it","Ġg ent","Ġbenef ited","ĠHait i","Ġpoly g","ãĥ İ","Ġ20 30","Ġpro w","Ġrecon struction","Ġwas t","Ġpsych ic","ĠGree ks","Hand ler","16 2","ĠP ulse","Ġsol icit","Ġsy s","Ġinflu x","ĠG entle","per cent","Ġprolifer ation","Ġtax able","Ġdisreg ard","Ġesc aping","Ġg inger","Ġwith stand","Ġdevast ated","ĠD ew","ser ies","Ġinject ed","ela ide","Ġturn over","he at","Ļ Ĥ","H appy","ĠSil ent","ãĤ Ń","iv ism","Ġir rational","AM A","Ġre ef","r ub","Ġ16 2","Ġbank ers","ĠEth ics","v v","Ġcritic isms","K n","18 6","M ovie","ĠT ories","Ġno od","Ġdist ortion","F alse","od ore","Ġt asty","Res earch","ĠU ID","- )","Ġdivor ced","ĠM U","ĠHay es","ĠIs n","ian i","ĠH Q","Ġ\" #","ign ant","Ġtra umatic","ĠL ing","H un","Ġsab ot","on line","r andom","Ġren amed","ra red","K A","d ead","é t","ĠAss istance","Ġse af","++++ ++++","Ġse ldom","ĠWeb b","Ġbo olean","u let","Ġref rain","ĠDI Y","ru le","Ġshut ting","Ġutil izing","load ing","ĠPar am","co al","oot er","Ġattract ing","ĠD ol","Ġher s","ag netic","ĠRe ach","im o","Ġdisc arded","ĠP ip","01 5","ü r","Ġm ug","Im agine","C OL","Ġcurs ed","ĠSh ows","ĠCurt is","ĠSach s","spe aking","ĠV ista","ĠFram ework","ong o","Ġsub reddit","Ġcr us","ĠO val","R ow","g rowing","Ġinstall ment","Ġgl ac","ĠAdv ance","EC K","ĠLGBT Q","LE Y","Ġac et","Ġsuccess ive","ĠNic ole","Ġ19 57","Qu ote","Ġcircumst ance","ack ets","Ġ14 2","ort ium","Ġguess ed","ĠFr ame","Ġperpet rators","ĠAv iation","ĠBen ch","Ġhand c","A p","Ġ19 56","25 9","r and","Net Message","d in","urt les","h ig","ĠV III","ff iti","ĠSw ords","b ial","Ġkidn apping","dev ice","Ġb arn","ĠEl i","auc as","S end","Con structed","Ġ ½","Ġneed les","Ġad vertisements","Ġv ou","Ġexhib ited","ĠFort ress","As k","B erry","TY PE","Ġcan cers","ump ing","ĠTerrit ory","Ġpr ud","Ġn as","Ġathe ist","Ġbal ances","ãģ Ł","ĠSh awn","& &","Ġland sc","ĠR GB","Ġpet ty","Ġex cellence","Ġtransl ations","Ġpar cel","ĠChe v","E ast","ĠOut put","im i","Ġamb ient","ĠTh reat","Ġvill ains","Ġ5 50","IC A","Ġtall er","Ġle aking","c up","Ġpol ish","Ġinfect ious","ĠK C","Ġ@ @","back ground","Ġbureaucr acy","ĠS ai","un less","it ious","ĠSky pe","At l","ID ENT","00 8","Ġhyp ocr","Ġpit chers","Ġguess ing","ĠF INAL","Bet ween","Ġvill agers","Ġ25 2","f ashion","ĠTun is","Be h","ĠEx c","ĠM ID","28 8","ĠHas kell","19 6","ĠN OR","Ġspec s","Ġinv ari","Ġgl ut","ĠC ars","Ġimp ulse","Ġhon ors","g el","Ġjurisd ictions","ĠBund le","ul as","Calif ornia","ĠIncre ase","Ġp ear","Ġsing les","Ġc ues","Ġunder went","ĠW S","Ġexagger ated","Ġdub ious","Ġfl ashing","L OG",") ].","J ournal","t g","V an","ĠI stanbul","ĠIn sp","ĠFrank en","D raw","Ġsad ness","Ġiron ic","ĠF ry","x c","Ġ16 4","is ch","W ay","ĠProtest ant","h orn","Ġun aff","ĠV iv","ill as","ĠProduct ions","ĠH ogan","Ġper imeter","ĠS isters","Ġspont aneous","Ġdown side","Ġdescend ants","Ġor n","w orm","Japan ese","Ġ19 55","Ġ15 1","ĠDo ing","els en","umb les","Ġrad ically","ĠDr um","ĠB ach","Ġli abilities","ĠO B","ĠElement ary","Ġmem e","yn es","Ġfinger print","ĠGr ab","Ġundert ake","Mem bers","ĠRead er","ĠSim s","g od","Ġhypot hetical","s cient","ĠA J","Ġchar ism","Ġad missions","ĠMiss ile","tr ade","Ġexerc ising","ĠBack ground","W ritten","Ġvoc als","whe ther","Ġv i","ĠW inner","Ġl itter","ĠSh ooting","ST EM","ãĤ ¡","ĠA FL","Ġvari ability","Ġe ats","ĠD PS","b row","Ġeleph ants","Ġstr at","Ġ Å","Ġsett lers","Matt hew","Ġin advert","H I","ĠIM F","ĠGo al","Ġnerv es","John son","ey e","ablish ment","Th ursday","BIL ITY","H ad","am oto","het amine","ep s","Ġmit ochond","Ġcomp ressed","ĠTre vor","ĠAnim als","T ool","L ock","Ġtwe ak","Ġpin ch","Ġcancell ation","P ot","Ġfoc al","ĠAst ron","17 3","ĠA SC","ĠO THER","umn i","Ġdem ise","d l","Ù ħ","Sem itism","Ġcr acking","Ġcollabor ative","Ġexpl ores","s ql","Ġher bs","Ġconfig urations","m is","ĠRes ult","ace y","ĠSm oke","Ġsan ct","el ia","Ġdeg ener","Ġdeep est","Ġscream ed","Ġn ap","Soft ware","ĠST AR","E F","ĠX in","spons ored","mans hip","23 3","Ġprim aries","Ġfilter ing","Ġas semble","m il","ĠMy ers","b ows","Ġpun ched","M ic","Ġinnov ations","Ġfun c","and o","Ġfr acking","ĠV ul","о Ð","osh op","ĠIm mun","Ġsett ling","Ġadolesc ents","Ġreb uilding","Ġtransform ing","Ġpar ole","Ġhar bor","Ġbook ing","ot ional","onge vity","ĠY o","b ug","Ġemer ges","ĠMethod s","ĠCh u","P res","ĠDun geons","Ġtra iling","ĠR um","ĠH ugh","å¤ ©","ĠE ra","ĠBatt les","Res ults","ĠTr ading","Ġvers a","c ss","ax ies","he et","Ġgre ed","19 89","Ġgard ens","Ġconting ent","P ark","ĠLeaf s","h ook","ro be","Ġdiplom acy","ĠF uel","ĠInv asion","Ġupgr ading","M ale","Ġe lic","Ġrelent less","ĠCo venant","ap esh","ĠT rop","T y","pro duction","art y","Ġpun ches","ak o","cyclop edia","ĠR abbit","ĠHD MI","Ġ14 1","Ġf oil","Item Image","ĠF G","Ġimplement ations","ĠP om","ixt ures","Ġaw ait","Ġ3 30","am us","Ġumb rella","Ġfore see","se par","Ġcircum cision","Ġperipher al","S ay","ĠExper t","In c","Ġwithd rew","ĠAnd ers","f ried","Ġradio active","ĠOp ening","Ġboard ing","ĠN D","Ġover throw","Act iv","W P","ĠAct s","× Ļ","Ġmot ions","v ic","ĠM ighty","ĠDef ender","a er","Ġthank ful","ĠK illing","ĠBr is","mo il","Ġpredict ing","26 6","ch oice","Ġkill ers","Ġinc ub","ĠChe st","ather ing","Ġpro claimed","fl ower","oss om","umbled ore","ĠCy cling","ĠOccup y","AG ES","P en","ĠY ug","Ġpack aged","Ġheight ened","c ot","st ack","C ond","Ġst amps","m age","Ġpersu aded","Ġens l","ĠCard inal","Ġsol itary","Ġpossess ing","ĠC ork","Ġev id","ĠT ay","Ġbl ues","Ġextrem ism","Ġlun ar","Ġcl own","Te chn","Ġfest ivals","ĠPv P","ĠL ar","Ġconsequ ently","p resent","Ġsom eday","ç İĭ","ĠMet eor","Ġtour ing","c ulture","Ġbe aches","S hip","c ause","ĠFl ood","ãĥ ¯","Ġpur ity","th ose","Ġem ission","b olt","Ġch ord","ĠScript ure","L u","Ġ$ {","cre ated","Other s","25 8","Ġelement al","Ġannoy ed","ĠA E","d an","ĠS ag","Res earchers","Ġfair y","âĢĵ âĢĵ","======== ====","Sm art","GG GG","Ġskelet ons","Ġpup ils","link ed","Ġur gency","en abled","ĠF uck","Ġcoun cill","r ab","U AL","T I","Ġlif es","Ġconf essed","B ug","Ġharm on","ĠCON FIG","ĠNe utral","D ouble","Ġst aple","ĠSH A","Brit ish","ĠSN P","AT OR","oc o","Ġswing ing","ge x","ole on","pl ain","ĠMiss ing","ĠTro phy","v ari","ran ch","Ġ3 01","4 40","00000000 00000000","Ġrest oring","Ġha ul","uc ing","ner g","Ġfut ures","Ġstrateg ist","quest ion","Ġlater al","ĠB ard","Ġs or","ĠRhod es","ĠD owntown","????? -","ĠL it","ĠB ened","Ġco il","st reet","ĠPort al","FI LE","ĠG ru","* ,","23 1","ne um","Ġsuck ed","Ġr apper","Ġtend encies","ĠLaure n","cell aneous","26 7","Ġbrow se","Ġover c","head er","o ise","Ġbe et","ĠG le","St ay","Ġm um","Ġtyp ed","Ġdiscount s","T alk","ĠO g","ex isting","ĠS ell","u ph","C I","ĠAust rian","ĠW arm","Ġdismiss al","Ġaver ages","c amera","Ġalleg iance","L AN","=\" #","Ġcomment ators","ĠSet ting","ĠMid west","Ġpharm ac","ĠEX P","Ġstain less","Ch icago","Ġt an","24 4","Ġcountry side","ĠV ac","29 5","Ġpin ned","Ġcr ises","Ġstandard ized","T ask","ĠJ ail","ĠD ocker","col ored","f orth","\" },","Ġpat rons","Ġsp ice","Ġm ourn","ĠM ood","Ġlaund ry","Ġequ ip","ĠM ole","y ll","ĠTH C","n ation","ĠSher lock","Ġiss u","ĠK re","ĠAmeric as","ĠA AA","Ġsystem atically","Ġcont ra","ĠS ally","Ġrational e","Ġcar riage","Ġpe aks","Ġcontrad iction","ens ation","ĠFail ure","Ġpro ps","Ġnames pace","Ġc ove","field s","ãĤ ĭ","Ġw ool","ĠC atch","Ġpresum ed","ĠD iana","r agon","ig i","Ġh amm","Ġst unt","ĠG UI","ĠObserv atory","ĠSh ore","Ġsmell s","ann ah","Ġcock pit","ĠD uterte","8 50","Ġopp ressed","bre aker","ĠCont ribut","ĠPer u","ĠMons anto","ĠAtt empt","Ġcommand ing","Ġfr idge","ĠR in","ĠChe ss","ual ity","Ġo l","Republic an","ĠGl ory","ĠW IN",".... ...","ag ent","read ing","Ġin h","J ones","Ġcl icks","al an","Ġ[ ];","ĠMaj esty","ĠC ed","op us","ate l","à ª","AR C","ĠEc uador","ãĥ ł","ĠK uro","Ġritual s","Ġcapt ive","Ġoun ce","Ġdisag reement","Ġsl og","f uel","P et","M ail","Ġexerc ised","Ġsol ic","Ġrain fall","Ġdev otion","ĠAss essment","Ġrob otic","opt ions","ĠR P","ĠFam ilies","ĠFl ames","Ġassign ments","00 7","aked own","Ġvoc abulary","Re illy","Ġc aval","g ars","Ġsupp ressed","ĠS ET","ĠJohn s","Ġwar p","bro ken","Ġstat ues","Ġadvoc ated","Ġ2 75","Ġper il","om orph","ĠF emin","per fect","Ġh atch","L ib","5 12","Ġlif elong","3 13","Ġche eks","Ġnum bered","ĠM ug","B ody","ra vel","We ight","ĠJ ak","ĠHe ath","Ġkiss ing","ĠJ UST","Ġw aving","u pload","Ġins ider","ĠPro gressive","ĠFil ter","tt a","ĠBe am","Ġviol ently","ip ation","Ġskept icism","Ġ19 18","ĠAnn ie","ĠS I","Ġgen etics","Ġon board","at l","ĠFried man","ĠB ri","cept ive","Ġpir ate","ĠRep orter","27 8","Ġmyth ology","Ġe clipse","Ġsk ins","Ġgly ph","ing ham","F iles","C our","w omen","Ġreg imes","Ġphotograp hed","K at","ĠMA X","Offic ials","Ġunexpected ly","Ġimpress ions","F ront",";;;; ;;;;","Ġsuprem acy","Ġs ang","Ġaggrav ated","Ġabrupt ly","ĠS ector","Ġexc uses","Ġcost ing","ide press","St ack","ĠR NA","ob il","Ġghost s","ld on","at ibility","Top ics","Ġreim burse","ĠH M","ĠDe g","Ġth ief","y et","ogen esis","le aning","ĠK ol","ĠB asketball","Ġf i","ĠSee ing","Ġrecy cling","Ġ[ -","Cong ress","Ġlect ures","P sy","Ġne p","Ġm aid","Ġori ented","A X","Ġrespect ful","re ne","fl ush","ĠUn loaded","re quest","gr id","ĠAltern atively","ĠHug o","Ġdec ree","ĠBuddh ism","and um","And roid","ĠCong o","ĠJoy ce","Ġacknowled ging","hes ive","ĠTom orrow","ĠH iro","th ren","ĠM aced","Ġho ax","ĠIncre ased","ĠPr adesh","W ild","____ __","16 1","Ġa unt","Ġdistribut ing","ĠT ucker","ĠSS L","ĠW olves","B uilding","ou lt","ĠLu o","ĠY as","ĠSp ir","ĠSh ape","ĠCamb od","ĠIP v","Ġm l","Ġext rad","39 0","ĠPenn y","d ream","Ġstation ed","opt ional","ew orthy",". ","ĠWorks hop","ĠRet ail","ĠAv atar","6 25","N a","ĠV C","ĠSec ure","M Y","19 88","oss ip","Ġpro state","Ġund en","Ġg amer","ĠCont ents","ĠWar hammer","ĠSent inel","3 10","Ġse gregation","ĠF lex","ĠM AY","Ġdr ills","ĠDrug s","Islam ic","Ġsp ur","Ġca fe","Ġimag inary","Ġgu iding","Ġsw ings","ĠThe me","ob y","Ġn ud","Ġbe gging","Ġstr ongh","Ġreject ing","Ġpedest rians","ĠPro spect","R are","s le","Ġconcess ions","ĠConst itutional","Ġbe ams","Ġfib ers","p oon","Ġinstinct s","pro perty","ĠB IG","Sand ers","im ates","Ġco ating","Ġcorps es","ĠTR UE","check ed","Ġ16 6","A sh","ĠJ S","ĠF iction","Ġcommun al","Ġener getic","oooo oooo","Ġnow adays","IL D","ib o","ĠSU V","R en","Ġdwell ing","Sil ver","Ġt ally","ĠM oving","Ġcow ard","Ġgener als","Ġhorn s","Ġcirc ulated","Ġrob bed","ĠUn limited","Ġharass ed","Ġinhib it","Ġcomp oser","ĠSpot ify","Ġspread s","3 64","Ġsu icidal","Ġno ises","ĠSt ur","Ġs aga","ĠK ag","is o","Ġtheoret ically","M oney","Ġsimilar ity","Ġslic ed","ut ils","ing es","\" -","Ġan th","Ġimp ed","Mod ule","Through out","Ġmen us","comm ittee","and i","ob j","in av","f ired","ĠAb dullah","Ġund ead","Ġfont s","H old","EN G","Ġsustain ability","Ġfl ick","Ġr azor","ĠF est","ĠChar acters","Ġword ing","Ġpopul ist","Ġcritic izing","Ġm use","v ine","Ġcard board","Ġkind ly","Ġfr inge","ĠThe ft","icult ural","Ġgovern ors","Ġ ����","Ġ16 3","Ġtime out","ĠA uth","Child ren","A U","Ġred emption","ĠAl ger","Ġ19 14","Ġw aved","Ġastron auts","og rams","Ġsw amp","ĠFinn ish","Ġcand le","Ġton nes","ut m","Ġr ay","Ġsp un","Ġfear ful","art icles","Ġca us","or ically","ĠRequ ires","ĠG ol","Ġpop e","Ġinaug ural","Ġg le","AD A","ĠIS IL","ĠOff ensive","Ġwatch dog","Ġbal con","ent ity","ĠH oo","Ġgall on","AC C","Ġdoub ling","Ġimpl ication","ĠS ight","Ġdoct r","---- ---","Ġ\\ \\","Ġm alt","R oll","Ġâī ¥","Ġrec ap","add ing","u ces","ĠB end","fig ure","Ġtur key","Ġsoc ietal","ĠT ickets","Ġcommer cially","Ġsp icy","Ġ2 16","ĠR amp","Ġsuperior ity","à ¯","ĠTr acker","C arl","ĠC oy","ĠPatri ot","Ġconsult ed","Ġlist ings","Ġsle w","reens hot","ĠG one","Ġ[ ...]","30 9","Ġh ottest","Ø ±","Ġrock y","ĠD iaz","Ġmass age","Ġpar aly","Ġp ony","A z","Ġcart ridge","ĠN Z","Ġsn ack","ĠLam ar","ple ment","ĠLes lie","Ġm ater","Ġsn ipp","24 6","Ġjoint ly","ĠBris bane","ĠiP od","Ġpump ing","Ġgo at","ĠSh aron","eal ing","Ġcor on","Ġan omal","rah im","ĠConnect ion","Ġsculpt ure","Ġsched uling","ĠD addy","at hing","Ġeyeb rows","Ġcur ved","Ġsent iments","Ġdraft ing","D rop","( [","Ġnom inal","ĠLeaders hip","ĠG row","Ġ17 6","Ġconstruct ive","iv ation","Ġcorrupt ed","ger ald","ĠC ros","ĠChe ster","ĠL ap","ãģ ª","OT H","D ATA","Ġal mond","pro bably","I mp","Ġfe ast","ĠWar craft","F lor","Ġcheck point","Ġtrans cription","Ġ20 4","Ġtwe aks","Ġrel ieve","S cience","Ġperform er","Z one","Ġtur moil","ig ated","hib it","ĠC afe","the med","Ġflu or","ben ch","Ġde com","ĠU nt","ĠBar rett","ĠF acts","Ġt asting","ĠPTS D","ĠSe al","ĠJuda ism","ĠDynam ic","ĠC ors","V e","ĠM ing","ĠTrans form","v on","ĠDef enders","ĠTact ical","ĠV on","ĠUn ivers","Ġdist orted","ĠB reath","?' \"","Ġag on","ĠDead ly","Ġl an","ĠCy cle","orn ed","Ġrel iably","Ġgl or","ĠMon key","ãĥ ¡","Ġad ren","Ġmicrow ave","ĠAl ban","irc raft","dig it","sm art","ĠD read","¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯","{ {","ĠRoc hester","Ġsimpl ified","Ġinf licted","Ġtake over","Ġyour selves","ad itional","Ġmus cular","K S","Ġing en","T ax","ĠFe ature","27 7","Ġcru c","Ġcr ate","Ġun identified","Ġacclaim ed","ĠM anga","ĠFr ances","ĠNep al","ĠG erald","ĠKu wait","Ġsl ain","ĠHe b","ĠG oku","ãģ® æ","28 6","M rs","ĠC ody","ĠSan ctuary","01 6","Ġdism ant","Ġdatas et","ĠH ond","b uck","ĠPat terson","Ġpal ette","ĠG D","ic ol","ĠL odge","Ġplanet ary","ak in","ĠRegist ered","ab we","ĠPeters burg","Ġha iled","ĠP iece","S che","ĠDO J","Ġen umer","18 1","ĠObs erver","ĠB old","f ounded","com merce","Ġexplo its","ĠF inding","UR N","ĠS ne","ĠAc id","ay ette","ĠVal ues","Ġdr astic","Ġarchitect ural","Ġ\" .","× ķ","ump ed","Ġwra pping","Ġwid ow","ĠSl ayer","l ace","on ce","German y","av oid","Ġtem ples","P AR","à ´","ĠLuc ifer","ĠFl ickr","l ov","for ces","Ġsc outing","Ġlou der","tes y","Ġbefore hand","Ä ĵ","ĠNe on","ĠW ol","ĠTyp ically","ĠPolit ico","-+ -+","Ġbuild er","Ġder ive","K ill","Ġp oker","Ġambig uous","Ġlif ts","Ġcy t","Ġrib s","ood le","ĠS ounds","h air","ĠSynd rome","t f","Ġproport ional","u id","Ġper taining","ĠKind le","ĠNeg ro","Ġreiter ated","ĠTon ight","oth s","ĠCorn ell","Ġo wing","Ġ20 8","elf are","oc ating","ĠB irds","Sub scribe","Ġess ays","Ġburd ens","Ġillust rations","ar ious","ER AL","ĠCal cul","Ġx en","ĠLink edIn","ĠJ ung","Ġredes ign","Con nor","29 6","Ġrevers al","ĠAd elaide","ĠL L","Ġs inking","Ġg um","US H","c apt","ĠGr imm","Ġfoot steps","ĠCB D","isp ers","Ġpro se","Wed nesday","ĠM ovies","ed in","Ġoverturn ed","Ġcontent ious","US B","~~~~~~~~ ~~~~~~~~","ĠCo pper","Ġpoint less","N V","val ues","olph in","d ain","Ġdepos ited","ĠG W","Ġpreced ed","ĠCl a","ĠGo lem","ĠN im","ĠÎ ²","ĠEngine ers","m iddle","Ġfl att","oper ative","Ġcouncil s","imb abwe","el in","Ġstress ful","ĠL D","Ġres h","l ake","Ġwheel chair","ĠAltern ative","Ġoptim ize","oper ation","Ġpe ek","Ġones elf","ig il","Ġtrans itions","op athy","bl ank","Ġ16 9","17 1","________________________________ ________________________________","Ġl aundering","En c","ĠD EC","Ġwork outs","Ġsp ikes","Ġdin osaurs","Ġdiscrim inatory","P ool","R ather","38 5","R NA","tes ters","et o","ĠIdent ity","Ġve in","ĠBur ton","Ġarc ade","4 20","Ult imately","ĠSad ly","à °","p ill","Ġcub ic","ĠSpect rum","the se","st ates","Ġun official","h awks","ĠEVER Y","Ġrain bow","Ġincarcer ation","and ing","Ġsy ll","ĠEver ton","Ġ17 9","ĠSer bia","Ġ18 9","m eter","ĠMic key","Ġant iqu","Ġfact ual","ne ck","ĠN are","n orm","m ust","Ġhigh ways","Ġgl am","Ġdivid ing","ĠSquad ron","ĠMar tha","Ġbirth s","C over","//////// ////////","ĠW ong","Ph ot","ĠA LS","ri o","ĠNon etheless","ĠL emon","Ġ20 6","ĠE E","Ġderiv ative","ĠWW II","v ote","Ġthere in","Ġsepar ating","44 6","sy nc","ĠStre ets","Ġr att","Ġmunicip ality","ĠShort ly","Ġmon k",") ,\"","Ġscr ub","Ġoper atives","Ne ither","Pl ace","ĠLim it","F emale","ĠAct or","Char acter","Ġconstit uted","35 7","Ġprotest ed","ĠSt raw","ĠHe ight","ild a","ĠTy ph","Ġflood s","Ġcos metic","W AY","pert ure","up on","t ons","ess ing","ĠP ocket","Ġro oft","ĠC aucas","Ġant idepress","Ġincomp atible","EC D","Ġoper a","ĠCont est","Ġgener ators","l ime","Def ense","19 87","for um","Ġsav age","ĠHung arian","n z","Ġmet allic","Ġex pelled","Ġres idency","Ġdress es","66 6","ĠC lement","f ires","C ategory","Ġge ek","al is","Ġc emetery","educ ated","Ġc rawl","ĠUn able","ĠT yson","ak is","Ġp ardon","ĠW ra","Ġstrengthen ed","ĠF ors","33 5","ĠH C","ĠM ond","Ġvisual s","ĠBeat les","ett lement","Ġ ï","g ro","Ġb ash","Ġpo orest","Ġex cel","Ġaspir ations","ĠM unicip","ens ible","Ġceremon ies","Ġintimid ation","ĠCON TR","be ck","ĠK ap","as u","Ġtradem arks","ĠS ew","ĠComp etition","net work","ĠAr ri","ĠT et","Ro aming","W C","D at","Ġso b","Ġpair ing","Ġoverd ose","SA Y","ab er","Ġrev olt","ĠF ah","act ing","e q","est ation","F ight","ĠMar ks","27 3","Ġ17 8","R aw","ãģ ĭ","34 9","bl ocks","Ġver ge","est ine","ĠPod esta","Ġinv asive","Ġprofound ly","ĠA o","e ach","Ġl est","inter pret","Ġshr inking","Ġerr one","Ġche es","ly s","ĠI vy","ĠDirect ory","Ġhint ed","V ICE","Ġcontact ing","ĠG ent","he i","Ġlabel ing","Ġmerc ury","ĠL ite","Ġexp ires","Ġdest abil","rit is","c u","Ġfeather s","Ġste er","Ġprogram med","ĠV ader","Go ing","ĠE lim","Ġy o","ĠMic he","Ġ20 3","Ġslee ves","Ġb ully","ĠHum ans","36 8","Ġcomp ress","ĠBan ner","AR S","Ġa while","Ġcal ib","Ġspons orship","ĠDiff iculty","ĠP apers","Ġident ifier","} .","Ġy og","ĠSh ia","Ġclean up","Ġvib e","int rodu","im ming","Austral ia","Ġout lines","ĠY outube","tr ain","ĠM akes","Ġde ported","Ġcent r","ĠD ug","ĠB oulder","ĠBuff y","Ġinj unction","ĠHar ley","ĠG roups","ĠD umbledore","ĠCl ara","Ġ\" -","Ġsacrific ed","ep h","Sh adow","ib ling","Ġfreel ance","Ġevident ly","ph al","Ġret ains","M ir","Ġfin ite","d ar","ĠC ous","Ġrep aired","Ġperiod ic","Ġchampions hips","Ġaster oid","bl ind","Ġexpress ly","ĠAst ros","Ġsc aled","Ġge ographical","ĠRap ids","En joy","Ġel astic","ĠMoh amed","Mark et","be gin","Ġdisco vers","Ġtele communications","Ġscan ner","Ġen large","Ġsh arks","Ġpsy chedel","ĠRou ge","Ġsnap shot","is ine","X P","Ġpestic ides","ĠL SD","ĠDist ribution","re ally","Ġde gradation","Ġdisgu ise","Ġbi om","ĠEX T","Ġequ ations","Ġhaz ards","ĠComp ared",") *","Ġvirt ues","Ġeld ers","Ġenh ancing","ĠAc ross","er os","ang ling","Ġcomb ust","ucc i","Ġconc ussion","Ġcontrace ption","ĠK ang","Ġexpress es","Ġa ux","ĠP ione","Ġexhib its","Deb ug","OT AL","ĠAl ready","ĠWheel er","Ġexp ands","? :","Ġreconc iliation","Ġpir ates","Ġpur se","Ġdiscour age","Ġspect acle","R ank","Ġwra ps","ĠTh ought","Ġimp ending","O pp","ĠAng lo","ĠE UR","Ġscrew ed","ret ched","Ġencour agement","mod els","Ġconf use","mm m","ĠVit amin","âĸij âĸij","C ru","Ġkn ights","Ġdisc ard","Ġb ishops","ĠW ear","ĠGar rett","k an","ãĥ Ł","Ġmascul ine","cap ital","ĠA us","Ġfat ally","th anks","ĠA U","ĠG ut","12 00","Ġ 00000000","Ġsur rog","ĠBI OS","ra its","ĠWat ts","Ġresur rection","ĠElect oral","ĠT ips","4 000","Ġnut rient","Ġdepict ing","Ġspr ink","Ġm uff","ĠL IM","ĠS ample","ps c","ib i","gener ated","Ġspec imens","Ġdiss atisf","Ġtail ored","Ġhold ings","ĠMonth ly","ĠE at","po ons","Ġne c","ĠC age","ĠLot us","ĠLan tern","Ġfront ier","Ġp ensions","Ġj oked","ĠHard y","=-=- =-=-","r ade","U ID","Ġr ails","Ġem it","Ġsl ate","Ġsm ug","Ġsp it","ĠCall s","ĠJac obs","f eat","ĠU E","Ġrest ruct","Ġregener ation","Ġenerg ies","ĠCon nor","OH N","ĠChe ese","Ġg er","Ġresur rect","man agement","N W","Ġpres ently","ĠBru ins","M ember","ĠM ang","id an","Ġboost ing","w yn","+ .","requ isite","ĠNY PD","ĠMe gan","ĠCond itions","Ġp ics","nes ium","ĠR ash","Ġ17 4","ĠD ucks","Ġemb ro","z u","on ian","rel igious","Ġc raz","ĠAC A","ĠZ ucker","EM A","ĠPro s","We apon","ĠKn ox","ĠAr duino","Ġst ove","Ġheaven s","ĠP urchase","Ġher d","Ġfundra iser","Dig ital","5 000","Ġprop onents","/ âĢĭ","Ġj elly","ĠVis a","Ġmon ks","Ġadvance ment","ĠW er","Ġ18 7","e us","ert ility","Ġfet al","Ġ19 36","L o","Ġout fits","Ġstair case","b omb","Ġcustom ized","cl air","T ree","Ġm apped","ĠConsider ing","ĠTor res","Ġmeth yl","Ġapprox imate","Ġdo om","ĠHans en","Ġc rossover","Ġstand alone","ä ¼","Ġinv ites","Ġgra veyard","Ġh p","Donald Trump","Ġesc ort","G ar","Ġpredec essors","Ġh ay","Ġen zyme","ĠStra ight","vis ors","I ng","ane ously","ĠApp lied","Ġf ec","ĠDur ant","Ġout spoken","or b","Ġz eal","Ġdisgr ace","' ).","ĠChe ng","28 9","ĠRen a","ĠSu icide","29 4","Ġout raged","ĠNew man","ĠN vidia","ĠA ber","ĠB ers","Ġrecre ation","Wind ow","ĠD P","x e","Ġped oph","Ġfall out","ambo o","Ġpresent ations","ĠApp s","Ġh tml","3 45","ĠX XX","Ġrub bing","ĠLe ather","Ġhum idity","se ys","est ablished","ĠUn its","64 6","Ġrespect able","A uto","Ġthri ving","ĠInn ovation","ang s","Ext ra","reg ulation","29 8","p ick","Ex amples","ĠC J","Att ack","Ġdr acon","L T","Ġstick er","re rs","Ġsun ny","I ss","reg ulated","d im","ĠAb stract","Ġhus bands","Off ice","om ination","it ars","AN GE","asc al","ĠK ris","ĠInf antry","Ġm alf","ĠA the","ĠR ally","bal anced","................ ........","OU P","Ġmole cule","met ics","ĠSpl it","ĠInstruct ions","ĠN ights","c ards","Ġt ug","Ġcon e","å Ń","Ġt x","ĠDisc ussion","Ġcatast rophe","pp e","g io","Ġcommun ism","Ġhal ted","ĠGu ant","cle an","ĠSc hed","ĠK anye","Ġw ander","ĠSer iously","Ġ18 8","enn ial","f ollow","product ive","ĠFl ow","ĠS ail","Ġc raw","Ġsim ulations","or u","ang les","ĠN olan","Ġmen stru","4 70","Ġ20 7","aj a","Ġcas ually","board ing","Ġ2 22","ov y","ĠN umbers","um at","O E","28 7","ĠCle mson","Ġcert s","Ġsl id","ĠT ribe","Ġto ast","Ġfort unes","Ġf als","ĠComm ittees","Ġg p","Ġf iery","ĠN ets","ĠAn ime","Pack age","ĠComp are","l aughter","in fect","Ġatroc ities","Ġjust ices","Ġins ults","ĠVern on","Ġsh aken","Ġperson a","est amp","36 7","br ain","Ġexperiment ing","K en","ĠElect ronics","Ġ16 1","dom ain","Ġgraph ical","b ishop","Ġwho pping","ĠEv angel","Ġadvertis ers","ĠSpe ar","Ġb ids","Ġdestro ys","ut z","Ġunders c","ĠAD D","Ġan ts","ĠC um","ipp les","ĠF ill","Ġgl anced","Ġind icted","ĠE ff","Ġmis con","ĠDes ktop","Ġab ide","ãĥ Ģ","ĠI o","ĠC oul","Ġcaps ule","ĠCh rys","M ON","Ġund es","ĠI RA","Ġc itation","Ġdict ate","ĠNet works","ĠConf lict","ĠSt uff","x a","is ec","ĠChem istry","Ġquarter ly","William s","an an","O pt","ĠAlexand ria","out heastern","ĠSpring field","ĠBlack s","Ġge ography","24 2","Ġut most","ĠEx xon","ab outs","E VA","ĠEn able","ĠBar r","Ġdisag reed","ĠCy prus","Ġdement ia","Ġlab s","Ġubiqu itous","ĠLO VE","Ġconsolid ated","s r","Ġcream y","ĠTim ber","Reg ardless","ĠCert ificate","Ġ\" ...","ogen ous","Capt ain","Ġinsult ing","ĠSor os","ĠInst r","ĠBulgar ia","bet ter","Ġsuck ing","ĠDavid son","at z","Ġcoll ateral","g if","Ġplag ued","ĠC ancel","ĠGard ner","R B","Ġsix teen","Rem ove","ur istic","c ook","R od","Ġcompr ising","f le",") âĢĶ","ĠVik ing","g rowth","agon al","Ġsr f","af ety","m ot","N early","st own","ĠF actor","Ġautom obile","Ġproced ural","m ask","amp ires","Ġdisapp ears","j ab","3 15","Ġ19 51","ne eded","Ġd aring","le ader","Ġp odium","Ġun healthy","Ġm und","Ġpy ramid","oc re","Ġkiss ed","Ġdream ed","ĠFant astic","ĠG ly","å Ĭ","Ġgreat ness","Ġsp ices","Ġmet ropolitan","Ġcomp uls","i ets","101 6","ĠSh am","ĠP yr","fl ies","ĠMid night","Ġswall owed","Ġgen res","ĠL ucky","ĠRew ards","Ġdisp atch","ĠI PA","ĠApp ly","Ġa ven","al ities","3 12","th ings","Ġ( ).","Ġm ates","ĠS z","ĠC OP","ol ate","O FF","Ġre charge","c aps","ĠYork er","ic one","Ġgal axies","ile aks","D ave","ĠP uzz","ĠCelt ic","ĠA FC","27 6","ĠS ons","Ġaffirm ative","H or","Ġtutorial s","ĠC ITY","ĠR osa","ĠExt ension","Ser ies","Ġf ats","Ġr ab","l is","Ġun ic","Ġe ve","ĠSp in","Ġadul thood","ty p","Ġsect arian","Ġcheck out","ĠCy cl","S ingle","Ġmart yr","Ġch illing","88 8","ou fl","Ġ] ;","Ġcongest ion","m k","ĠWhere as","Ġ19 38","ur rencies","er ion","Ġbo ast","ĠPat ients","Ġch ap","ĠB D","real DonaldTrump","Ġexam ines","h ov","Ġstart ling","ĠBab ylon","w id","om ew","br ance","ĠOd yssey","w ig","Ġtor ch","ĠV ox","ĠMo z","ĠT roll","ĠAn s","Similar ly","ĠF ul","00 6","Un less","ĠAl one","st ead","ĠPub lisher","r ights","t u","ĠDoes n","Ġprofession ally","Ġcl o","ic z","Ġste als","Ġ á","19 86","Ġst urdy","ĠJoh ann","Ġmed als","Ġfil ings","ĠFr aser","d one","Ġmult inational","Ġf eder","Ġworth less","Ġp est","Yes terday","ank ind","Ġg ays","Ġb orne","ĠP OS","Pict ure","Ġpercent ages","25 1","r ame","Ġpot ions","AM D","ĠLeban ese","Ġr ang","ĠL SU","ong s","Ġpen insula","ĠCl ause","AL K","oh a","ĠMac Book","Ġunanim ous","Ġl enders","Ġhang s","Ġfranch ises","ore rs","ĠUp dates","Ġisol ate","and ro","S oon","Ġdisrupt ive","ĠSur ve","Ġst itches","ĠSc orp","ĠDomin ion","Ġsupp lying","Ar g","Ġtur ret","ĠL uk","Ġbr ackets","* )","ĠRevolution ary","ĠHon est","Ġnot icing","ĠSh annon","Ġafford ed","Ġth a","ĠJan et","! --","ĠNare ndra","ĠPl ot","H ol","se ver","e enth","Ġobst ruction","Ġ10 24","st aff","j as","or get","sc enes","l aughs","ĠF argo","cr ime","Ġorche str","Ġde let","ili ary","rie ved","Ġmilit ar","ĠGreen e","âĹ ı","ãģ ¦","ĠGu ards","Ġunle ashed","ĠWe ber","Ġadjust able","Ġcal iber","Ġmotiv ations","Ġà ł","m Ah","ĠL anka","hand le","Ġp ent","ĠR av","ĠAng ular","ĠK au","umb ing","Ġphil anthrop","Ġde hyd","Ġtox icity","e er","ĠY ORK","w itz","å ¼","ĠI E","commun ity","ĠA H","Ġret ali","Ġmass ively","ĠDani els","ĠD EL","Ġcar cin","Ur l","Ġrout ing","ĠNPC s","ĠR AF","ry ce","Ġwa ived","ĠGu atem","Every body","Ġco venant","Ġ17 3","Ġrelax ing","Ġqu art","al most","Ġguard ed","ĠSold iers","ĠPL AY","Ġout going","L AND","Ġre write","ĠM OV","ĠIm per","ĠS olution","Ġphenomen al","Ġl ongevity","Ġimp at","ĠN issan","ir ie","Ġod or","ĠZ ar","ok s","Ġmilit ias","ĠSP EC","Ġtoler ated","ars er","ĠBrad ford","+ ,","Ġsur real","s f","Can adian","Ġresemb lance","Ġcarbohyd rate","VI EW","Ġaccess ory","me al","larg est","ieg el","Some one","Ġtoug hest","os o","Ġfun nel","Ġcondemn ation","lu ent","Ġw ired","ĠSun set","Jes us","ĠP ST","ĠP ages","ĠTy coon","ĠP F","Ġselect ions","Ġ à¤","part isan","Ġhigh s","ĠR une","Ġcraft s","le ad","ĠParent s","Ġre claim","ek er","ĠAll ied","ae per","Ġlo oming","Ġbenefic iaries","ĠH ull","Stud ents","Jew ish","d j","Ġp act","tem plate","ĠOffic ials","ĠBay lor","Ġhe mp","Ġyouth s","ĠLevel s","ĠX iao","ĠC hes","Ġende avor","ĠRem oved","Ġhipp ocamp","H ell","ãĤ Ĭ","80 5","Ġd inosaur","ĠWr ath","ĠIndones ian","Ġcalcul ator","ĠD ictionary","Ġ4 20","ĠM AG","( _","! ,","t arians","Ġrestrict ing","rac use","Ġweek day","OU NT","Ġsh rugged","leg round","Ġb ald","ĠDo ctors","Ġt outed","ĠMax well","Ġ2 14","Ġdiplom at","Ġrep ression","Ġconstitu ency","v ice","r anked","ĠNap oleon","g ang","ĠFore ver","t un","Ġbul b","ĠPD T","ĠC isco","V EN","Ġres umed","Ste ven","ĠManit oba","Ġfab ulous","ĠAg ents","19 84","Ġam using","ĠMyster ies","Ġor thodox","fl oor","Ġquestion naire","Ġpenet rate","Ġfilm makers","ĠUn c","Ġst amped","Ġth irteen","Ġout field","Ġforward ed","Ġapp ra","Ġa ided","t ry","Ġunf ocused","ĠL iz","ĠWend y","ĠSc ene","Ch arg","Ġreject s","Ġleft ist","ĠProv idence","ĠBr id","reg n","Ġprophe cy","ĠL IVE","4 99","Ġfor ge","ĠF ML","Ġintrins ic","ĠF rog","Ġw ont","ĠH olt","Ġfam ed","CL US","aeper nick","ĠH ate","ĠC ay","Ġregister ing","ort ality","rop y","ocaly ptic","a an","n av","Ġfasc ist","IF IED","Ġimpl icated","ĠRes ort","ĠChand ler","ĠBr ick","P in","ys c","Us age","ĠHel m","us ra","âĺħ âĺħ","ĠAb bas","Ġunanim ously","Ġke eper","Ġadd icted","?? ?","Ġhelm ets","Ġant ioxid","aps ed","80 8","gi ene","Ġwa its","Ġmin ion","ra ved","ĠP orsche","Ġdream ing","Ġ17 1","ĠC ain","Ġun for","ass o","ĠConfig uration","k un","hard t","Ġn ested","ĠL DS","L ES","Ġt ying","en os","Ġc ue","ĠMar qu","sk irts","Ġclick ed","Ġexp iration","ĠAccording ly","ĠW C","Ġbless ings","Ġaddict ive","ĠN arr","y x","ĠJagu ars","Ġrent s","ĠS iber","Ġt ipped","ous se","ĠFitz gerald","Ġhier arch","out ine","Ġwa velength","> .","ch id","ĠProcess ing","/ +","r anking","E asy","ĠConst ruct","Ġt et","ins ured","H UD","Ġqu oting","Ġcommun icated","in x","Ġin mate","Ġerect ed","ĠAbs olutely","ĠSure ly","Ġun im","ĠThr one","he id","Ġcl aws","Ġsuper star","ĠL enn","ĠWh is","U k","ab ol","Ġsk et","ĠN iet","Ġper ks","Ġaff inity","Ġopen ings","phas is","Ġdiscrim inate","T ip","v c","Ġgr inding","ĠJenn y","Ġast hma","hol es","ĠHom er","Ġreg isters","ĠGl ad","Ġcre ations","Ġlith ium","Ġappl ause","unt il","Just ice","ĠTur ks","Ġsc andals","Ġb ake","t ank","M ech","ĠMe ans","ĠM aid","Republic ans","is al","wind ows","ĠSant os","Ġveget ation","33 8","t ri","Ġfl ux","ins ert","Ġclar ified","Ġmort g","ĠCh im","ĠT ort","Ġdiscl aim","met al","ĠAs ide","Ġindu ction","Ġinf l","Ġathe ists","amp h","Ġe ther","ĠV ital","ĠBu ilt","M ind","Ġweapon ry","S ET","Ġ18 6","ad min","g am","cont ract","af a","Ġderiv atives","Ġsn acks","Ġch urn","E conom","Ġca pped","ĠUnder standing","ĠH ers","ĠI z","Ġd uct","I ENT","augh ty","Ġâľ Ķ","ĠN P","Ġsa iling","In itialized","Ġt ed","Ġreact ors","ĠL omb","Ġcho ke","ĠW orm","Ġadm iration","Ġsw ung","ens ibly","Ġr ash","ĠGo als","ĠImport ant","Sh ot","ĠR as","Ġtrain ers","ĠB un","Work ing","Ġhar med","ĠPand ora","ĠL TE","Ġmush room","ĠCH AR","ĠF ee","ĠM oy","B orn","ol iberal","ĠMart ial","Ġgentle men","Ġling ering","Offic ial","Ġgra ffiti","ĠN ames","D er","Ġqu int","ist rate","aze era","ĠNOT ICE","ĠFlore nce","Ġpay able","Ġdep icts","ĠSpe cies","He art","âĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢ","Ġencl osed","Incre ases","D aily","ĠL is","Ġenact ment","ĠB acon","ĠSt eele","dem and","Ġ18 3","Ġmouth s","Ġstr anded","Ġenhance ment","01 1","ĠWh ats","Ġhe aled","en y","ĠR ab","Ġ3 40","ĠLab yrinth","ro ach","ĠY osh","ĠCl ippers","Ġconcert s","Intern et","35 5","Ġstick ers","Ġter med","ĠAx e","Ġgrand parents","Fr ance","ĠCl im","ĠU h","ul ic","Ġthr ill","cent ric","ĠOver view","ĠCond uct","Ġsubstant ive","Ġ18 2","m ur","Ġstr ay","ĠCo ff","Ġrep etitive","ĠFor gotten","Ġqual ification","ew itness","ĠZ imbabwe","Ġsim ulated","ĠJ D","25 3","ĠW are","Ġun sc","T imes","Ġsum mons","Ġdis connected","Ġ18 4","ci us","ĠGu jar","od ka","Ġer ase","ĠTob acco","elect ed","Ġun cont","ĠShe pard","ĠL amp","Ġalert ed","Ġoper ative","arn a","u int","Ġneglig ence","ac ements","Ġsup ra","Ġprev ail","ĠSh ark","Ġbel ts","ãģ «","Ġt ighter","Engine ers","Ġin active","Ġexp onent","ĠWill ie","a ples","Ġhe ir","ĠH its","ian n","ĠS ays","Ġcurrent s","ĠBeng al","Ġar ist","B uffer","Ġbree ze","ĠWes ley","Col a","Ġpron oun","Ġde ed","ĠK ling","Ġof t","Ġinf lict","Ġpun ishing","Ġn m","ik u","OD UCT","01 4","Ġsubsid y","ĠDE A","ĠHer bert","ĠJ al","B ank","Ġdef erred","Ġship ment","B ott","Ġal le","b earing","HT ML","Off line","Ġ2 13","Ġscroll ing","Ġsc anned","ĠLib yan","ĠT OP","ch rom","d t","col umn","Psy NetMessage","Z ero","Ġtor so","0 50","âķ IJ","Ġimp erson","ĠSchw artz","ud ic","Ġpiss ed","ĠS app","25 7","ĠIS Ps","og l","Ġsuper vised","Ġad olescent","Ġatt ained","ĠDel ivery","ĠB unny","Ġ19 37","Ġmini ature","Ġo s","Ġ3 70","60 8","ĠMour inho","Ġinn ate","Ġtem po","ĠN M","ĠFall en","00 9","Ġprov ocative","Stream er","ĠBened ict","ĠBol she","Ġt urtle","ĠPC B","ĠEqu al","Direct or","ĠR end","Ġflu ids","Author ities","Ġcous ins","requ ency","ĠNeigh bor","s ets","sh ared","Char les","pass word","Ġg ears","Ġ2 11","ĠHard ware","ri ka","Ġup stream","H om","Ġdisproportion ately","iv ities","Ġund efined","Ġelect rons","Ġcommem or","Event ually","Ġ> <","Ġir responsible","2 18","ĠRe leased","ĠO VER","ĠI GN","ĠB read","st ellar","ĠS age","tt ed","dam age","ed ition","ĠPre c","Ġl ime","Ġconf inement","Ġcal orie","we apon","Ġdiff ering","ĠS ina","m ys","am d","Ġintric ate","k k","ĠP AT","ã o","st ones","lin ks","Ġr anch","Sem itic","Ġdifferent iate","ĠS inger","occup ied","Ġfort ress","c md","Ġinter ception","ĠAnk ara","Ġre pt","ĠSol itaire","Ġrem ake","p red","Ġd ared","aut ions","ĠB ACK","Run ning","Ġdebug ging","Ġgraph s","3 99","ĠNig el","Ġb un","Ġpill ow","Ġprog ressed","fashion ed","Ġob edience","ER N","Ġrehe ars","C ell","t l","S her","Ġher ald","ĠPay ment","ĠC ory","ĠDe pt","Ġrep ent","ĠWe ak","uck land","Ġple asing","Ġshort ages","Ġjur ors","ĠK ab","q qa","Ant i","Ġw ow","ĠRC MP","Ġt sun","ĠS ic","Ġcomp rises","Ġsp ies","Ġprec inct","n u","Ġur ges","Ġtim ed","Ġstrip es","ĠB oots","Ġy en","Adv anced","Ġdisc rete","ĠArch angel","employ ment","D iff","Ġmon uments","Ġ20 9","work er","Ġ19 6","ĠI g","utter stock","T PS","J ac","Ġhomeless ness","Ġcomment ator","Ġrac ially","f ing","se ed","E le","ell ation","Ġeth anol","Ġpar ish","ĠD ong","ĠAw akening","Ġdev iation","ĠB earing","ĠTsu k","Ġrec ess","Ġl ymph","ĠCann abis","å ľ","ĠNEW S","Ġd ra","ĠStef an","ĠWr ong","ĠS AM","Ġloose ly","Ġinterpre ter","ĠPl ain","Go vernment","Ġbigot ry","Ġgren ades","ave z","pict ured","Ġmand ated","ĠMon k","ĠPed ro","Ġl ava","27 4","Ġcyn ical","ĠScroll s","l ocks","M p","Ġcon gregation","orn ings","ph il","ĠI bid","Ġf erv","Ġdisapp earing","Ġarrog ant","sy n","ĠMa ver","ĠSu it","24 1","Ġab bre","ack ers","P a","ĠY el","Whe never","Ġ23 5","ĠV ine","ĠAn at","Ġext inct","LE T","Ġexecut able","V ERS","ox ide","D NA","ĠP rel","Ġresent ment","Ġcompr ise","ĠAv iv","Ġinter ceptions","Ġprol ific","IN A","ĠEr in","though t","2 19","ĠPsychiat ry","un ky","chem ist","H o","ĠMcC oy","Ġbr icks","L os","ri ly","ĠUS SR","Ġr ud","Ġl aud","ĠW ise","ĠEmer ald","Ġrev ived","Ġdam ned","ĠRep air","id em","ct ica","Ġpatri arch","ĠN urs","me g","Ġcheap est","re ements","empt y","ĠCele br","Ġdepri vation","ch anted","ĠTh umbnails","E nergy","ĠEth an","ĠQ ing","Ġopp oses","W IND","v ik","ĠM au","ĠS UB","66 7","G RE","ĠVol unte","nt on","C ook","å IJ","es que","Ġplum met","Ġsu ing","Ġpron ounce","Ġresist ing","ĠF ishing","ĠTri als","Ġy ell","Ġ3 10","Ġin duct","Ġpersonal ized","oft en","R eb","EM BER","Ġview point","Ġexist ential","() )","rem ove","MENT S","l asses","Ġev apor","Ġa isle","met a","Ġreflect ive","Ġentit lement","Ġdev ised","mus ic","asc ade","Ġwind ing","off set","Ġaccess ibility","ke red","Bet ter","ĠJohn ston","th inking","S now","ĠCroat ia","ĠAt omic","27 1","34 8","Ġtext book","ĠSix th","Ġ اÙĦ","Ġsl ider","ĠBur ger","b ol","S ync","Ġgrand children","Ġc erv","+ )","Ġe ternity","Ġtweet ing","Ġspec ulative","Ġpiv otal","ĠW P","ĠT ER","ynam ic","Ġu pl","ĠC ats","per haps","Ġclass mates","Ġblat ant","' -","Ġl akh","ant ine","ĠB org","i om","/ (","ĠAthlet ic","Ġs ar","OT A","ĠHoff man","Never theless","Ġad orable","Ġspawn ed","Ass ociated","ĠDom estic","Ġimpl ant","ĠLux em","ĠK ens","Ġp umps","ĠS AT","Att ributes","50 9","av our","Ġcentral ized","ĠT N","Ġfresh ly","ĠA chieve","Ġouts iders","her ty","ĠRe e","ĠT owers","ĠD art","ak able","Ġm p","ĠHeaven ly","Ġr ipe","ĠCarol ine","ry an","Ġclass ics","Ġret iring","Ġ2 28","Ġa h","Ġdeal ings","Ġpunch ing","ĠChap man","O ptions","max well","vol ume","Ġst al","Ġex ported","ĠQu ite","Ġnumer ical","B urn","F act","ĠKey stone","Ġtrend ing","Ġalter ing","ĠAfric ans","47 8","ĠM N","ĠKn ock","Ġtempt ation","Ġprest ige","Over view","ĠTrad itional","ĠBah rain","Priv ate","ĠH OU","Ġbar r","ĠT at","C ube","US D","ĠGrand e","ĠG at","ĠFl o","Ġres ides","Ġind ec","vol ent","Ġperpet ual","ub es","Ġworld view","ĠQuant um","Ġfil tered","Ġen su","orget own","ERS ON","ĠM ild","37 9","OT T","à ¥","Ġvit amins","Ġrib bon","Ġsincere ly","ĠH in","Ġeight een","Ġcontradict ory","Ġgl aring","Ġexpect ancy","Ġcons pir","Ġmon strous","Ġ3 80","re ci","Ġhand ic","Ġpump ed","Ġindic ative","Ġr app","Ġav ail","ĠLEG O","ĠMar ijuana","19 85","ert on","Ġtwent ieth","################ ################","ĠSw amp","Ġval uation","Ġaffili ates","adjust ed","ĠFac ility","26 2","Ġenz ymes","itud inal","Ġimp rint","S ite","Ġinstall er","ĠT RA","m ology","lin ear","ĠCollect ive","ig ating","ĠT oken","Ġspec ulated","K N","ĠC ly","or ity","Ġdef er","Ġinspect ors","appro ved","R M","ĠSun s","Ġinform ing","ĠSy racuse","ib li","7 65","Ġgl ove","Ġauthor ize","â̦â̦â̦â̦ â̦â̦â̦â̦","ĠCru ise","Ġcontract ing","she ll","IF E","ĠJew el","p ract","ĠPhot oshop","ĠKnow ing","h arm","Ġattract ions","ad an","et us","01 8","w agen","Al t","Ġmultip ly","Ġequ ilibrium",": {","ĠF ighters","ĠEd gar","Ġfour teen","Go vern","Ġmis use","Ġab using","Ġancest ry","ram er","64 4","Ġwor ms","Ġthick er","ĠComb ine","Ġpeas ants","Ġv ind","Ġcon quest","Ġm ocked","Ġc innamon","ĠC ald","ĠGall up","Ġavoid ance","Ġincarn ation","ĠStr at","Ġt asted","ent a","ĠN eal","p ared","Ġtermin ology","ject ion","Scient ists","ĠIN S","ĠDe e","Ġdirect ories","R oad","ĠSh ap","br ight","ĠDirect ors","ĠCol umn","Ġb ob","Ġprefer ably","Ġgl itch","f urt","Ġe g","id is","C BC","Ġsur rendered","Ġtest ament","33 6","ug gest","ĠN il","an other","Ġpat hetic","ĠDon na","Ġ2 18","ĠA very","Ġwhis key","Ġf ixture","ĠCon quest","Ġbet s","O cc","ĠLe icester","] .\"","Ġ) );","Ġfl ashes","45 6","Ġmask ed","ge bra","Ġcomput ed","che l","aud er","Ġdefe ats","ĠLiber ation","ĠOs ama","ĠV ive","Ch anges","Ch annel","Ġtar iffs","Ġm age","ĠS ax","Ġinadvert ently","ĠC RE","ĠRe aper","ink y","gr ading","Ġstere otyp","Ġcur l","ĠF ANT","Ġfram eworks","M om","ĠAn ch","Ġflav our","car bon","Ġperm itting","let cher","ĠMo zilla","ĠPark ing","ĠCh amp","Sc roll","Ġmurd erer","Ġrest ed","Ġow es","ĠP oss","AD D","IF F","res olution","ĠMin ing","Ġcompar ative","D im","Ġneighbour ing","ĠA ST","ĠT oxic","Ġbi ases","Ġgun fire","ur ous","ĠMom ent","19 83","Ġper vasive","tt p","ĠNorm ally","r ir","S arah","ĠAlb any","Ġun sett","ĠS MS","ip ers","l ayer","ĠWh ites","up le","Ġtur bo","ĠLe eds","Ġthat s","ĠMin er","M ER","ĠRe ign","Ġper me","ĠBl itz","Ġ19 34","Ġintimid ating","t ube","Ġecc entric","ab olic","box es","ĠAssoci ates","v otes","Ġsim ulate","um bo","aster y","Ġship ments","FF FF","an th","Ġseason ed","Ġexperiment ation","âĸ ł","law s","Me et","idd les","ant ics","R ating","IS IS","h ift","Ġfront s","b uf","01 7","Ġun att","ĠD il","le ases","ĠGard ens","77 7","t ouch","ve ll","45 8","Ġ= ====","s aving","Ġer osion","ĠQu in","Ġearn s","Ġaccomplish ment","ĠWe i","Ġ< [","____ _","Ġir rig","ĠT eddy","Ġconqu ered","ĠArm ored","Ġassert s","Ġmanip ulating","r é","Ġtranscript s","G allery","Ġplot ting","Ne il","Ġbetray al","load er","ĠS ul","Ġdispl acement","Ġroy alty","ĠW I","he it","ĠDev ices","alle l","Ġmunicipal ities","Ġcan al","St ars","ĠU AE","Ġ\" â̦","ĠC U","ab ove","Ġreson ance","ĠguiActive Un","add ed","ĠBra ves","ĠI bn","Ġhere by","ĠB RE","Ġshare holder","ĠH ir","ĠJ i","Ġstrange ly","Ġadm ired","Ġpl ight","Ġb achelor","ĠP ole","cipl inary","T ony","ĠArmen ian","Ġun man","ĠZion ist","St age","isco ver","Ġautom otive","Ġs idelines","Ġsl ick","ĠRena issance","ĠF UN","Im ages","ĠH aj","Ġp ing","Ġshort cut","ĠBl vd","ĠLook s","Ġbur sts","Ġcl amp","Ġm ish","Ġsort ing","Ġpatri ot","Ġcorrect ness","ĠScand inav","ĠCaval iers","p ython","az ar","Ġ3 75","ĠJa une","40 9","Ġdetrim ental","Ġstab bing","Ġpoison ed","Ġf ountain","oc ent","or st","ĠMar i","Ġr ains","ĠO vers","ĠInst itution","ud get","AM Y","t ale","ĠK R","ĠPr ices","Ġhead aches","Ġlands l","ĠA ura","Bon us","ĠZ hao","ĠH ip","Ġhop s","ĠKurd istan","Ġexplo iting","ry n","Ġhypocr isy","op ening","Ġgun shot","Ġw ed","inter stitial","Inter stitial","Ġam en","Bre aking","Ġmarket ed","W ire","ĠC rowd","Contin ue","ĠK nown","ĠEffect ive","ore an","iz ons","Jose ph","Ġescal ation","us ername","Ġcur tain","AT ES","ĠP AR","ĠM iy","Ġcounter fe","l ene","Ġcont enders","d aily","ĠAs c","ĠPhill ip","most ly","Ġfil ename","he ne","Ġresemb ling","Ġst aging","ĠCh loe","Ġw iring","H on","ĠRen ew","ott age","ĠHy brid","m uch","Ġstro kes","Ġpolicy makers","AP TER","ĠArk ham","pl ot","Ġassist ants","Ġde port","ĠSe ga","Ġinflu enza","ĠC ursed","ĠK obe","Ġskin ny","Prov ider","ĠR ip","Ġincrement al","product s","B F","Ġd ome","ĠC redits","Ġlos ers","int s","ĠBet ty","ĠTal ent","ĠD AM","L v","E ss","Ġd ens","tem p","J udge","od ic","Ġ' (","UR ES","ets k","V O","Ġretrie ved","Ġarchitect s","Ù ĩ","Ġeth ic","ĠSecond ary","st ocks","ad ia","Ġ3 25","ĠOp inion","Ġsimultane ous","Ġd izz","ul p","Ġsmugg ling","ipp ery","R andom","f acing","ĠD as","Ġstock p","Ġdiscl osures","po inter","Ġcor al","ĠSe lection","ĠP ike","ival ent","Ġruth less","ĠR im","Ġensu ing","ĠExper iment","Ġcongress man","Ġbelie ver","Ġun specified","ĠM ord","Ġknowledge able","ĠV ERY","T X","Ġstra ps","Ġtur f","apesh ifter","Ġmar ital","Ġfl ock","ãģ Ĩ","26 3","AM ES","ĠOpp osition","Ġtre asures","ĠG OD","Ġmodel ed","ĠWOR LD","Ġ( [","ĠUs age","H F","Ġ$ (","uss ed","Ġpione er","E ight","par se","b read","rit z","ĠMir anda","ĠK ant","++ )","ore n","Ġprov oked","Ġbre eds","ĠIn cludes","ĠPast ebin","ĠFl ip","J ava","Ġbr ink","Ġrum ored","Ġun seen","Ġgar nered","ĠDef in","al ted","Ġtatt oos","Ġhes itation","is itions","ĠWe aver","ĠReport ing","Ġtherap ies","Ġconsult ants","Ġresid ual","ĠMal i","ĠRom a","i ago","ĠRes idents","ub i","Ġremed ies","Ġadapt ive","ĠAl ive","ĠBar cl","Ġwal lets","c rypt","etermin ation","ĠPel osi","Ġsl ipping","oton in","Ġall iances","pat rick","ir is","Ġor th","ĠPer kins","ĠDe V","ĠG ets","Ġdry ing","ge e","fore st","ĠFor get","ore m","33 9","Ġvague ly","ĠD ion","ĠP orn","ĠH OW","Ġp neum","Ġrub ble","ĠT aste","enc ia","ĠG el","Ġd st","Ġ24 5","ĠMoroc co","inf lamm","ĠTw ins","Ġb ots","d aughter","ĠB alk","Ġbre thren","Ġlog os","Ġgo bl","f ps","Ġsub division","Ġp awn","Ġsquee zed","Ġmor ale","ĠD W","' \"","Ġkn ot","ook y","Ġdiv isive","Ġboost ed","ch y","ãĥ IJ","if act","Ġnewcom ers","ĠWrest ling","Ġsc outs","w olves","R at","Ġnin eteenth","ĠOs borne","St ats","Ġem powered","Ġpsych opath","ĠO EM","ugg age","ĠP K","ĠMoh ammad","P ak","Ġanarch ists","ĠExt ract","est hes","ĠStock holm","l oo","ĠG raph","Ġdeploy ing","ĠStr anger","ĠM old","Ġstaff er","Ġdiscount ed","uck le","ple ase","ĠLand ing","ÃŃ a","Ġ19 3","Ġan te","Ġrep etition","Ġ+ /-","Ġpar ody","Ġlive ly","AA A","ĠHor us","Ġp its","ind ers","L OC","ĠVen ice","40 6","ĠDis cover","â Ĩ","ellect ual","Ġp ens","Ġey el","ig uous","Im pl","Ġj oking","Ġinv al","ĠBel fast","Ġcredit ors","ĠSky walker","ov sky","Ġcease fire","Ġse als","is oft",") ).","ĠFel ix","IT S","Ġt resp","ĠBlock chain","ew are","ĠSch war","en ne","mount ed","ĠBe acon","les h","Ġimmense ly","Ġche ering","Em ploy","sc ene","ish ly","atche wan","ĠNic olas","Ġdr ained","ĠEx it","ĠAz erb","j un","Ġflo ated","u ania","De ep","Ġsuper v","Ġmyst ical","ĠD ollar","ĠApost le","ĠR EL","ĠProv ided","ĠB ucks","ãĥ ´","cut ting","Ġenhance ments","ĠPengu ins","ĠIsa iah","Ġj erk","ĠW yn","Ġst alled","Ġcryptoc urrencies","ĠR oland","sing le","Ġl umin","ĠF ellow","ĠCap acity","ĠKaz akh","W N","Ġfin anced","38 9","Ġt id","Ġcoll usion","ĠMy r","î Ģ","Sen ator","Ġped iatric","Ġneat ly","Ġsandwic hes","ĠArchitect ure","Ġt ucked","Ġbalcon y","Ġearthqu akes","qu ire","F uture","Ġhe fty","é Ĺ","Ġspecial izes","Ġstress es","Ġs ender","Ġmisunder standing","Ġep ile","Ġprov oke","ĠCol ors","Ġdis may","uk o","[ _","58 6","ne utral","Ġdon ating","ĠRand all","Mult i","Ġconvenient ly","ĠS ung","ĠC oca","Ġt ents","ĠAc celer","Ġpart nered","27 2","ir ming","ĠB AS","s ometimes","Ġobject ed","ub ric","p osed","LC S","gr ass","Ġattribut able","V IS","Israel i","Ġrepe ats","ĠR M","v ag","ut a","in ous","Ġin ert","ĠMig uel","æ Ń","ĠHawai ian","B oard","Ġart ific","ĠAzerb ai","as io","ĠR ent","A IN","Ġappl iances","Ġnational ity","Ġass hole","ĠN eb","Ġnot ch","h ani","ĠBr ide","Av ailability","Ġintercept ed","Ġcontin ental","Ġsw elling","ĠPers pect","b ies",". <","ith metic","ĠL ara","Ġtempt ing","add r","Ġoversee ing","cl ad","ĠD V","ĠGing rich","Ġm un","ĠApp ropri","Ġalter ations","ĠPat reon","Ġha voc","Ġdiscipl ines","Ġnotor iously","aku ya","ier i","? ).","ĠW ent","Ġsil icon","Ġtre mb","Cont ainer","K nown","Ġmort ar","est e","ick a","Ar thur","ĠPre viously","ĠMart y","Ġsp arse","g ins","Ġin ward","ĠParticip ant","C opy","ĠM isc","Ġantib iotic","ĠRet ro","Ġel usive","Ġass ail","ĠBatt alion","ĠB ought","Ġdimin ish","ĠEuro pa","s ession","ĠDanger ous","ies el","Ġdisbel ief","Ġbl asts","ext reme","ĠBoy d","ĠProject s","ĠGu ys","Ġunder gone","Ġgr ill","ĠDw ight","Ġ19 7","US ER","Ġfiles ystem","Ġcl ocks","T aylor","Ġwra pper","Ġfold ing","ous and","ĠPhilipp ine","ATION AL","ĠPer th","Ġas hes","Ġaccum ulate","ĠGate way","Sh op","orks hire","H an","ĠBar rel","ĠLe h","ĠX V","Ġwh im","Ġrep o","ĠC G","ĠM am","Ġincorpor ating","Ġbail out","Ġlingu istic","Ġdis integ","C LE","Ġcinem atic","ĠF iber","S yn","il ion","ĠCom pos","c hens","Ġne oc","Ġbo iled","F INE","on o","un cle","ik en","ĠB M","Î ¹","Ġreceipt s","Ġdisp osed","ĠTh irty","ĠR ough","ĠA BS","Ġnot withstanding","oll en","# $","Ġunrel iable","Ġbl oom","Ġmedi ocre","Ġtr am","ĠTas man","Ġsh akes","Ġmanifest o","ĠM W","Ġsatisf actory","Ġsh ores","Ġcomput ation","Ġassert ions","orm ons","ar ag","ab it","Dem ocrats","ĠL oot","ĠVol ks","ha ired","Ġgrav itational","S ing","ĠM iz","Ġthro ttle","Ġtyr anny","ĠView s","Ġrob ber","ĠMinor ity","Ġsh rine","sc ope","pur pose","Ġnucle us","our cing","ĠUS DA","ĠD HS","w ra","ĠBow ie","Sc ale","ĠB EL","x i","I ter","Ġ( ),","w right","Ġsail ors","ous ed","NAS A","ĠPro of","ĠMin eral","t oken","ĠF D","R ew","Ġe ll","6 30","Ġchance llor","ĠG os","Ġamount ed","ĠRec re","ome z","ĠOpt im","ĠOl ive","Ġtrack er","ow ler","ĠUn ique","R oot","Ġmar itime","ĠQur an","ĠAd apt","Ġecosystem s","ĠRe peat","ĠS oy","ĠI MP","Ġgrad uating","and em","P ur","ĠRes et","ĠTr ick","ĠPh illy","ĠT ue","ĠMalays ian","Ġclim ax","Ġb ury","Ġcons pic","ĠSouth ampton","ĠFl owers","Ġesc orted","ĠEduc ational","ĠI RC","Ġbrut ally","e ating","Ġpill ar","ĠS ang","ĠJ ude","ar ling","ĠAm nesty","Ġrem inding","ĠAdminist rative","hes da","Ġfl ashed","ĠP BS","per ate","fe ature","Ġsw ipe","Ġgra ves","oult ry","26 1","bre aks","ĠGu er","Ġsh rimp","ĠV oting","qu ist","Ġanaly tical","Ġtables poons","ĠS OU","Ġresear ched","Ġdisrupt ed","Ġj our","Ġrepl ica","Ġcart oons","b ians","} )","c opy","G ot","ou ched","P UT","Ġsw arm","not ations","s aid","Ġreb uilt","Ġcollabor ate","Ġr aging","Ġn ar","Ġdem ographics","ĠD DR","Ġdist rust","oss ier","ĠK ro","Ġpump kin","Ġreg rets","Ġfatal ities","ĠL ens","ĠO le","p d","Ġpupp et","ĠOut look","ĠSt am","O l","F air","U U","Ġre written","Ä ±","Ġfasc inated","Ġve ctors","Ġtrib unal","u ay","ĠM ats","ĠCo ins","[ [","Ġ18 1","Ġrend ers","ĠK aepernick","Ġesp ionage","Ġsum m","Ġd itch","Acc ount","Ġspread sheet","Ġmut ant","p ast","40 7","Ġd ye","Ġinit iation","Ġ4 000","Ġpunish able","Ġth inner","ĠKh al","Ġinter medi","D un","ĠGoth am","Ġeager ly","Ġvag inal","p owers","V W","ĠWATCH ED","Ġpred ator","ams ung","Ġdispar ity","Ġ[ *","Ġam ph","Ġout skirts","ĠSpir its","Ġskelet al","Ð »","ĠR ear","Ġissu ance","ĠLog ic","re leased","Z Z","ĠB ound","Ent ry","Ġex its","is ol","ĠFound er","Ġw re","ĠGreen land","ĠM MO","t aker","IN C","ãģ ¾","Ġhour ly","hen ko","Ġfantas ies","Ġdis ob","Ġdemol ition","ãĥ ĭ","Ġen listed","rat ulations","Ġmis guided","Ġens ured","Ġdiscour aged","m ort","Ġfl ank","Ġc ess","Ġreact s","ĠS ere","s ensitive","ĠSer pent","ass ad","Ġ24 7","Ġcalm ly","b usters","Ġble ed","ĠSt ro","Ġamuse ment","ĠAntar ctica","Ġs cept","ĠG aw","a q","ason ic","Ġsp rawling","n ative","atur ated","ĠBattle field","IV ERS","E B","ĠG ems","ĠNorth western","ĠFil ms","ĠAut omatic","Ġappre hend","ãģ ¨","Ġgui Name","Ġback end","Ġevid enced","ge ant","01 2","ĠS iege","Ġexternal To","Ġunfocused Range","ĠguiActiveUn focused","Ġgui Icon","ĠexternalTo EVA","ĠexternalToEVA Only","F ri","ch ard","en aries","Ġchief s","Ġc f","ĠH UD","Ġcorro bor","Ġd B","ĠT aken","ĠPat ricia","ra il","ĠCh arm","ĠLiber tarian","rie ve","Person al","ĠO UR","ger ies","Ġdump ing","Ġneurolog ical","it imate","ĠClint ons","raft ed","ĠM olly","Ġtermin als","reg ister","Ġfl are","Ġenc oded","Ġautop sy","p el","m achine","Ġexempt ions","ĠRoy als","d istance","Ġdraft s","Ġl ame","ĠC unning","Ġsp ouses","ĠMark ets","ĠCar rier","Ġimp lying","ĠY ak","s id","Ġl oser","Ġvigil ant","Ġimpe achment","Ġaug mented","ĠEmploy ees","Ġunint ended","tern ally","ĠW att","Ġrecogn izable","ess im","æ Ŀ","Ġco ated","r ha","Ġlie utenant","ĠLegisl ation","pub lished","44 4","01 3","Ġide ally","ĠPass word","Ġsimpl ify","ĠMet a","ĠM RI","Ġple ading","organ ized","hand ler","Ġun ravel","cor rect","Ġ icy","Ġparan oid","Ġpass er","Ġinspect ions","of er","ĠHealth care","28 3","ĠBr ut","iol a","for ge","ĠMed ieval","MS N","ie vers","ĠProgram ming","å ī","Ġ2 23","m u","ĠC LE","ug a","Ġsho ppers","Ġinform ative","ĠPl ans","Ġsupplement ation","ĠT ests","ty ard","ocy tes","ĠVeg a","ĠGujar at","erman ent","Ex cept","ĠL OT","all a","ĠC umm","ĠO sw","Ġven om","ĠDeb t","ĠD OWN","Ġreun ion","Ġm uc","ĠRel ief","Ġge op","ĠðŁ ĺ","al ogue","An th","ech o","Ġcor ros","Ġrepl ication","ĠBl azing","ĠD aughter","Ġinf lic","ĠLind sey","Ù Ī","28 4","Ex it","Ġgl oom","TA IN","Ġundermin ing","Ġadv ising","h idden","Ġover flow","Ġg or","urd ue","Ġe choes","enh agen","Ġimp uls","d rug","c ash","Ġas ync","Ġmir ac","at ts","p unk","Ġpiv ot","ĠLegisl ative","Ġblog gers","ĠCl aw","s burg","d yl","ĠRecomm end","Ġver te","Ġprohib iting","ĠPant her","Jon athan","Ġo min","Ġhate ful","28 1","ĠOr che","ĠMurd och","down s","Ġas ymm","G ER","Al ways","Ġinform s","ĠW M","ĠP ony","ĠApp endix","ĠAr lington","J am","Ġmedic inal","ĠS lam","IT IES","Ġre aff","ĠR i","F G","S pring","b ool","Ġthigh s","Ġmark ings","ĠRa qqa","ĠL ak","p oll","ts ky","ĠMort y","ĠDef inition","Ġdeb unk","end ered","ĠLe one","a vers","Ġmortg ages","App arently","N ic","ha us","ĠTh ousands","au ld","Ġm ash","sh oot","Ġdi arr","Ġconscious ly","H ero","e as","ĠN aturally","ĠDestroy er","Ġdash board","serv ices","R og","Ġmillenn ials","Ġinv ade","- (","Ġcomm issions","ĠA uckland","Ġbroadcast s","Ġfront al","Ġcr ank","ĠHist oric","Ġrum ours","CT V","Ġster il","Ġboost er","rock et","ãĤ ¼","ut sche","ĠP I","Ġ2 33","ĠProdu cer","ĠAnaly tics","Ġinval uable","Ġunint ention","ĠC Y","Ġscrut in","Ġg igg","Ġeng ulf","Ġprolet ariat","Ġh acks","ĠH ew","ar ak","ĠSl ime","ield ing","ag her","ĠEll iot","Ġtele com","Ġ2 19","ult an","ĠAr bor","ĠSc outs","B an","Ġlifes pan","Ġbl asp","38 8","Ġjud iciary","ĠContin ental","ask ing","Mc C","L ED","Ġbag gage","ĠSorce rer","Ġrem nants","ĠGriff ith","ets u","ĠSub aru","ĠPerson ality","des igned","ush ima","agn ar","Ġrec oil","Ġpass ions","\\ \":","Ġte e","Ġabol ition","ĠCreat ing","j ac","Ġ19 4","01 9","Ġpill ars","ric hed","/ \"","t k","Ġlive lihood","Ġro asted","ah on","ĠH utch","ass ert","Ġdivid end","Ġkn it","Ġd aunting","Ġdisturb ance","Ġsh ale","Ġcultiv ated","Ġrefriger ator","L B","ĠN ET","Ġcommercial s","Ġthink ers","45 5","Ġch op","B road","Ġsuspic ions","Ġtag ged","l ifting","Ġsty lish","ĠShield s","Short ly","Ġt ails","A uth","ST E","ĠG AME","Ġse ism","ĠK is","olog ne","Ġcow ork","Ġforc ibly","Ġthy roid","ĠP B","AN E","mar ried","h orse","Ġpoly mer","ĠCh al","od or","DE BUG","ĠCon text","Ġbl iss","Ġpin point","ĠMat hemat","leg ram","ĠWeek end","Ġlab elled","Ġb art","it les","Ġest rogen","âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ","\" '","Ġvis ibly","Ġouts ider","aid a","Are a","Ġdisse min","Ġdish onest","ĠCl osed","ĠBullet in","ĠRam sey","sw ord","ĠX I","our ced","S ame","34 6","ĠRe pe","ĠK ou","c ake","em is","C ache","ĠMe aning","ĠEn light","onom y","Ġmanifest ation","sw orth","J ay","Ġch ore","ö r","D ream","Ġsanction ed","Ġcult urally","ĠA ra","N av","Ġthe ological","Ġstr ut","ĠV O","ĠHand book","Ġconstruct ing","Ġ ¶","ĠBenef its","ĠPsych ological","s ac","å ¸","p olicy","ĠMat ters","ĠReport ed","ĠBy te","Ġvit ro","ĠM aiden","Ġl am","ĠJenn ings","Ġgar ment","ĠRut gers","ĠStaff ord","ĠWell ington","Ġinter mitt","Ġn pm","Ġord eal","Ġplug ged","o oming","in ished","fram ework","Ġtim ber","Ġc ass","Ġ8 50","il ess","ĠRed ux","7 68","St re","Ġsurpass ed","w hel","Ġparalle ls","Ġve il","ĠG I","ĠR EST","Ġread iness","s ort","Ġmod ifying","ĠSl ate","ru ff","Ġmar ble","Ġinf rared","Ġaud itor","ĠFANT ASY","ĠP overty","ĠS PD","Ġ\" (","K y","RA Y","Ġexecut ions","ĠBever ly","ĠMarx ism","ĠBur st","ĠK ali","est ones","Clear ly","E ll","ãģ §","ĠProceed ings","T oken","IF IC","ñ a","Cent ral","ĠH aley","ĠD rama","Ġform ations","OR N","Book s","Ġdom inating","ĠFly ers","ĠCompan ion","Ġdiscipl ined","ĠYug oslav","ĠSpell s","Ġv engeance","Ġland lords","L en","ĠO gre","ano ia","Ġpier cing","Ġcon greg","Ġscore r","ob ia","Ġnic kel","ĠLear ns","Ġre jo","Ġmaster piece","Fl ash","Ġinhab ited","ĠOpen GL","ĠD ud","ĠI CO","Ġar ter","Ġpl ur","Ġmaster y","Ġlong standing","st ed","Ġw ines","Ġtelev ised","ĠSh rine","ĠBay ern","Ġâ ĵĺ","Ġencl osure","j ohn","Ġprophe ts","ĠRes urrection","ĠOrd ers","Ġun even","r als","Ġd wind","ĠL ah","ĠSl oven","37 8","Ġins istence","aff le","ĠCl one","Ġhard ship","ĠCongress man","Ġple ad","Ġreview ers","Ġc ured","Ġ19 35","as ley","f ake","ĠTh inking","yd ia","P ART","ĠD ota","o it","Ġwh ipped","Ġb ouncing","ĠHispan ics","com ings","Ġcann abin","ĠCh ambers","ĠZ ack","Option al","Ġco ats","Ġprow ess","ĠNort on","Ġplain ly","Ġfre ight","Ġinhib ition","Ġcl am","Ġ30 3","ke f","ale igh","L uke","Ġpsych o","ator ium","M ED","Ġtreat ies","Ġind isc","Ġd c","OP S","Ġresil ient","ĠInter state","Ġsl ack","Ġmund ane","Ġestab lishes","35 9","Ġstr ained","Ġn ond","S us","Ġcast e","ar ate","ie ving","Ġunfair ly","Ġpars er","on ial","urs ive","V ia","ĠOtt o","ĠAuthor ities","stro ke","K R","ĠMer cy","Ġfurn ished","Ġout set","Ġmet ic","19 82","olith ic","ĠT ent","og ical","ĠA ircraft","Ġh ides","ĠBec ame","Ġeduc ators","re aching","Ġvol atility","Ġtodd ler","ĠNAS CAR","ĠTw elve","ĠHigh lights","Ġgra pe","Ġspl its","Ġpe asant","Ġre neg","ĠMS I","Tem p","st ars","Ġtre k","ĠHy de","b inding","Ġreal ism","Ġox ide","ĠH os","Ġmount s","Ġbit ing","Ġcollaps ing","Ġpost al","Ġmuse ums","Ġdet ached","Ġrespect ing","Ġmonop ol","Ġwork flow","ĠC ake","Tem plate","ĠOrgan isation","Ġpers istence","36 9","C oming","B rad","Ġredund ant","ĠG TA","Ġb ending","Ġrev oked","Ġoff ending","Ġfram ing","Ġprint f","Comm un","mem bers","Out side","Ġconst rued","Ġc oded","F ORE","Ġch ast","Ch at","Ind ian","ĠY ard","? !\"","ĠP orts","ĠX avier","ĠR ET","' .\"","ĠBo at","iv ated","ich t","umer able","D s","ĠDun n","Ġcoff in","Ġsecure ly","ĠRapt ors","ĠB es","Install ation","Ġin ception","ĠHealth y","end ants","Ġpsych ologists","ĠShe ikh","c ultural","ĠBlack Berry","sh ift","F red","oc he","Ġc akes","ĠS EO","ĠG ian","ĠAs ians","og ging","e lement","Ġpund its","ĠV augh","ĠG avin","Ġh itter","Ġdrown ed","Ġch alk","ĠZ ika","Ġmeas les","80 2","â̦ ..","ĠAW S","] \"","Ġdist ort","ĠM ast","Ġantib odies","ĠM ash","Mem ory","ĠUg anda","ĠPro b","Ġvom iting","ĠTurn s","Ġoccup ying","Ġev asion","ĠTher apy","Ġprom o","Ġelect r","Ġblue print","ĠD re","pr iced","ĠDep ot","Ġallev iate","ĠSom ali","m arg","n ine","Ġnostalg ia","ĠShe pherd","Ġcaval ry","Ġtor ped","ĠBlood y","x b","Ġs ank","Ġgo alt","report print","embed reportprint","clone embedreportprint","ĠIn itially","ĠF ischer","Ġnot eworthy","c ern","Ġin efficient","raw download","rawdownload cloneembedreportprint","c ation","ĠD ynasty","l ag","D ES","Ġdistinct ly","ĠEston ia","Ġopen ness","Ġg ossip","ru ck","W idth","ĠIb rahim","Ġpet roleum","Ġav atar","ĠH ed","ath a","ĠHog warts","Ġc aves","67 8","Ġsafegu ard","ĠM og","iss on","ĠDur ham","sl aught","ĠGrad uate","Ġsub conscious","ĠEx cellent","ĠD um","---- -","Ġp iles","ĠW ORK","ĠG arn","ĠF ol","ĠAT M","Ġavoid s","ĠT ul","Ġble ak","EL Y","iv ist","light ly","P ers","ĠD ob","ĠL S","Ġins anity","Î µ","atal ie","En large","Ġtw ists","Ġfault y","Ġpir acy","Ġimp over","Ġrug ged","ĠF ashion","Ġs ands","' ?","sw ick","Ġn atives","Ġhe n","ĠNo ise","ãĥ Ĺ","Ġg reens","Ġfree zer","Ġd ynasty","ĠFather s","ĠNew ark","Ġarchae ological","Ġo t","ob ar","Ġblock ade","Ġall erg","L V","Ġdeb it","ĠR FC","ĠMil ton","ĠPress ure","Ġwill ingly","Ġdisproportion ate","Ġopp ressive","Ġdiamond s","Ġbelong ings","19 70","Ġbell s","Ġimperial ism","Ġ2 27","Ġexpl oding","ĠE clipse","Ġ19 19","Ġr ant","Ġnom inations","34 7","Ġpeace fully","ric a","ĠF UCK","Ġvib ration","mal ink","Ġro pes","ĠIv anka","ĠBrew ery","ĠBook er","ĠOw ens","go ers","Serv ices","ĠSn ape","Ġ19 1","39 5","Ġ2 99","just ice","Ġb ri","Ġdisc s","Ġprom inently","Ġvul gar","Ġsk ipping","l ves","Ġtsun ami","37 4","ĠU rug","ĠE id","rec ated","p hen","Ġfault s","ĠStart ed","9 50","Ġp i","Ġdetect or","Ġbast ard","Ġvalid ated","Space Engineers","OUR CE","Ġ( ~","Ġuns ur","Ġaff irmed","Ġfasc ism","Ġres olving","ĠCh avez","ĠC yn","Ġdet ract","L ost","Ġrig ged","Ġhom age","ĠBrun o","55 5","ec a","Ġpress es","Ġhum our","Ġsp acing","Ġ' /","olk ien","C oun","OP ER","T re","S on","ĠCambod ia","ier re","m ong","o zy","Ġliquid ity","ĠSov iets","ĠFernand o","Ġ2 29","Ġsl ug","ĠCatal an","elect ric","Ġsc enery","ĠH earth","Ġconst rained","Ġgoal ie","ĠGu idelines","ĠAm mo","ĠPear son","Ġtax ed","Ġfet us","Resp onse","ĠAlex is","th ia","G uy","Ġrecon struct","Ġextrem es","Ġconclud ing","ĠP eg","ook s","Ġded uctions","R ose","Ġground breaking","ĠT arg","ãĥ ģ","ĠRe ve","res ource","Ġmo ons","Ġelectrom agnetic","Ġamid st","ĠVik tor","N ESS","B ACK","Ġcomm ute","ĠAna heim","Ġfluct uations","6 40","Ġnood les","ĠCop enhagen","ĠT ide","ĠGri zz","ĠS EE","Ġpip elines","Ġsc ars","end o","ag us","ĠE TF","/ #","ĠBec ome","44 8","Ġvis c","ĠRecomm ended","Ġj umper","Ġcogn ition","Ġassass in","Ġwitness ing","ĠSet up","Ġl ac","v im","IS M","p ages","SS L","35 8","Ġad ject","indust rial","l ore","cher y","Ġgl itter","Ġc alf","Flor ida","Ġspoil ers","Ġsucceed s","Ġch anting","Ġslog ans","ĠTr acy","Vis it","rol ogy","Ġm ornings","Ġline age","Ġs ip","Ġintense ly","Ġflour ish","ĠSle eping","ĠF em","or por","ĠK lan","ĠDar th","h ack","ĠNi elsen","Ġtum ors","Ġprocure ment","ĠY orkshire","Ġra ided","K Y","An na","Ġ// [","ĠDis order","ĠMust ang","ĠW en","ĠTry ing","s q","Ġdeliver ies","Ġshut ter","Ġcere bral","Ġbip olar","ĠC N","l ass","j et","Ġdeb ating","> :","Ġe agle","gr ades","ĠD ixon","UG C","M AS","ĠDr aco","ĠMach ines","aff er","Ġem an"," ²","pr on","ĠG ym","Ġcompar atively","ĠTrib unal","PR O","Ġle x","Ġfert ile","Ġdep ressing","Ġsuperf icial","ess ential","ĠHun ters","g p","Ġprom inence","L iber","ĠAn cest","ote chnology","Ġm ocking","ĠTra ff","ĸ ļ","Med ium","I raq","Ġpsychiat rist","Quant ity","ĠL ect","Ġno isy","5 20","G Y","Ġsl apped","ĠM TV","Ġpar a","p ull","Mult iple","as her","Ġn our","ĠSe g","Spe ll","v ous","ord ial","Sen ior","ĠGold berg","ĠPl asma","ne ed","Ġmess enger","ere t","Ġteam ed","Ġliter acy","ĠLe ah","ĠD oyle","Ġem itted","U X","Ġev ade","Ġm aze","Ġwrong ly","ĠL ars","Ġstere otype","Ġpled ges","Ġarom a","ĠM ET","Ġac re","ĠO D","Ġf f","Ġbrew eries","ĠH ilton","und le","ĠK ak","ĠThank fully","ĠCan ucks","in ctions","ĠApp ears","Ġco er","Ġundermin ed","ro vers","And re","Ġbl aze","um ers","Ġfam ine","amp hetamine","ulk an","Am ount","Ġdesper ation","wik ipedia","develop ment","ĠCor inth","uss ia","Jack son","L I","N ative","R s","Oh io","ĠKath leen","F ortunately","Ġattend ant","ĠPre ferred","ĠDid n","ĠV s","M is","Ġrespond ent","Ġb oun","st able","Ġp aved","Ġunex pl","ĠChe ney","L M","ĠC ull","bl own","Ġconfront ing","oc ese","serv ing","W i","ĠLith uania","ann i","Ġst alk","h d","Ġv ener","AP H","ynchron ous","UR R","um ably","hist oric","H alf","H ay","Ġresil ience","spe ction","Ġabandon ing","O bs","ĠDeb bie","Ġgrad ient","ĠPl aint","ĠCan al","AR CH","Ġexpans ive","Ġfun g","Ġb ounced","U nd","Ġprec autions","Ġclar ification","Ġd agger","Ġgri ps","Ġ µ","ĠRiver a","ĠUnd ead","is ites","ĠFIR ST","ñ o","aud i","Ġhost ages","Ġcompl iant","Ġal umni","Se ven","Ġcyber security","e ither","Col lect","Ġinvari ably","ĠS oci","Ġlaw maker","Ġa le","ĠPerson ally","N azi","Ġcustom ization","ĠPro c","ĠSask atchewan","eat uring","Ġsp ared","Ġdiscontin ued","Ġcomput ational","ĠMotor ola","Ġsuprem acist","government al","Ġparad ise","ĠDown ing","ĠNik on","Ġcat alyst","ber ra","Tor onto","8 75","bet a","ĠMac ron","Ġunreal istic","ve ctor","ĠVeh icles","it iveness","ĠR V","ĠCol bert","s in","o ji","ent in","ĠKr ish","hell o","ff ield","ok y","ĠT ate","Ġmap le","Ġa ids","chem ical","33 4","n uts","ĠWar p","Ġx x","ĠRob b","umer ous","_- _","ft ime","ĠV W","Ġw inger","ĠD ome","t ools","ĠP V","ĠGe orgetown","Ġg eared","Ġjihad ists","Ġc p","Ġster oids","M other","cler osis","ĠDR M","nes ia","Ġl inger","Ġimm ersive","ĠC OUN","Ġoutwe igh","ens ual","B and","Ġtransform s","mat ched","ps ons","ĠJud icial","f actor","Ġrefer ral","Ġodd ly","ĠW enger","B ring","ĠB ows","60 2","IC LE","Ġl ions","ĠAcad emic","ĠTh orn","ĠRa ider","kef eller","St orage","L ower","ĠOr t","ĠEqu ality","AL T","ĠS OC","T ypes","Ġl yn","ĠAss et","co at","TP P","C VE","ĠPione er","app lication","Mod ern","ĠH K","En vironment","Al right","R ain","IP P","ĠShi ite","Ġm ound","ĠAb ilities","cond ition","St aff","Ġcompet ence","ĠM oor","ĠDi ablo","Ġwith held","Ġost ensibly","ĠB rom","Ġms g","Ġden omin","ĠRef erences","ĠF P","Ġplun ged","Ġp amph","m oving","cent ral","Ġdown right","Ġf ading","T al","T yp","ĠTh y","uk es","it he","Ġo ve","Ġbatt led","Ġseaf ood","Ġfig ur","ĠR D","c rop","Ġsqu ads","{ \\","à ¹","ĠE h","Ġinterview ing","ĠQ in","Ġas piring","PL IC","Ġcla uses","ĠG ast","ĠN ir","Ġl uggage","Ġh ose","Ġsystem d","Ġdesc ending","ĠRev ised","ĠR ails","al ign","70 9","33 7","Ġf ug","charg ing","t ags","Ġut er","k ish","WAR NING","49 0","prof its","Ġvoy age","Ġa ce","ĠV anguard","ĠT anks","ĠM uk","Ġ2 26","S afe","Ar mor","Ġvolcan ic","Ġwom b","ĠM IL","Ġbegin ner","ĠRec ogn","ĠA AP","PL AY",") !","Ġdetect ing","c n","Ġbre aches","Bas ically","ĠP ag","ĠMunicip al","ĠInd ie","ĠL af","ĠDis able","ĠOl son","Ġrest rained","Ġrul ings","Ġhum ane","ev ents","ĠCinem a","display Text","ĠH atch","action Date","onna issance","Ġassault ing","ĠL ug","CH AT","Ġvig orous","ĠPer se","Ġintoler ance","ĠSnap chat","ĠSh arks","Ġd ummy","ĠDi agn","ĠGu itar","im eters","40 3","RE G","A x","Ġsepar ates","ĠMah m","Ġt v","j ah","O OL","C irc","ĠWinds or","uss ian","Ġintu ition","Ġdis dain","ĠDon ovan","Ġ2 21","E mb","Ġcondem ning","Ġgener osity","zz y","Ġpant ies","ĠPre vent","Action Code","AN A","34 2","external ActionCode","Ġspec ifying","Ġcryst all","J ere","Ġru pt","ĠApp rentice","Ġprof iling","Ð º","St rike","Ġsid eline","Ġoblig ated","Ġocc ult","Ġbureaucr atic","ant ically","rupt ed","neg ative","ĠEthiop ia","ĠC ivic","Ġins iders","el igible","ĠTV s","ĠB AR","ĠT I","i ologist","ĠA IR","Ġsubstit uted","Ar ab","ĠS aul","ĠY og","p rem","Ġbuild ers","Ġstation ary","Ġdoubt ful","Ġvig orously","Ġthr illing","Ph ysical","ĠCare y","ĠHyd ra","geon ing","ĠS ly","y ton","Ġborrow ers","ĠPark inson","Ġ ë","ĠJama ica","Ġsat ir","Ġinsurg ents","ĠF irm","Ġis ot","ĠK arn","our ning","ak ens","doc s","l ittle","ĠMon aco","CL ASS","Tur key","L y","ĠCon an","ass ic","Ġstar red","ĠPac ers","et ies","Ġt ipping","M oon","ĠR w","s ame","Ġcav ity","Ġgo of","ĠZ o","Sh ock","um mer","Ġemphas izes","Ġreg rett","Ġnovel ty","Ġen vy","ĠPass ive","r w","50 5","Ġind ifferent","ĠR ica","ĠHim self","ĠFred die","Ġad ip","ä¸ Ģ","Ġbreak out","Ġhur ried","ĠHu ang","ĠD isk","Ġro aming","?????- ?????-","U V","ĠRick y","ĠS igma","Ġmarginal ized","Ġed its","Ġ30 4","mem ory","Ġspec imen","29 3","ãģ ¯","Ġvert ically","Ġaud ition","ĠHe ck","Ġc aster","ĠHold ings","ad al","ĠC ron","ĠL iam","Ġdef lect","P ick","ĠDeb ug","RE F","Ġvers atility","ot hes","class ified","ĠMah ar","ĠH ort","C ounter","st asy","not iced","33 1","ĠSh im","f uck","ĠB ie","Ġair ing","ĠPro tein","ĠHold ing","Ġspect ators","ili ated","ĠThat cher","n osis","ãĥ¼ ãĥ³","Te le","B oston","ĠTem pl","st ay","Ġdecl arations","47 9","Vol ume","ĠDesign er","ĠOver watch","id ae","Ġon wards","Ġn ets","ĠMan ila","part icularly","Ġpolit ic","o other","Ġport raits","Ġpave ment","c ffff","Ġs aints","Ġbegin ners","ES PN","Ġshort comings","âķIJ âķIJ","Ġcom et","ĠOrgan ic","qu el","Ġhospital ized","Bre ak","Ġpe el","dyl ib","asp x","ur ances","ĠT IM","P g","Ġread able","ĠMal ik","Ġm uzzle","Ġbench marks","d al","ĠV acc","ĠH icks","60 9","ĠB iblical","he ng","Ġover load","ĠCivil ization","Ġimm oral","Ġf ries","ãĤ Ĵ","Ġreprodu ced","Ġform ulation","j ug","ire z","g ear","Ġco ached","Mp Server","ĠS J","ĠK w","In it","d eal","ĠO ro","ĠL oki","ĠSong s","Ġ23 2","ĠLou ise","asion ally","Ġunc ond","olly wood","Ġprogress ives","ĠEn ough","ĠDo e","Ġwreck age","Ġbr ushed","ĠBase Type","Ġz oning","ish able","het ically","ĠC aucus","ĠH ue","Ġk arma","ĠSport ing","Ġtrad er","Ġseem ing","ĠCapt ure","4 30","b ish","Ġt unes","Ġindo ors","ĠSp here","ĠD ancing","TER N","Ġno b","ĠG ST","m aps","Ġpe ppers","F it","Ġoverse es","ĠRabb i","ĠR uler","vert ising","off ice","xx x","Ġra ft","Ch anged","Ġtext books","L inks","ĠO mn","ãĢ ij","Ġinconven ience","ĠDon etsk","= ~","Ġimplicit ly","Ġboost s","ĠB ones","ĠBo om","Cour tesy","Ġsens ational","AN Y","Ġgre edy","ed en","Ġinex per","ĠL er","ĠV ale","Ġtight en","ĠE AR","ĠN um","Ġancest or","S ent","ĠH orde","urg ical","all ah","Ġsa p","amb a","ĠSp read","tw itch","Ġgrand son","Ġfract ure","Ġmoder ator","ĠSe venth","ĠRe verse","Ġestim ation","Cho ose","Ġpar ach","Ġbar ric","ãĢ IJ","Ġcomp ass","Ġall ergic","âĢ ķ","OT HER","err illa","Ġw agon","Ġz inc","Ġrub bed","ĠFull er","ĠLuxem bourg","ĠHoo ver","Ġli ar","ĠEven ing","ĠCob b","est eem","Ġselect or","ĠB rawl","is ance","ĠE k","Ġtro op","Ġg uts","ĠApp eal","ĠTibet an","Ġrout ines","ĠM ent","Ġsummar ized","steam apps","Ġtr anqu","Ġ19 29","or an","ĠAut hent","Ġg maxwell","Ġappre hens","Ġpo ems","Ġsa usage","ĠWeb ster","ur us","Ġthem ed","Ġl ounge","Ġcharg er","Sp oiler","Ġsp illed","h og","ĠSu nder","ĠA in","ĠAng ry","Ġdis qual","ĠFrequ ency","ĠEther net","Ġhel per","Per cent","Ġhorr ifying","Ġa il","ĠAll an","EE E","ĠCross ing","44 9","Ġh olog","ĠPuzz les","ĠGo es","eren n","60 4","ãģ ı","ĠRaf ael","Ġatt en","ĠE manuel","Ġup ro","ĠSus p","P sych","ĠTr ainer","ĠN ES","ĠHun ts","bec ue","Ġcounsel or","R ule","Ġtox ins","Ġb anners","r ifice","Ġgreet ing","Ġfren zy","Ġall ocate","Ġ* )","ex pr","50 3","ĠCh ick","ĠT orn","Ġconsolid ation","ĠF letcher","sw itch","fr ac","cl ips","ĠMcK in","ĠLun ar","Mon th","IT CH","Ġscholar ly","rap ed","39 8","Ġ19 10","Ġe greg","Ġin secure","Ġvict orious","cffff cc","Ġsing led","Ġel ves","ĠW ond","bur st","Ġcam oufl","ĠBL ACK","Ġcondition ed","ç ī","ans wered","Ġcompuls ory","asc ist","Ġpodcast s","ĠFrank furt","bn b","Ġne oliberal","ĠKey board","ĠBel le","w arm","Ġtrust s","Ġins ured","ĠBu cc","us able","60 7","ĠPl ains","Ġ18 90","Ġsabot age","Ġlod ged","f elt","Ġg a","ĠN arc","ĠSal em","Ġsevent y","ĠBl ank","p ocket","Ġwhis per","Ġm ating","om ics","ĠSal man","ĠK ad","Ġan gered","Ġcoll isions","Ġextraord inarily","Ġcoerc ion","G host","b irds","è Ģ","k ok","Ġper missible","avor able","Ġpo inters","Ġdiss ip","ac i","Ġtheat rical","ĠCos mic","Ġforget ting","Ġfinal ized","å¤ §","y out","l ibrary","Ġbo oming","ĠBel ieve","ĠTe acher","ĠL iv","ĠGOOD MAN","ĠDomin ican","OR ED","ĠPart ies","Ġprecip itation","ĠSl ot","R oy","ĠComb ined","Ġinteg rating","Ġch rome","Ġintest inal","ĠRe bell","Ġmatch ups","Ġblock buster","ĠLore n","ĠLe vy","Ġpre aching","ĠS ending","ĠPur pose","ra x","f if","Ġauthor itative","ĠP ET","ast ical","Ġdish on","Ġchat ting","Ġ\"$ :/","Connect ion","Ġrecre ate","Ġdel inqu","Ġbro th","ĠD irty","ĠAd min","z man","Ġscholars hips","Ġ25 3","cont act","als a","7 67","c reen","abb age","Ġ19 15","Ġbl ended","Ġal armed","L anguage","35 6","Ġbl ends","ĠCh anged","W olf","Ġhe pat","Creat ing","Ġper secut","Ġsweet ness","art e","Ġforfe iture","ĠRober to","im pro","N FL","ĠMag net","Det ailed","Ġinsign ificant","ĠPOL IT","ĠBB Q","ĠC PS","Ġse aw","amin er","m L","end if","f inals","Ġ26 5","u ish","Ġ} )","ĠPro blems","Ġem blem","Ġserious ness","Ġpars ing","Ġsubst itution","Ġpress ured","Ġrecy cled","ale b","Rub y","Ġprof iciency","Dri ver","ĠW ester",": '","AF TA","Ġm antle","ĠClay ton","fl ag","Ġpractition er","c overed","ĠSt ruct","add afi","4 25","ĠTown ship","ĠHyd ro","Lou is","34 3","Ġcond o","ĠT ao","Ġutil ization","Ġnause a","ĠDem s","rid ges","p ause","Ġform ulas","Ġchall enger","37 6","Ġdefect ive","ĠRail way","ĠPub Med","Ġyog urt","l bs","ĠNor folk","OP E","ĠMood y","Ġdistribut or","Ġscroll s","Ġextract s","St an","Ġv iability","Ġexp oses","Ġstar vation","ĠStep s","ĠD odd","f ew","ST D","33 2","Ġclos ures","Ġcomplement ary","ĠS asha","ump y","Ġmon et","Ġartic ulate","ĠDo ct","k iller","Ġsc rim","Ġ2 64","Ġprost itutes","Ġse vered","Ġattach ments","Ġcool ed","L ev","ĠF alk","f ail","Ġpolic eman","ĠD ag","Ġpray ed","ĠK ernel","Ġcl ut","Ġc ath","Ġan omaly","St orm","em aker","ĠBreak fast","ul i","o ire","J J","h z","Oper ation","ĠS ick","35 4","ĠGuatem ala","R ate","Ġexp osures","f aces","ĠArch ae","ra f","ĠM ia","Ġ20 25","Ġop aque","Ġdisgu ised","ĠHead quarters","S ah","Ġp ots","9 78","ĠM alf","Ġfrown ed","Ġpoison ous","ĠCon vers","ee ks","Ġcr ab",".\" \"","Ġtre ason","Ġr anc","Ġescal ating","Ġwar r","Ġmob s","Ġl amps","ĠSun shine","ĠBrun swick","Ph ones","Ġspe lled","ĠSk ip","Ġ20 50","Ġ19 11","ĠPl uto","ĠAm end","Ġme ats","38 7","Ġst omp","ĠZh ou","ĠLevi athan","ĠHaz ard","ad v","ĠOr well","Ġal oud","Ġb umper","ĠAn arch","ub untu","ĠSer ious","f itting","ĠOption al","ĠCec il","RE AM","Ġser otonin","Ġcultiv ate","ag ogue","} \\","Ġmos ques","ĠSun ny","Ġre active","rev olution","ĠL up","ĠFed ora","Ġdefense man","ĠV ID","ist ine","Ġdrown ing","ĠBroad casting","Ġthr iller","ĠS cy","Ġacceler ating","Ġdirect s","od ied","b ike","d uration","Ġpain fully","R edd","Ġproduct ions","Ġg ag","Ġwh ist","Ġs ock","Ġinf initely","ĠConc ern","ĠCit adel","Ġlie u","Ġcand les","ogene ous","arg er","Ġheaven ly","inflamm atory","Per formance","C s","ruct ose","az aki","Ġp essim","Ġinf erence","Ġpow d","ĠZ oe","Ġpain ts","Ġd azz","pt a","-------- ---","Ġins pir","ĠExper imental","ĠKn ife","reg or","b ors","Ġshow ers","rom eda","Ġs aint","Ġben ign","ĠJ iang","Ġenvision ed","Ġsh roud","IF T","H O","Ġsh uff","ĠI CC","Ġse greg","Ġrevis it","ighth ouse","L i","Ġsub strate","ĠSe as","ĠRew ard","ĠH ep","ĠBr ass","s bm","Ġelim inates","Ġst amina","ĠV AT","ĠLo an","Ġconst raint","Ġappropri ated","Ġp es","ĠA LE","r anging","Ġ40 4","39 2","Ġintellectual s","ach u","Ġrestruct uring","ĠLe vin","Ġrun es","Ġdelight ful","Ġcarbohyd rates","ĠMod els","ĠExp o","Ġtransport ing","all oc","Ġring ing","S amsung","Ġscarce ly","ĠURL s","ĠM AS","Ġprot otypes","Ġnarr ator","ĠCPU s","cd n","ĠBart on","Ġdecided ly","ĠSh u","ix ir","oc ious","ĠMy st","N intendo","Ġre use","Ġforg iven","F ew","in ical","n at","Ġseam less","ĠEv a","ĠE VE","ĠJ O","land ers","Ġso fter","neg ie","Ġtrans ient","Ġorb ital","Ġfulf il","ĠK om","Hop efully","Ġdynam ically","ĠHun ger","å Ľ","ĠArmen ia","el man","ber to","Ġp ige","ĠID s","lim it","Ġve ins","Ġso aring","p acks","Gold en","ĠCr ab","ist or","ĠR PM","Ġ$ $","g ression","Ġjihad ist","Ġgam ble","Ġcare g","Ġinf lated","F ace","ĠFire arms","ĠEm manuel","â Ŀ","Ġsh ocks","gr ab","Ġspl end","ĠHP V","ab ortion","Ab ove","Ent ity","play ers","Ġcomm enced","ul ence","Ġfulfill ment","Ġembod iments","ĠW elfare","Ġha il","Ġ< @","tt en","Ġcat cher","ĠJ azeera","Ġvolcan o","Ġstabil ize","ĠHand ler","Ġintens ified","ĠAb rams","Ġhum iliation","p aced","60 5","ĠCent OS","Spe cific","Ġhe ed","ĠC AM","ĠGal ile","D ie","Ġabol ished","ĠThom son","ĠTe achers","ĠW ass","j ong","ĠIS BN","ĠAll ies","sh ake","å ·","v ict","How ard","Ġde em","Ġexceed ingly","ĠSmart stocks","ib e","Ġdoor way","Ġcompet ed","ig mat","Ġnational ists","Ġg room","ĠKe en","Ġdispos able","de cl","ĠT olkien","ĠSche me","Ġb iod","Ġav id","ĠEl on","ag ar","ĠT SA","R oman","Ġartific ially","Ġadvis ors","X L","ĠInf erno","36 6","Ġted ious","ĠPhot ography","ĠCar rie","Ġtro pe","ĠSand ra","Ġdec imal","Que en","ĠGund am","ĠO M","ote ch","N BA","Ġ19 32","Ġent renched","ĠMar ion","Ġfr aternity","Lab our","Hen ry","Ġlat itude","E ither","Ġenh ances","ĠPot ential","Ġsh ines","id ad","Ġbread th","Ġcapac ities","ĠðŁ ĻĤ","ĠBron x","Ġsex es","Ġdifferent iation","Ġheavy weight","ĠT aj","d ra","Ġmigr ate","Ġexhaust ion","ĠR UN","els ius","ĠCu omo","Ġgu itars","Ġcl ones","ĠSom ew","ĠP ry","------------ -","Ġwarr anted","cy cles","Ġsalv age","Ġdis ks","R ANT","ĠNGO s","ĠMart ian","\":[ {\"","Ġadd icts","oj ure","il let","Ġamazing ly","art ments","p ixel","ĠGPU s","Lay out","è £","ĠTam il","ĠBas il","Ġimpart ial","ĠSt ructure","f ork","b ryce","Ġr idge","ĠHamb urg","ri ous","Ġbl itz","cig arettes","Ġcan ned","40 2","Ġiron ically","Ġcompassion ate","ĠHaw kins",". #","ĠCat hedral","Ġrall ied","in ternal","Ġqu ota","st akes","T EXT","m om","Ġcomple tes","Ġ23 8","Ġsh rug","ãĥ ij","ĠN inth","Ġrev ise","ĠProv ider","Ġtre acher","Ġqu asi","ĠPR ES","Ġdep osition","Ġconfidential ity","iss ors","Ġim balance","Ġspan ning","Ġang ular","ĠC ul","commun ication","ĠNor a","ĠGen ius","op ter","Ġs acked","Sp ot","Ġfine ly","ĠCH R","28 2","w aves","Pal est","ĠRo hing","N L","è ¿","Ġsh itty","ĠSc alia","4 75","Pro gress","Ġreferen cing","Ġclass rooms","ab ee","Ġs od","hes ion","70 8","ĠZucker berg","ĠFin ish","ĠScot ia","ĠSav ior","ĠInstall ation","an tha","( -","Ġ30 2","ĠP unk","Ġcr ater","yout u","Ġro ast","Ġinflu encing","Ġd up","ĠJ R","ĠG rav","Ġstat ure","Ġbath rooms","A side","W iki","me an","ĠZ ak","ĠOn es","ĠN ath","Ġhyper t","Ġcommence ment","C ivil","Ġmoder ately","Ġdistribut ors","Ġbreast feeding","Ġ9 80","ĠS ik","ĠC ig","ĠAM ER","R IP","ĠCare er","ust ing","Ġmess ed","Ġe h","ĠJ ensen","/ $","Ġblack mail","Ġconvers ions","Ġscientific ally","Ġmant ra","p aying","Ġiv ory","ĠCour ts","OU GH","aunt let","Ser ial","B row","ĠH undreds","3 23","Ġpe e","Ġlin ux","Ġsub mer","ĠPrinc ipal","48 5","ĠD SL","ĠCous ins","Ġdoctr ines","ĠAthlet ics","Ġ3 15","ĠK arma","Ġatt ent","ur ger","Ġpresc ribe","Ġenc aps","ĠC ame","Ġsecret ive","ĠCr imes","d n","C lean","ĠEgypt ians","ĠCar penter","Ġ ll","H um","ĠMil o","Ġcapital ists","Ġbrief ed","T we","ĠBas in","elve t","M os","Ġplun ge","ĠKa iser","ĠFu j","ill in","Ġsafegu ards","Ġo ste","ĠOpportun ity","ĠM afia","ĠCall ing","ap a","ur ban","br ush","ill ard","c é","int elligence","ĠL ob","ĠDru id","Ġsm oother","Ġfoot ing","Ġmotor ists","arc ity","Ġmascul inity","Ġm ism","Ġabdom inal","ĠTa vern","ĠR oh","Ġesc apes","s igned","Anth ony","Ġsacrific ing","Ġintim acy","Ġan terior","ĠK od","Ġmot if","Ġg raz","Ġvisual ization","Ġguitar ist","ĠTro tsky","m agic","D ar","ĠMor i","Ġw ards","Ġtoile ts","l est","Ġtele port","ĠSund ays","ĠPl at","ET S","Ġe Sports","Pat rick","ĠK atherine","en ko","Ġhas sle","ĠM ick","gg les","Ġh ob","aint ain","Ġair borne","Ġsp ans","Ġch ili","Ġa perture","Ġvolunte ered","ĠInc ident","ĠF res","ĠVeter an","augh tered","ing o","Ġun insured","CL OSE","Ġf use","Ġer otic","Ġadvert ise","ra ising","Text ure","Ġatt ends","ĠRE AL","udd led","Ġsm oot","Ġ30 5","ĠWill is","Ġbl ond","An alysis","ĠV T","on ica","Ġstrongh old","R F","N M",". >>","Ġprosper ous","Ġbo asted","29 2","ĠManufact uring","PR ESS","g ren","Ġpharm acy","ĠRoc kefeller","k ai","Ġth umbs","ĠH ut","Ġmother board","Ġguard ians","ĠAl ter","ll ular","Ġsh ack","Ġwise ly","Ġback bone","erv a","Ġsu icides","ĠMcG regor","ij ah","E mer","ĠB rav","Ġdesign ate","P OST","produ ced","Ġcleans ing","irl wind","ex istent","ĠHum ph","ĠPay ne","Ġv ested","Å ¡","Ġstring ent","ion a","Ġuns ub","Ġsum med","ĠHer cules","sub ject","ĠR agnar","ĠN os","Ġcharacter ization","Ġsav vy","ĠDaw son","ĠCas ino","Ġf ri","ĠBar rier","Ġmis information","Ġins ulation","Ġcorrid ors","Ġair planes","ĠNo ct","ah i","Ġ19 16","k b","arm ac","Ġsh un","Ġsche ma","Ġhorr ified","Ġ23 9","aund ers","N B","i ates","er ity","ĠSh ard","Ġr arity","Ġgroup ed","ĠGh ana","again st","ĠBi ological","ĠA ware","ow ell","Ï Ħ","ĠBe au","sh aw","H ack","ĠJul ius","US S","ol son","aun a","c ru","ĠMaur ice","ĠI k","Ġsequ encing","Ġradical s","Ġ( ?,","v irtual","Ġany ways","Ġreper c","Ġhand lers","Ġhes itant","é ĥ","ĠM F","ple mentation","ass ociated","Ġcampaign ed","ĠY ue","ut ations","ĠY oga","Ġsim mer","Ġro ds","Ġmel ody","Ġconv oy","v ideos","Ġscreen ed","N eg","ochem ical","Ġ( ))","Ġultr as","Ġant ip","ĠIsland ers","70 4","Ġfet ish","Ġridic ulously","ĠK art","Ġmitochond rial","Ġinterf ering","Build er","Ġover fl","Ġac ne","ĠM ud","ĠK err","f lex","ĠPost al","ĠBalt ic","47 7","ĠPers ons","our age","H B","ĠM use","ĠImm ortal","ĠDri ving","Ġpet itions","Ġsubsc ript","Ġs orce","ĠProcess or","ut on","S ony","Ġph on","Ġr aced","ĠAnth rop","Ġday time","ĠEx ercise","Add ing","Ġeng ages","ĠQual comm","Ġmir acles","Ġmem es","ĠDr ink","ĠOri oles","Ġhair s","ĠPol ar","ath om","Ġsl ippery","ĠR emy","Ġcar amel","ĠY EAR","Ġal k","I gn","a ution","ĠMer lin","ĠC ran","Ġap ologies","Ġ4 10","Ġout ing","ĠMem ories","app ointed","Ġcount ered","u ld","pos ing","Ġfire wall","ĠW ast","ĠW et","work ed","se ller","Ġrepe aled","ere o","ass uming","BL IC","m ite","ĠCEO s","ĠChap el","ellig ent","________________ ________","D og","Ġw art","Ġsubsc riber","s ports","Ġbe gged","ĠM V","Ġsem if","eth ical","Ġpre ach","Ġrev ital","Ġpun itive","Ġshort cuts","Ġinstit uted","ĠWars aw","Ġabdom en","ĠK ING","Ġsuper intendent","Ġf ry","ĠGe o","T OR","Ġcontrad ictions","apt ic","Ġlandsc apes","b ugs","Ġcl ust","Ġvol ley","c ribed","Ġt andem","Ġrob es","WH AT","Ġpromot er","Ġel oqu","review ed","ĠD K","ĠPl ato","Ġf ps","T ank","ĠDer rick","Ġpriorit ize","as per","ĠHond uras","ĠCom pleted","ne c","Ġm og","n ir","ĠMay o","DE F","st all","in ness","ĠVolks wagen","Ġprec aution","ĠM ell","i ak","ist ries","Ġ24 8","Ġoverl apping","Sen ate","ĠEnh ance","res y","rac ial","OR TS","ĠM ormons","Str ong","ĠCo ch","Mex ico","ĠMad uro","Ġj ars","Ġcan e","W ik","oll a","iff erence","Ġphysic ist","ĠMag gie","Ġ28 5","Ġdep iction","ĠMcL aren","J u","Ġsl ows","Ġcommission ers","ĠWill ow","ĠExpl os","hov ah","Ġtechn ician","Ġhom icides","ĠFl av","ĠTr uman","Ġ100 00","u ctor","Ġsh ader","News letter","45 7","Ġre ver","Ġhard ened","Ġwhere abouts","Ġrede velop","Ġcar bs","Ġtra vers","Ġsqu irrel","Ġfoll ower","Ġs ings","50 8","Ġrabb its","emon ium","Ġdocument ing","Ġmisunder stood",") '","R ick","gg ies","Ġprem ie","Ġsk ating","Ġpass ports","Ġf ists","aged don","H aw","AC P","0 80","ĠThough ts","ĠCarl son","Ġpriest hood","h ua","Ġdun geons","ĠLo ans","Ġant is","Ġfamiliar ity","ĠS abb","op al","ĠIn k","st rike","Ġc ram","Ġlegal ized","Ġcu isine","Ġfib re","Tra vel","ĠMon ument","OD Y","eth y","Ġinter state","ĠP UR","em porary","ĠArab ian","develop ed","Ġsadd le","Ġg ithub","ĠOff er","ĠIS P","ro let","ĠSUP ER","ĠDen is","Ġmultipl ier","Ġstir red","Interest ingly","Ġcustom ary","Ġbill ed","he x","Ġmultipl ied","Ġfl ipping","ĠCros by","Ġfundament als","ia e","ĠPlay ed","ĠAt om","am azon","ĠFl am","ee z","activ ated","Ġtables poon","Ġliberal ism","ĠPal in","ĠP atel","N um","ĠT AM","Ġs urn","ĠRel oaded","Ġco ined","\" ],","ĠCl ash","ĠAg u","Ġprag matic","ĠActiv ate","Ġ8 02","Ġtrail ers","Ġsil hou","Ġprob es","Ġcirc us","ĠB ain","ĠLind say","ĠAb bey","Del ivery","Ġconcess ion","Ġgast ro","ĠSpr ite","Ä Ł","and el","Ġg imm","Ġaut obi","ĠT urtle","Ġwonder fully","ĠHar am","ĠWorld wide","ĠHand le","Ġtheor ists","Ġsle ek","ĠZh u","ograph ically","EG A","ĠOwn ers","ath s","ĠAntar ctic","n atal","=\" \"","fl ags","`` ``","Ġs ul","K h","Ġpot assium","Ġlinem an","Ġcere al","ĠSe asons","Ġ20 22","Ġmat hematic","Ġastron omers","prof essional","Ġf ares","cknow led","Ġch i","Ġyoung sters","Ġmistaken ly","Ġhem isphere","ĠDiv inity","r one","Ġ\" ,","r ings","Ġattract s","v ana","å ¹","C AP","Ġplay list","Ġpor ch","ãģ £","Ġincorpor ates","Ġso ak","Ġassert ing","ĠTerror ism","ĠP ablo","J a","ces ter","Ġfear ing","ĠPr ayer","Ġescal ated","G W","Ġro be","ĠBright on","ac ists","ĠSym phony","ĠDwar f","ĠPar ade","ĠLe go","Ġinex pl","Ġl ords","le af","RA G","l iber","Ġcig ars","ĠJe hovah","60 6","WIND OWS","ĠLiber ia","eb us","He avy","Ġl ubric","ĠR W","angu ages","Ġnarrow ed","com puter","ĠE mber","Ġmurder ing","Ġdown stream","ĠT uls","ĠT ables","Top ic","ĠAcc uracy","= /","l ost","ĠRe i","Ġprogress es","b ear","Ġestablish ments","Just in","ĠPe ach","ĠG omez","å ¿","ĠTri angle","Id ent","ĠH ive","Res ources","Ġmix es","ĠAss uming","M u","Ġhyp oc","Ġs ane","ĠW an","id ious","Su ccess","Ġ io","Ang el","Ġdanger ously","ĠCreat ure","W ORK",": [","ĠKat rina","List ener","M iller","ĠId lib","h ang","Ġcircum vent","h ref","Ġcel estial","ĠWe eks","ĠP ug","ĠDal ton","Ġsubpoen a","uk u","Ġpers isted","pe i","old ing","ĠDoc uments","ĠH ast","ĠC ENT","Ġprim er","Ġsyn onymous","Ġn ib","om bs","Ġnot ation","ĠD ish","ĠAt mosp","Ġforb id","ĠAN G","pat tern","l os","Ġproject iles","b rown",".\" ,","ĠVen om","Ġfierce ly","ub lished","ĠU ran","ĠNic arag","4 10","ĠC AL","OT OS","ĠMir acle","ĠEn chant","Ġguard ing","app end","Att ach","Ġlevel ed","Ġcond oms","ih ilation","64 9","Ġnight mares","ĠTHE Y","ĠST ART","ĠK inn","Ġroomm ate","Ġhy giene","o pping","J ob","Ġl vl","ĠV ER","ĠKe eping","ab etic","Ġformat ting","eral a","Ġrev isions","Ġres urg","T el","ĠGood man","35 3","p od","Ġind isp","ĠTrans lation","Ġg own","ĠM und","Ġc is","Ġby stand","col lect","ĠPun jab","act ively","ĠG amb","te ll","Ġimport ing","g encies","Ġloc om","ĠBr ill","H oly","ĠBer ger","Ġshow down","Ġrespond ers","IL Y","Ġt akedown","le ted","Ġmat tered","Ġpredict ive","Ġover lay","G PU","ĠV ick","Ġconvey ed","T ab","pe er","Sc an","Ġdefensive ly","v ae","Ġappro ving","Ġt iers","ĠV ia","quer ade","ĠSaud is","Ġdemol ished","ĠProp he","Ġmon o","Ġhospital ity","H AM","ĠAri el","M OD","ĠTor ah","Ġbl ah","ĠBel arus","erent ial","ĠT uc","Ġbank er","39 7","Ġmosqu it","ĠScient ist","ĠMus ical","Ġh ust","Sh ift","Ġtor ment","Ġstand off","E duc","ĠF og","Ġampl ifier","Sh ape","Inst ance","ĠCrit ics","Ġda emon","H ouston","Ġmatt ress","ĠID F","Ġobsc ene","ĠA mer","hett i","Ġcomp iling","35 2","vere tt","ĠRed uction","ist ration","ĠBl essed","ĠB achelor","3 16","Ġpr ank","ĠVul can","dd ing","Ġm ourning","ĠQu int","ĠBl aster","test ing","Ġsed iment",">> >","ĠE ternity","ĠWH ERE","ĠM aze","Ġreact ing","ĠAl v","oms day","ĠC RA","Ġtransl ator","Ġbog us","at u","We bsite","oll s","Ġbapt ism","Ġs ibling","ĠAut umn","ve z","ãģ® é","gu ards","Ge org","assad ors","ĠFre ud","Ġcontin ents","ĠReg istry","Bern ie","ĸļ 士","Ġtoler ant","ĠU W","Ġhor ribly","99 5","ĠMID I","Ġimpat ient","oc ado","er i","ĠWor st","ĠNor ris","ĠTalk ing","Ġdef ends","ens able","Ġ20 21","Ġanat omy","L ew","Ġdraw er","ĠCan berra","Ġpatri otic","é¾įå ĸļ士","ĠAv g","AR M","Ġundis closed","Ġfare well","45 9","b able","ĠAll ison","OL OG","Ġcon co","t ight","ĠAC PI","ĠM ines","l ich","ĠâĶ ľ","represent ed","200 000","Ġenthusi ast","OT S","b il","ĠIng redients","Ġinvent or","ĠMy SQL","³³ Âł","ĠAB OUT","with in","Ġm k","B ul","ĠF ake","Ġdracon ian","W a","hel m","ĠTer ran","erv ille","Ġcommon place","SI ZE","Ġ\" <","re place","ograph s","ĠSE LECT","inc ible","ĠMost ly","ĠShe ffield","ĠID E","ugg le","Ġcit ations","h urst","ĠUn ix","Ġunle ash","ĠP iper","ĠN ano","Ġsucc umb","Ġreluct ance","Ġ25 00","ĠMer chant","Ġwire t","Ġcomb os","ĠBirth day","Ġchar coal","ĠU PS","ĠFair fax","Ġdrive way","ĠT ek","ĠP itch","ove re","Ġtechn icians","ĠAct ual","fl ation","ĠF iscal","ĠEm pty","an amo","Ġmag nesium","Ġsl ut","Ġgrow ers","Invest igators","( ):","ĠS atellite","ĠKe ynes","miss ive","l ane","Ġb orough","3 44","ĠTE AM","ĠBet hesda","C V","h ower","ĠR AD","Ġch ant","ĠR iy","Ġcompos itions","Ġmild ly","Ġmedd ling","Ġag ility","ane ers","5 01","Ġsyn th","ling er","29 1","Ġex claimed","Part y","Ġcont amin","ĠMan or","ĠResp ond","Ġpra ising","Ġman ners","fle et","Sum mer","ĠLy nd","ĠDef initely","gr im","Ġbow ling","st ri","ç Ľ","y nt","Ġmand ates","D IV","Ġreconc ile","view s","ĠDam on","vet te","F lo","ĠGreat est","il on","ic ia","Ġportray al","Ġcush ion","50 4","19 79","oss al","App lic","sc ription","Ġmit igation","AT S","p ac","Ġer ased","Ġdefic iencies","ĠHolland e","ĠX u","Ġb red","Ġpregn ancies","f emin","Ġem ph","Ġpl anners","Ġout per","utter ing","Ġperpet rator","Ġm otto","ĠEll ison","ĠNE VER","Ġadmitted ly","AR I","ĠAzerbai jan","Ġmill isec","Ġcombust ion","ĠBott le","ĠL und","ĠP s","ĠD ress","Ġfabric ated","Ġbat tered","Ġs idel","ĠNot ting","Fore ign","ĠJer ome","0 20","ĠAr bit","Ġkn ots","ĠR IGHT","M oving","ãģ Ļ","Ġsur geries","Ġcour thouse","Ġm astered","Ġhover ing","ĠBr an","ĠAl ison","Ġsaf est","m ilitary","Ġbull ied","Ġbar rage","Read er","ES E","ĠGe ographic","T ools","3 14","ĠGe ek","ro th","gl ers","ĠF IN","Ï ģ","ĠA ston","al tern","48 8","Ġveter in","G amer","Ġint el","ren ches","Sh ield","Ġam nesty","ĠB har","Ġp iled","Ġhonor able","ĠInst itutes","Ġso aked","Ġcom a","ĠE FF","34 1","by tes","ĠG mail","le in","ĠCanad iens","m aterial","I l","Ġinstruct ors","ĠK Y","Ġconce ive","ub b","ĠP ossible","Ġeas ing","ĠChrist ina","Ġcar ic","ĠHD R","R OM","Ġsho vel","de lete","Ġp uff","ĠCh anging","Ġseam lessly","Att ribute","Ġacqu isitions","ak ery","ĠE F","Ġaut istic","ĠT akes","ĠPow der","ĠSt ir","5 10","ĠBub ble","sett ings","ĠF owler","Ġmust ard","Ġmore over","Ġcopyright ed","ĠLED s","15 00","æ ī","ĠH IS","en f","Ġcust od","ĠH uck","G i","Ġim g","An swer","C t","j ay","ĠInf rastructure","Ġfeder ally","L oc","Ġmicro bes","Ġover run","dd s","ot ent","adi ator",">>>> >>>>","Ġtorn ado","Ġadj ud","Ġintrig ued","Ġs i","ĠRevel ation","pro gress","Ġburgl ary","ĠSai yan","ĠK athy","Ġser pent","ĠAndre as","Ġcomp el","ess ler","ĠPl astic","ĠAd vent","ĠPos itive","ĠQ t","ĠHind us","reg istered","ular ity","Ġrighteous ness","Ġdemon ic","u itive","ĠB DS","ĠGre gg","c ia","ĠCrus ade","ĠSina i","W ARE","+ (","Ġme ll","Ġder ail","y ards","A st","Ġnotice ably","ĠO ber","R am","Ġun noticed","Ġse q","av age","T s","Ġ6 40","Ġconced e","Ġ] )","F ill","Ġcapt ivity","ĠImprove ment","ĠCrus ader","ara oh","M AP","æ Ĺ","Ġstr ide","al ways","F ly","N it","Ġal gae","ĠCook ing","ĠDo ors","Mal ley","Ġpolic emen","ãģ į","Ġastron aut","access ible","49 5","ĠR AW","cl iffe","udic rous","Ġdep ended","al ach","Ġvent ures","ra ke","Ġt its","ĠH ou","Ġcond om","ormon al","Ġind ent","Ġupload ing","Foot note","Import ant","Ġ27 1","Ġmind ful","Ġcont ends","C ra","Ġcal ibr","ĠO ECD","plug in","F at","ĠIS S","ĠDynam ics","ans en","68 6","' ),","Ġsp rite","Ġhand held","ĠH ipp","=~ =~","Tr ust","Ġsem antics","ĠBund es","ĠRen o","ĠLiter ature","s ense","G ary","ĠA eg","ĠTr in","EE K","Ġcler ic","ĠSS H","Ġch rist","Ġinv ading","ib u","Ġen um","aur a","Ġal lege","ĠInc redible","B BC","Ġth ru","Ġsa iled","Ġem ulate","Ġin security","Ġc rou","Ġaccommod ations","Ġincompet ent","Ġsl ips","ĠEarth qu","s ama","IL LE","Ġi Phones","as aki","Ġby e","Ġar d","Ġext ras","Ġsl aughtered","Ġcrowd funding","res so","Ġfil ib","ĠER ROR","ĠT LS","e gg","ĠIt al","Ġen list","ĠCatal onia","ĠSc ots","Ġser geant","Ġdiss olve","N H","Ġstand ings","ri que","I Q","Ġbenef iciary","Ġaqu arium","You Tube","ĠPower Shell","Ġbright est","ĠWar rant","S old","Writ ing","Ġbegin nings","ĠRes erved","ĠLatin os","head ing","Ġ4 40","Ġrooft op","AT ING","Ġ3 90","VP N","G s","k ernel","turn ed","Ġprefer able","Ġturn overs","ĠH els","S a","ĠShin ji","ve h","ĠMOD ULE","V iol","Ġex iting","Ġj ab","ĠVan illa","Ġac ron","ĠG ap","ber n","A k","ĠMc Gu","Ġend lessly","ĠFar age","ĠNo el","V a","M K","Ġbr ute","ĠK ru","ĠES V","ĠOl ivia","âĢ ł","ĠK af","Ġtrust ing","Ġh ots","3 24","Ġmal aria","Ġj son","Ġp ounding","ort ment","Count ry","Ġpostp oned","Ġunequ iv","? ),","ĠRo oney","udd ing","ĠLe ap","ur rence","sh apeshifter","ĠH AS","os ate","Ġca vern","Ġconserv atism","ĠB AD","Ġmile age","Ġarrest ing","V aults","Ġmix er","Dem ocratic","ĠB enson","Ġauth ored","8 000","Ġpro active","ĠSpirit ual","t re","Ġincarcer ated","ĠS ort","Ġpe aked","Ġwield ing","re ciation","×Ļ ×","P atch","ĠEm my","Ġex qu","tt o","ĠRat io","ĠP icks","ĠG ry","ph ant","Ġf ret","Ġeth n","Ġarch ived","% -","c ases","ĠBl aze","Ġim b","c v","y ss","im ony","Ġcount down","Ġaw akening","ĠTunis ia","ĠRe fer","ĠM J","Ġun natural","ĠCar negie","iz en","ĠN uggets","he ss","Ġev ils","64 7","Ġintrodu ctory","l oving","ĠMcM ahon","Ġambig uity","L abel","ĠAlm ighty","Ġcolor ing","ĠCl aus","set ting","N ULL","ĠF avorite","ĠS IG","> (","ĠSh iva","ĠMay er","Ġstorm ed","ĠCo verage","we apons","igh am","Ġun answered","Ġle ve","Ġc oy","c as","b ags","as ured","Se attle","ĠSant orum","ser ious","Ġcourage ous","ĠS oup","Ġconfisc ated","Ġ// /","Ġuncon ventional","Ġmom s","ĠRohing ya","ĠOrche stra","ĠPot ion","Ġdisc redit","ĠF IL","f ixed","ĠDe er","do i","ĠDim ension","Ġbureaucr ats","et een","Ġaction Group","oh m","Ġb umps","ĠUt ility","Ġsubmar ines","ren heit","re search","ĠShap iro","Ġsket ches","Ġde ceptive","ĠV il","es ame","ĠEss entially","Ġramp age","isk y","Ġmut tered","th ritis","Ġ23 6","f et","b ars","Ġpup il","ĠTh ou","o S","s ong","Ġfract ured","Ġre vert","pict ure","Ġcrit erion","us her","Ġreperc ussions","ĠV intage","ĠSuper intendent","Offic ers","Ġflag ged","Ġbl ames","Ġin verse","ograp hers","Ġmakes hift","Ġdev oid","Ġfoss ils","ĠArist otle","ĠFund s","Ġde pleted","ĠFl u","ĠY uan","Ġw oes","Ġlip id","Ġsit u","requ isites","Ġfurn ish","ĠSam ar","Ġshame ful","Ġadverse ly","Ġad ept","Ġrem orse","Ġmurder ous","uck les","ĠE SL","Ġ3 14","s ent","Ġred ef","ĠC ache","ĠP urs","ig ans","Ġ4 60","Ġpres criptions","Ġf res","F uck","ocr ates","Tw enty","ĠWe ird","ĠT oggle","ĠC alled","itiz ens","Ġp oultry","Ġharvest ing","ãĤ¦ ãĤ¹","Bott om","Ġcaution ed","t n","39 6","ĠNik ki","Ġeval uations","Ġharass ing","Ġbind ings","ĠMon etary","Ġhit ters","Ġadvers ary","un ts","Ġset back","Ġenc rypt","ĠC ait","Ġl ows","eng es","ĠN orn","Ġbul bs","Ġbott led","ĠVoy ager","3 17","Ġsp heres","p olitics","Ġsubt ract","Ġsens ations","Ġapp alling","Ġ3 16","Ġenvironment ally","ĠST EM","Ġpub lishes","5 60","Ġdilig ence","48 4","Ġadv ises","Ġpet rol","Ġimag ining","Ġpatrol s","ĠInt eger","ĠAs hes","act us","ĠRad iant","ĠL T","it ability","ht aking","Set ting","Ġnu anced","ĠRe ef","ĠDevelop ers","N i","pie ces","99 0","Lic ense","Ġlow ers","ĠOtt oman","3 27","oo o","Ġqu itting","mark ets","Beh ind","Ġbas in","Ġdoc s","an ie","fl ash","ct l","Ġcivil ized","ĠFuk ushima","\"] ,\"","ĠK S","ĠHonest ly","ar at","Ġconstruct s","ĠL ans","ĠD ire","ĠLI KE","ĠTrou ble","Ġwith holding","ĠOb livion","Ġsan ity","any a","Con st","Ġgro cer","ĠC elsius","Ġrecount ed","ĠW ife","B order","ate red","h appy","Ġspo iler","Ġlog ically","H all","Ġsucceed ing","Ġpoly morph","Ġax es","ĠShot gun","ĠS lim","ĠPrin ciples","ĠL eth","art a","Ġsc or","Sc reenshot","Ġrelax ation","#$ #$","Ġdeter rent","idd y","Ġpower less","Ġles bians","Ġch ords","ĠEd ited","se lected","Ġseparat ists","000 2","Ġair space","Ġturn around","Ġc unning","P ATH","P oly","Ġbomb ed","Ġt ion","x s","Ġwith hold","Ġw aged","ĠLiber ties","Fl ag","Ġcomfort ing","45 4","ĠI ris","are rs","Ġr ag","Ġrel ocated","ĠGu arant","Ġstrateg ically","Ġgam ma","uber ty","ĠLock heed","g res","Ġgr illed","ĠLow e","st ats","ĠR ocks","Ġsens ing","Ġrent ing","ĠGe ological","ا Ø","ot rop","Ġse w","Ġimproper ly","48 6","Ġâĸ ł","Ġstar ving","ĠB j","Disc ussion","3 28","ĠCom bo","ĠFix es","N AT","Ġstri ving","th ora","Ġharvest ed","ĠP ing","Ġplay ful","Ġaven ues","Ġoccup ational","Ġw akes","ĠCou rier","Ġdrum mer","ĠBrow ser","ĠH outh","it u","Ġapp arel","p aste","Ġhun ted","ĠSecond ly","l ain","X Y","ĠP IN","ic ons","Ġcock tails","Ġs izable","Ġhurd les","est inal","ĠRecre ation","Ġe co","64 8","ĠD ied","m int","Ġfinger prints","Ġdis pose","ĠBos nia","ts y","22 00","Ġins pected","ĠF ou","Ġf uss","Ġamb ush","ĠR ak","Ġmanif ested","Pro secut","Ġsuff ice","ren ces","Ġcompens ated","ĠC yrus","Ġgen us","ĠWolver ine","ĠTrend s","Ġh ikes","ĠSe en","Ġen rol","C old","Ġpol itely","ĠSl av","ĠRu pert","Ġey ewitness","ĠAl to","Ġun comp","Ġposter ior","M ust","ĠHer z","Ġprogress ively","Ġ23 4","Ġind ifference","ĠCunning ham","Ġacadem ia","Ġse wer","Ġast ounding","ĠA ES","r ather","Ġeld est","Ġclim bs","ĠAdd s","Ġout cry","Ġcont ag","ĠH ouses","Ġpe pt","ĠMel ania","interest ed","ĠU CH","ĠR oots","ĠHub bard","ĠT BD","ĠRoman ian","fil ename","St one","ĠIm pl","Ġchromos ome","C le","d x","Ġscram bled","ĠP t","Ġ24 2","OP LE","Ġtremend ously","St reet","Ġcra ving","Ġbund led","ĠR G","p ipe","Ġinj uring","Ġarc ane","Part icip","ĠHero ic","st y","Ġto pping","ĠTemp est","rent ices","b h","Ġpar anoia","ĠUnic ode","Ġegreg ious","Ġ\\ '","ĠOsw ald","Ġgra vel","ĠSim psons","Ġbl and","ĠGuant anamo","Writ er","lin ers","ĠD ice","J C","Ġpar ity","Ġs ided","Ġ23 7","ĠPyr rha","at ters","d k","F ine","comp an","Ġform ulated","ĠId ol","il ers","hem oth","ĠF av","Ġintr usion","Ġcar rots","ĠL ayer","ĠH acker","Ġ ----------------","Ġmoder ation","é ģ","oc oc","Ġcharacter ize","ĠTe resa","Ġsocio economic","Ġper k","ĠParticip ation","tr aining","ĠPaul o","ph ys","Ġtrust worthy","Ġembod ied","ĠMer ch","c urrency","ĠPrior ity","Ġte asing","Ġabsor bing","Ġunf inished","ĠCompar ison","Ġdis ple","writ ers","Ġprofess ions","ĠPengu in","Ġang rily","ĠL INK","68 8","ĠCor respond","Ġprev ailed","Ġcart el","l p","as ms","ĠRed emption","ĠIslam ists","effect s","d ose","ĠL atter","ĠHal ifax","Ġv as","ĠTop ics","ĠN amed","advert ising","zz a","IC ES","Ġret arded","ach able","ĠPupp et","ĠItem Level","Ġret ract","Ġident ifiable","A aron","ĠB uster","s ol","hel le","as semb","H ope","r anged","B a","ĠP urch","é Ģ","ĠSir i","Ġarri vals","Ġ19 12","Ġshort ened","Ġ3 12","Ġdiscrep ancy","ĠTem perature","ĠWal ton","Ġkind erg","p olit","Ġrem ix","Ġconnect ors","ãĥĺ ãĥ©","ĠKazakh stan","dom inated","Ġsu gars","im ble","ĠPan ic","ĠDem and","ĠCol ony","on en","ĠM ER","7 75","ur ia","aza ar","ĠDeg ree","P ri","Ġsun shine","Ġ25 1","Ġpsychedel ic","Ġdigit ally","ĠBra un","Ġsh immer","Ġsh ave","ĠTel esc","ĠAst ral","ĠVenezuel an","ĠO G","Ġc rawling","Int eg","ĠFe ather","Ġunfold ing","Ġappropri ation","Ġè£ı è","ĠMob ility","ĠN ey","- .","b ilt","L IN","ĠT ube","ĠCon versely","Ġkey boards","ĠC ao","Ġover th","Ġla ure",">> \\","ĠV iper","ach a","Off set","ĠR aleigh","ĠJ ae","J ordan","j p","Ġtotal itarian","Connect or","Ġobserv es","ĠSpart an","ĠIm mediately","ĠSc al","C ool","Ġt aps","Ġro ar","P ast","Ġch ars","ĠB ender","ĠShe ldon","Ġpain ter","Ġbe acon","ĠCreat ures","Ġdownt urn","Ġh inder","ĠAnd romeda","à Ľ","cc oli","ĠF itness","et rical","Ġutil izes","Ġsen ate","Ġen semble","Ġche ers","T W","Ġaff luent","k il","ry lic","ord ering","Com puter","Ġgru esome","ost ics","ĠUb isoft","ĠKel ley","Ġw rench","Ġbourgeois ie","IB LE","ĠPrest on","w orn","ar ist","reat ing","Ġst ained","ar ine","Ġsl ime","EN N","Ġche sts","Ġground water","ann ot","ĠTr ay","ĠLoc ke","ĠC TR","Ġd udes","ĠEx ternal","ĠDec oder","Ġpar amed","ĠMed line","80 9","ĠD inner","rup al","g z","ĠG um","ĠDem o","j ee","Ġd h","ber man","arch s","Ġen qu","ĠEp stein","Ġdevast ation","Ġfriends hips","ĠAr d","Ġ23 1","ĠRub in","ĠDist ance","Ġsp urred","Ġd ossier","Ġover looking","\\\\\\\\\\\\\\\\ \\\\\\\\\\\\\\\\","Fore st","ĠCom es","\\ \",","ĠIran ians","Ġf ixtures","L aughs","Ġcur ry","ĠKing ston","Ġsqu ash","Ġcat alogue","Ġabnormal ities","Ġdigest ive",".... .....","Ġsubord inate","og ly","Ġ24 9","M iddle","Ġmass ac","Ġburg ers","Ġdown stairs","Ġ19 31","39 4","ĠV G","Ġl asers","ĠS ikh","ĠAlex a","der ived","Ġcycl ist","ãģ® éŃĶ","onel iness","!!!! !!!!","Ġbuff s","leg ate","Ġrap ing","Ġrecomm ending","ro red","Ġmult icultural","un ique","Ġbusiness men","Ġune asy","ĠM AP","Ġdisp ersed","cipl ine","J ess","ĠK erala","å §","Ġabst raction","Sur v","U h","Ġprin ters","ij a","ow der","Ġanalog ous","ĠA SP","af er","Ġunfold ed","Ġlevel ing","Ġbre ached","ĠH earing","Ġn at","Ġtransl ating","crit ical","Ġant agonist","ĠYes terday","Ġfuzz y","w ash","m ere","Ġbe wild","ĠM ae","V irgin","ph rase","Ġsign aled","ĠH IGH","Ġprot ester","Ġgar ner","unk nown","Ġk ay","Ġabduct ed","Ġst alking","am n","Ġdes erving","ĠR iv","ĠJ orge","Ġscratch ing","ĠS aving","ip ing","Ġte ase","Ġmission ary","ĠMor row","T IME","P resent","Ġchem otherapy","tern ess","ĠH omes","ĠP urdue","Ġst aunch","ĠWhit ney","ĠTH ERE","Î ¼","iat us","ĠErn est","ĠDe ploy","Ġcove ted","F ML","ĠDial ogue","Ġex ited","f ruit","Ġner d","\":\" \",\"","Ġv ivo","ru ly","4 60","ĠAm en","rehens ible","Ġâ ĺ","D IR","Ġad herence","Ġche w","ĠCo ke","ĠSerge i","dig ital","ĠNe ck","g ently","enth al","/ )","Ġwe ary","Ġgu ise","ĠConc ord","ĠOn ion","at cher","Ġb inge","ĠDirect ive","Ġman ned","ans k","Ġill usions","Ġbillion aires","38 3","oly n","odynam ic","ĠWhe at","ĠA lic","Ġcol oured","ĠN AFTA","ab o","Ġmac ros","ind ependent","s weet","Ġsp ac","ĠK abul","Ġ Ä","em e","Ġdict ated","Ġsh outs","= {","Ġr ipping","ĠSh ay","ĠCr icket","direct ed","Ġanalys ed","ĠWAR RANT","ag ons","ĠBlaz ers","Ġche ered","Ġar ithmetic","ĠTan z","37 3","ĠFl ags","Ġ29 5","Ġw itches","ĠIn cluded","ĠG ained","ĠBl ades","G am","ĠSam antha","ĠAtl antis","ĠPr att","Ġspo iled","ĠI B","ĠRam irez","Pro bably","re ro","ĠN g","ĠWar lock","t p","Ġover he","Ġadministr ations","Ġt int","Ġreg iment","Ġpist ols","Ġblank ets","Ġep ist","Ġbowl s","Ġhydra ulic","Ġde an","Ġj ung","Ġasc end","70 5","ĠSant iago","à ®","Ġun avoid","ĠSh aman","re b","Ġstem ming","99 8","ĠM G","st icks","esthes ia","ER O","Ġmor bid","ĠGr ill","ĠP oe","any l","Ġdele ting","ĠSurve illance","Ġdirect ives","Ġiter ations","ĠR ox","ĠMil ky","F ather","Ġpat ented","44 7","Ġprec ursor","Ġm aiden","ĠP hen","ĠVe gan","ĠPat ent","K elly","Redd itor","Ġn ods","Ġvent ilation","ĠSchwar z","Ġw izards","Ġomin ous","ĠHe ads","ĠB G","Ġl umber","ĠSp iel","Ġis Enabled","Ġancest ral","ĠSh ips","Ġwrest ler","ph i","Ġy uan","ĠRebell ion","Ġice berg","Ġmag ically","Ġdivers ion","ar ro","yth m","ĠR iders","ĠRob bie","ĠK ara","ĠMain tenance","ĠHer b","Ġhar ms","p acked","ĠFe instein","Ġmarry ing","Ġbl ending","ĠR ates","Ġ18 80","Ġwr ink","ĠUn ch","ĠTor ch","desc ribed","Ġhuman oid","ilit ating","ĠCon v","ĠFe ld","IGH TS","Ġwhistlebl ower","ort mund","ets y","arre tt","ĠMon o","ĠI ke","ĠC NBC","ĠW AY","ĠMD MA","ĠIndividual s","Ġsupplement al","Ġpower house","ĠSt ru","F ocus","aph ael","ĠCol leg","att i","Z A","Ġp erenn","ĠSign ature","ĠRod ney","Ġcub es","idd led","ĠD ante","ĠIN V","iling ual","ĠC th","Ġso fa","Ġintimid ate","ĠR oe","ĠDi plom","ĠCount ries","ays on","Ġextrad ition","Ġdis abling","ĠCard iff","Ġmemor andum","ĠTr ace","Ġ?? ?","se ctor","ĠRou hani","ĠY ates","ĠFree ze","Ġbl adder","M otor","ĠProm ise","ant asy","Ġforesee able","ĠC ologne","cont ainer","ĠTre es","ĠG ors","ĠSin clair","Ġbar ring","key e","Ġsl ashed","ĠStat istical","é ĩ","Ġâĸ º","All ows","Ġhum ility","Ġdr illed","ĠF urn","44 3","Ġse wage","Ġhome page","Ġcour tyard","Ġv ile","Ġsubsid iaries","aj o","direct ory","Ġam mon","V ers","charg es","Ġ} }","ĠCh ains","Ġ24 6","n ob","Ġper cept","Ġg rit","Ġfisher men","ĠIraq is","ĠDIS TR","ĠF ULL","ĠEval uation","g raph","at ial","Ġcooper ating","Ġmel an","Ġenlight ened","Ġal i","t ailed","Ġsal ute","Ġweak est","ĠBull dogs","U A","ĠAll oy","Ġsem en","oc ene","ĠWilliam son","s pr",", âĢĶ","ĠG F","itt ens","Be at","ĠJ unk","iph ate","ĠFarm ers","ĠBit coins","ig ers","d h","ĠL oyal","p ayer","Ġentert ained","Ġpenn ed","Ġcoup on","Que ue","Ġweaken ing","c arry","Ġunderest imate","Ġshoot out","Ġcharism atic","ĠProced ure","Ġprud ent","in ances","Ġric hes","Ġcort ical","Ġstr ides","Ġd rib","ĠOil ers","5 40","ĠPer form","ĠBang kok","Ġe uth","S ER","Ġsimpl istic","t ops","camp aign","Q uality","Ġimpover ished","ĠEisen hower","Ġaug ment","ĠH arden","Ġinterven ed","Ġlist ens","ĠK ok","Ġs age","Ġrub bish","ĠD ed","Ġm ull","pe lling","Ġvide ot","Produ ction","D J","m iah","Ġadapt ations","Ġmed ically","Ġboard ed","Ġarrog ance","Ġscra pped","Ġopp ress","FORM ATION","Ġj unction","4 15","EE EE","S kill","Ġsub du","ĠSug gest","ĠP ett","Ġle tt","ĠMan ip","ĠC af","ĠCooper ation","T her","Ġreg ained","¶ æ","ref lect","Ġth ugs","ĠShel by","Ġdict ates","ĠWe iner","ĠH ale","Ġbatt leground","s child","Ġcond ol","h unt","osit ories","Ġacc uses","Fil ename","Ġsh ri","Ġmotiv ate","Ġreflect ions","N ull","ĠL obby","¥ µ","ĠS ATA","ĠBack up","Ñ ĥ","n in","ĠCor rection","Ġju icy","ut ra","ĠP ric","Ġrest raining","ĠAir bnb","ĠAr rest","Ġappropri ations","Ġsl opes","Ġmans laughter","Ġwork ings","ĠH uss","ĠF rey","Le ave","ĠHarm ony","ĠF eder","Ġ4 30","Ġt rench","Ġglad ly","Ġbull pen","ĠG au","b ones","Ġgro ove","Ġpre text","ã ħĭ","Ġtransm itter","ĠComp onent","Ġunder age","ĠEm pires","T ile","Ġo y","ĠMar vin","ĠC AS","Ġbl oss","Ġrepl icated","ĠMar iners","Marc us","ĠBl ocks","Ġliber ated","Ġbutter fly","Fe el","Ġfer mentation","Ġyou tube","Ġoff end","ĠTer m","res ist","Ġcess ation","Ġinsurg ency","Ġb ir","ĠRa ise","59 5","Ġhypothes es","50 2","Ġpl aque","ocr at","Ġjack ets","ĠHuff Post","am ong","Ġconf er","48 7","ĠL illy","Ġadapt ing","ĠF ay","Ġsh oved","ve c","Ġref ine","Ġg on","Ġgun men","z ai","ĠShut tle","ĠI zan","Ġ19 13","Ġple thora","· ·","Ġ5 10","Ġp uberty","Ġ24 1","ĠWe alth","ĠAl ma","ĠM EM","ĠAd ults","C as","pr ison","R ace","Ġwater proof","Ġathlet icism","Ġcapital ize","ĠJu ice","Ġillum inated","ĠP ascal","Ġirrit ation","ĠWitness es","ad le","ĠAst ro","Ġf ax","ĠEl vis","Prim ary","ĠL ich","ĠEl ves","Ġres iding","Ġst umble","3 19","ĠP KK","Ġadvers aries","D OS","ĠR itual","Ġsm ear","Ġar son","ident al","Ġsc ant","Ġmon archy","Ġhal ftime","Ġresid ue","Ġind ign","ĠSh aun","ĠEl m","aur i","A ff","W ATCH","ĠLy on","hel ps","36 1","Ġlobby ist","Ġdimin ishing","Ġout breaks","Ġgo ats","f avorite","ĠN ah","son ian","ĠBo oster","Ġsand box","ĠF are","ĠMalt a","Ġatt Rot","ĠM OR","ld e","Ġnavig ating","T ouch","Ġunt rue","ĠDis aster","Ġl udicrous","Pass word","ĠJ FK","blog spot","4 16","ĠUN DER","ern al","Ġdelay ing","T OP","Ġimpl ants","ĠAV G","ĠH uge","att r","Ġjournal istic","ĠPe yton","ĠI A","R ap","go al","ĠProgram me","Ġsm ashing","w ives","print ln","ĠPl ague","in us","EE P","Ġcru iser","ĠPar ish","umin ium","Ġoccup ants","ĠJ ihad","m op","Ġp int","Ġhe ct","ĠMe cca","direct or","ĠFund ing","ĠM ixed","Ġst ag","T ier","Ġg ust","Ġbright ly","ors i","Ġup hill","R D","Ġles ions","ĠBund y","liv ious","Ġbi ologist","ĠFac ulty","ĠAuthor ization","Ġ24 4","All ow","ï ¸","ĠGi ul","Ġpert inent","ot aur","es se","ĠRo of","Ġunman ned","35 1","ĠSh ak","ĠO rient","Ġend anger","D ir","Ġrepl en","ed ient","Ġtail or","Ġgad gets","Ġaud ible","âĺ Ĩ","N ice","Ġbomb ard","ĠR ape","Ġdef iance","ĠTW O","ĠFilip ino","Ġunaff ected","erv atives","Ġso ared","ĠBol ton","Ġcomprom ising","ĠBrew ers","R AL","ĠA HL","icy cle","Ġv ampires","Ġdi pped","oy er","ĠX III","Ġsidew ays","ĠW aste","ĠD iss","ĠâĶľ âĶĢâĶĢ","$ .","Ġhabit ats","ĠBe ef","tr uth","tr ained","spl it","R us","And y","ĠB ram","RE P","p id","è£ ħ","ĠMut ant","An im","ĠMar ina","Ġfut ile","hig hest","f requency","Ġepile psy","Ġcop ing","Ġconc ise","Ġtr acing","ĠS UN","pan el","ĠSoph ie","ĠCrow ley","ĠAd olf","ĠShoot er","Ġsh aky","ĠI G","ĠL ies","ĠBar ber","p kg","Ġupt ake","Ġpred atory","UL TS","/ **","Ġintox icated","ĠWest brook","od der","he ment","Ġbas eman","AP D","st orage","ĠFif ty","ed itor","G EN","UT ION","ir ting","Ġse wing","r ift","Ġag ony","ĠS ands","Ġ25 4","C ash","Ġl odge","Ġp unt","N atural","ĠIde as","Ġerrone ous","ĠSens or","ĠHann ity","Ġ19 21","Ġm ould","ĠG on","kay a","Ġanonym ously","ĠK EY","Ġsim ulator","W inter","Ġstream ed","50 7","? \",","Ġte ased","Ġco efficient","Ġwart ime","ĠTH R","' '.","ĠBank ing","mp ire","Ġf andom","Ġl ia","G a","Ġdown hill","Ġinterpre ting","Ind ividual","N orm","Ġjealous y","bit coin","Ġple asures","ĠToy s","ĠChev rolet","ĠAd visor","IZ E","Ġrecept ions","70 6","C ro","Ġ26 2","Ġcit rus","ir u","Review er","ject ed","U ES","an z","19 81","ĠWork er","Ġcompl ied","ores cent","contin ental","T on","ĠPr ism","ĠShe ep","Ġ28 8","n ox","ĠV og","O rd","Ġreal ms","te k","Ġirrig ation","Ġbicy cles","Ġelectron ically","p oly","t all","() );","Ġaest hetics","ĠInteg rated","Expl ore","Ġd unk","47 6","p ain","ĠJac ques","ĠD mit","Fram es","Ġreun ited","Ġhum id","D ro","P olitical","Ġyouth ful","Ġent ails","Ġmosqu ito","36 3","spe cies","Ġcoord inating","ĠMay hem","ĠMagn us","M ount","Impro ved","ĠST ATE","ATT LE","Ġflow ed","Ġtack led","Ġfashion ed","Ġre organ","iv ari","f inger","Ġreluct antly","et ting","ĠV and","you ng","ĠGar land","Ġpresum ption","Ġamen ities","ĠPle asant","on ential","ĠO xy","Ġmor als","ĠY ah","Read y","Sim on","En h","D emon","Ġcl ich","Mon itor","ĠD U","Ġwel comes","Ġstand out","Ġdread ful","Ġban anas","Ġball oons","h ooting","bas ic","Ġsuff ix","Ġd uly","can o","Ch ain","at os","Ġgeop olitical","Ġ( &","ĠGem ini","ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ","Ġacqu itted","L uck","prot ect","10 24","Ġsc arcity","Ġmind fulness","ec ided","D N","pr ime","ĠPres idents","ĠVID EO","Ġ( âĪĴ","add ock","N OR","ĠP ru","p un","ĠL OL",")) ))","ĠL iqu","ĠS AS","Ġsty ling","Ġpunish ments","Ġnum b","Ġasc ertain","ĠRock ies","f lu","Th umbnail","Ġperpet rated","ĠSem i","Ġdis arm","ĠOld er","ĠEx ception","Ġexponent ially","ĠCommun ities","Ġabol ish","ĠPart ner","pt oms","Ġ7 77","ĠFo ley","ĠC ases","Ġgre ase","ĠReb irth","G round","Ġ; )","ĠDoct rine","ik ini","Y e","ĠBl ossom","Ġpers ists","b ill","Ġinf usion","Ġbud dies","9 11","ĠPat ient","Ġdem os","Ġacquaint ance","ĠP aw","at ari","Ġx ml","Ġfasc ination","ĠSer ve","Ï Ĥ","br anded","Ġa z","Return s","Ġover shadow","Ġro am","Ġspeed y","n umbered","hel ial","Ġdisc iple","Ġass urances","g iven","pect ing","ĠN atalie","çĶ °","Ġmosquit oes","rote in","Ġnumer ic","Ġindepend ents","Ġtrans itional","Ġreaction ary","ĠMech dragon","do ctor","Ġshort est","Ġsequ ential","ĠB ac","ĠAccount s","ãģ Į","ach y","ract ive","ĠReg iment","Ġbreat htaking","ffic iency","ĠB ates","Ġ3 11","Ġward robe","ft s","ĠBer k","Sim ply","ĠRivers ide","iver ing","ident ial","lu cent","Ġen riched","ĠCon ver","ĠG iving","ãĥ Ļ","Ġlegal ize","ĠF TC","Ġfre aking","M ix","Ġter restrial","es ian","ci ents","W ing","LO AD","Ġled ge","ĠViol ent","ĠMet all","Ġ30 8","Ġs outheastern","hett o","M eat","Ġslow down","Ġret reated","Jere my","end as","**** *","er ic","Ġre ins","opp able","ĠHuman ity","ear ances","rig an","C amera","Ġwa ivers","s oc","Ġalter ation","trans form","ĠC emetery","50 6","Ġindef inite","Ġstim ulating","y g","60 3","ĠS op","Ġdescript ive","Ph ase","ĠEd mund","Ġpneum onia","vent us","A mb","Ġlabor atories","ĠEx clusive","ug ar","W ere","Ġmalf unction","Ġhomosexual s","Ġ---- ---","un i","Ġturb ines","ĠEqu ity","D u","Ġmind ed","ĠR H","ĠBlack hawks","Ġfe ats","Ġ17 00","re pl","36 2","lad en","Ġindisp ensable","ly ss","tt i","Ġre el","Ġdiver ted","Ġlik eness","Ġsubscript ions","Ġfing ert","Ġfil thy","dest ruct","d raft","ĠBernard ino","l aunch","Ġper plex","ĠS UM","car b","Ġswe ater","ĠVent ure","ĠJ ag","ĠCele b","ĠV oters","Ġstead fast","Ġathlet ics","ĠHans on","ĠDr ac","Tr acker","Ġcomm end","ĠPres idency","ĠD ID","in formed","Ġweb page","P retty","Ġforce fully","ãĥĥ ãĤ¯","Ġrel ocation","Ġsat ire","â ī","ĠSunder land","æ Ħ","V oice","???? ????","Ġinform ant","Ġbow el","ĠUn iform","Ġ ...\"","Ġpur ge","Ġpic nic","ĠU mb","ĠU PDATE","ĠSapp hire","ĠSt all","le arn","Ġobject ively","Ġob liter","Ġlooph ole","Ġjour neys","Ġo mission","Pro s","ĠSid ney","pl oma","Ġspray ed","Ġg uru","Ġtra itor","Ġtim et","Ġsn apping","ĠSe vent","urn al","ĠUk ip","Ġb owed","por al","l iberal","R os","Quest ions","i OS","Ġsummar ize","ST AT","Ġ18 50","ap est","Ġl ender","ĠVari able","br inging","ĠL ORD",", )","Ġcollaps es","x iety","ĠN ed","Y D","ĠSch a","Ġantib ody","Ġdis band","y re","ill usion","Ġro ver","s hed","ĠHiro sh","cc i","Ġcal am","ĠMort on","P interest","Ġ19 28","ĠE uras","ord es","Ġf ences","ĠIn ventory","ĠVal encia","ĠU d","ĠT iff","Ġsqu e","Ġqu otation","Ġtroubles ome","er ker","QU EST","ĠKing doms","s outh","Ġle vy","Pr ince","ĠSt ing","Ġnick named","Ġapp e","Ġphot ographic","Ġcorp us","re ference","ĠT rog","U nt",") =(","ĠLat via","Ġactiv ating","Ġlicense e","Ġdispar ities","ĠNews letter","ãĥĥ ãĥĪ","Ġfree ing","ĠJe ep","ĠPer ception","ins k","Ġsil icone","ĠHay den","Le an","ĠSuz uki","ibr arian","66 8","Ġsp or","Ġcorrel ations","ag hetti","Ġtu ber","ĠIP CC","il us","ĠV u","Ġwealth iest","ĠCarb uncle","an za","Ġfool ed","ĠZ ur","Ġd addy","ran o","il ian","Ġknock out","f man","requ ired","ĠWik ileaks","ĠD uffy","ON T","Ġins ol","ĠObject s","Ġb ou","ĠNord ic","ĠIns ert","sc an","Ġd ancers","Ġid iots","major ity","ĠNev ille","ĠFree BSD","Ġt art","pan ic","69 0","Ġcoc oa","Ġsam pled","Ġlook up","Ind ust","Ġinject ions","gen re","Ġa u","Ġroad way","Ġgen itals","K ind","ĠEx aminer","ĠY az","F resh","Ġpar alysis","ĠAl uminum","Ġre ap","ok é","Ġsl oppy","ĠTun nel","pos ium","ner y","en ic","Ġher bal","ĠOut er","ĠBuild er","Ġinc ur","Ġide ologies","Ġback ups","cons uming","ĠDet ect","de ck","ĠKN OW","ĠG ret","ĠM IC","Ġtough ness","ĠEx hibit","Ġh ive","L es","ĠSCH OOL","ĠAt ari","ald e","ĠN ull","and estine","m ouse","Ġbrig ade","48 9","Ġrev ol","ĠLaw son","ĠW ah","op oly","eb ted","ĠS aunders","Ġ3 13","ĠW inc","Ġtab oo","ĠHel met","Ġw edge","ch ip","ĠT ina","b g","Ġinf uri","r n","Ġanomal ies","ĠSy nc","ĠEx am","ĠComm it","ĠDi ary","ĠALS O","ĠDe bor","omed ical","Ġcomprehens ion","6 55","Ġempower ing","Ġ ire","Ġju ices","ĠE TH","ĠBox ing","=\" /","Ġfacilit ated","p oke","ĠPars ons","ĠMod er","tra vel","Ġcivil izations","Ġliber tarians","Ġrun e","ĠCl arks","at hed","Ġcampaign ers","ĠDis patch","ĠFah renheit","ĠCap com","-------- --","Ġl ace","Ġdr aining","Ġl iner","ĠArt ificial","é n","t ask","] ).","ĠGM O","ĠOper ator","ord inary","ĠInf luence","ĠU ps","Ġpot ency","uss en","osp ons","ĠSw im","ĠDead line","Un ity","Ġcul inary","Ġenlight enment","Ġwe arer","Ġmin ed","Ġp ly","Ġinc est","ĠDVD s","W alk","B TC","Tr ade","Ġdev al","ib and","ĠOvers ight","Palest inian","Ġd art","Ġm ul","L R","Ġrem ovable","ĠReal ms","ì Ŀ","Ġmisc ar","ĠV ulkan","68 5","è re","ĠS ap","Ġmer ging","ĠCar ly","che ster","Ġbr isk","Ġlux urious","ĠGener ator","Ġbit terness","Ġed ible","Ġ24 3","T G","Ġrect angle","With No","bel ow","J enn","Ġdark est","Ġh itch","Ġdos age","Ġsc aven","ĠK eller","ĠIllust rated","Certain ly","ĠMaver icks","Marg inal","Ġdiarr hea","Ġenorm ously","Ġ9 99","sh r","qu art","Ġadam ant","ĠM ew","Ġren ovation","Ġcerv ical","ĠPercent age","en ers","ĠKim ber","Ġflo ats","Ġde x","ĠW itcher","ĠSwan sea","d m","Ġsal ty","y ellow","Ġca pe","ĠDr ain","ĠPaul a","ĠTol edo","les i","Mag azine","ĠW ick","ĠM n","ĠA ck","ĠR iding","AS ON","Ġhom ophobic","AR P","Ġwand ered","C PU","ood oo","ĠP ipe","Ġtight ening","ĠBut t","3 18","Ġdesert ed","S ession","Ġfacilit ating","J ump","Ġemer gencies","OW ER","Ġexhaust ive","ĠAF TER","Ġheart beat","ĠLab el","ack y","ĠCert ified","ilt ration","Z e","ĠU tt","Ġ13 00","Ġpres ume","ĠDis p","Ġsur ged","Ġdoll s","Col umb","Ġchim pan","ĠR azor","Ġt icks","Ġcouncill or","Ġpilgr image","ĠReb els","ĠQ C","ĠA uction","x ia","ik k","b red","Ġinsert ion","Ġco arse","d B","SE E","ĠZ ap","ĠF oo","Ġcontem por","ĠQuarter ly","ot ions","ĠAl chemist","ĠT rey","ĠDu o","S weet","80 4","ĠGi ov","Ġfun n","N in","h off","Ġram ifications","Ġ19 22","ĠExper ts","az es","Ġgar ments","ar ial","ĠN ab","Ġ25 7","ĠV ed","Ġhum orous","ĠPom pe","Ġn ylon","Ġlur king","ĠSerge y","ĠMatt is","Ġmisogyn y","ĠComp onents","ĠWatch ing","ĠF olk","ract ical","B ush","Ġt aped","Ġgroup ing","Ġbe ads","Ġ20 48","Ġcon du","quer que","Read ing","Ġgriev ances","Ult ra","Ġend point","H ig","ĠSt atic","ĠScar borough","L ua","ĠMess i","a qu","ĠPsy Net","ĠR udd","Ġa venue","v p","J er","Ġsh ady","ĠRes ist","ĠArt emis","Ġcare less","Ġbro kers","Ġtemper ament","Ġ5 20","T ags","ĠTurn ing","Ġut tered","Ġp edd","Ġimpro vised","Ġ: (","Ġtab l","Ġpl ains","16 00","press ure","ĠEss ence","marg in","friend s","ĠRest oration","Ġpoll ut","ĠPok er","ĠAugust ine","ĠC IS","ĠSE AL","or ama","Ġth wart","se ek","Ġp agan"," º","cp u","Ġg arn","Ġass ortment","ĠI LCS","t ower","Recomm ended","Ġun born","ĠRandom Redditor","ĠRandomRedditor WithNo","Ġparaly zed","Ġeru ption","Ġinter sect","ĠSt oke","ĠS co","B ind","å ¾","ĠP NG","ĠNeg ative","ĠNO AA","Le on","Ġall oy","ĠL ama","ĠD iversity","5 75","Ġunderest imated","ĠSc or","Ġm ural","Ġb usted","so on","l if","Ġnone x","Ġall ergy","ĠUnder world","ĠR ays","ĠBl asio","Ġh rs","ĠD ir","Ġ3 27","by ter","Ġrepl acements","Ġactiv ates","ri ved","M H","Ġp ans","ĠH I","Ġlong itudinal","Ġnu isance","al er","Ġsw ell","ĠS igned","s ci","ĠIs les","ĠA GA","Ġdef iant","Ġson ic","oc on","K C","ĠA im","t ie","ah ah","Ġm L","D X","Ġb isc","ĠBill board","ĠSY STEM","NE Y","ga ard","Ġdist ressed","former ly","Al an","Ġche fs","Ġopt ics","ĠC omet","ĠAM C","Ġredes igned","irm ation","Ġsight ings","38 2","3 11","ĠW B","Ġcont raction","ĠT OTAL","D ual","Ġstart led","Ġunderstand ably","Ġsung lasses","ETH OD","Ġd ocker","Ġsurf ing","ĠH EL","ĠSl ack","ton es","Ġsh alt","Vis ual","49 8","Dep artment","c ussion","Ġunrest ricted","Ġt ad","Ġre name","employ ed","Ġeduc ating","Ġgrin ned","bed room","ĠActiv ities","ĠV elvet","ĠSW AT","Ġsh uffle","ig or","Ġsatur ation","F inding","c ream","ic ter","Ġv odka","tr acking","te c","Ġfore ground","iest a","Ġve hement","ĠEC B","ĠT ie","E y","Ġt urtles","ĠRail road","ĠKat z","ĠFram es","Ġmen ace","ĠFell owship","ĠEss ential","ugg ish","Ġdri p","ch witz","ĠKy oto","s b","ĠN ina","Param eter","Ġal arms","ĠCl aud","Ġpione ering","Ġchief ly","ĠSc ream","Col lection","Ġthank fully","ĠRonald o","åŃ IJ","st rip","ĠDisney land","com mercial","See ing","S oul","Ġevac uate","Ġc iv","ĠAs he","Ġdiv ides","ĠD agger","rehens ive","Ġber ries","ĠD F","Ġs ushi","Ġplur ality","W I","Ġdisadvant aged","Ġbatt alion","ob iles","45 1","Ġcl ing","Ġunden iable","ĠL ounge","Ġha unt","p he","Ġquant ify","Ġdiff ered","Ġ[* ]","ĠV iz","c um","sl ave","Ġvide og","Ġqu ar","Ġbund les","ĠAl onso","t ackle","Ġneur onal","Ġlandsl ide","conf irmed","ĠDep th","Ġrenew ables","B ear","ĠMaced onia","Ġjer seys","Ġb unk","ĠSp awn","ĠControl s","ĠBuch anan","Ġrobot ics","Ġemphas izing","ĠTut orial","h yp","ist on","Ġmonument al","æ °","ĠCar ry","Ġt bsp","en ance","H ill","art hed","Ġro tten","De an","Ġtw isting","Ġgood will","Ġimm ersion","L iving","Ġbr ushes","ĠC GI","ĠAt k","tr aditional","Ġph antom","ĠSt amina","Ġexpans ions","ĠMar in","Ġembark ed","ĠE g","int estinal","ĠPE OPLE","ĠBo oth","ĠApp alach","Ġreleg ated","V T","M IT","Ġmust er","Ġwithdraw ing","Ġmicrosc ope","ĠG athering","ĠC rescent","ĠArgent ine","ĠDec re","ĠDomin ic","Ġbud s","ant age","ĠI on","Ġwid ened","ONS ORED","ĠGl oves","iann opoulos","raz en","fe el","Ġrepay ment","Ġhind sight","ĠRE ALLY","ĠPist ol","ĠBra h","Ġwat ts","Ġsurv ives","Ġfl urry","iss y","Al ert","ĠUrug uay","Ph oenix","S low","ĠG rave","ĠF ir","Ġmanage able","Ġtar iff","ĠU DP","ĠPist ons","ĠNiger ian","Ġstrike outs","Ġcos metics","whel ming","f ab","c ape","pro xy","Ġre think","Ġover coming","sim ple","Ġw oo","Ġdistract ing","ĠSt anton","ĠTuls a","ĠD ock","65 9","Ġdisc ord","ĠEm acs","ĠV es","ĠR OB","Ġreass uring","Ġcons ortium","Muslim s","3 21","Ġprompt s","se i","ĠH itch","imp osed","ĠF ool","Ġindisc rim","wr ong","bu querque","D avis","! ]","Ġtim eless","ĠNE ED","Ġpestic ide","Ġrally ing","ĠCal der","Ġå ¤","Ġx p","ĠUn le","ĠEx port","lu aj","B uff",") [","Ġsq or","S audi","Ġis tg","Ġindul ge","pro c","Ġdisg usted","Ġcomp ounded","Ġn em","Ġschool ing","ĠC ure","process ing","S ol","Ġpro verb","it ized","ĠAlv arez","Ġscar f","Ġrect angular","re ve","Ġh ormonal","ĠSt ress","itiz en","Ġ4 25","girl s","ĠNo ir","ĠR app","Ġmar ches","ch urch","ĠUs es","Ġ40 5","ĠBer m","Ġord inances","ĠJud gment","Charg es","ĠZ in","Ġdust y","Ġstraw berries","Ġper ce","ĠTh ur","ĠDebor ah","net flix","ĠLam bert","Ġam used","ĠGu ang","Y OU","R GB","ĠC CTV","Ġf iat","r ang","Ġf ederation","ĠM ant","ĠB ust","ĠM are","respect ive","ĠM igration","ĠB IT","59 0","Ġpatriot ism","Ġout lining","reg ion","ĠJos é","Ġbl asting","ĠEz ra","B s","Ġundermin es","ĠSm ooth","Ġcl ashed","rad io","Ġtransition ing","ĠBucc aneers","ĠOw l","Ġplug s","Ġh iatus","ĠPin ball","Ġm ig","ĠNut r","ĠWolf e","Ġinteg ers","Ġor bits","ĠEd win","ĠDirect X","b ite","Ġbl azing","v r","Ed ge","ĠP ID","ex it","ĠCom ed","ĠPath finder","ĠGu id","ĠSign s","ĠZ er","ĠAg enda","Ġreimburse ment","M esh","i Phone","ĠMar cos","ĠS ites","h ate","en burg","Ġs ockets","p end","Bat man","v ir","ĠSH OW","Ġprovision al","con n","ĠDeath s","AT IVE","Pro file","sy m","J A","Ġnin ja","inst alled","id ates","eb ra","ĠOm aha","Ġse izing","ĠBe asts","Ġsal ts","M ission","Gener ally","ĠTr ilogy","he on","leg ates","Ġd ime","Ġf aire","par able","G raph","Ġtotal ing","Ġdiagram s","ĠYan uk","ple t","ĠMe h","Ġmyth ical","ĠStep hens","aut ical","ochem istry","Ġkil ograms","Ġel bows","anc ock","ĠB CE","ĠPr ague","Ġimpro v","ĠDev in","Ġ\" \\","par alle","Ġsuprem acists","ĠB illion","Ġreg imen","inn acle","Ġrequ isite","ang an","ĠBur lington","ain ment","ĠObject ive","oms ky","G V","Ġun ilateral","Ġt c","Ġh ires","ment al","Ġinvol untary","Ġtrans pl","ĠASC II"," ¨","Ev ents","Ġdoub ted","ĠKa plan","ĠCour age","ig on","ĠMan aging","ĠT art","Ġfalse hood","ĠV iolet","Ġair s","Ġfertil izer","Brit ain","Ġaqu atic","ou f","W ords","ĠHart ford","Ġeven ings","ĠV engeance","qu ite","G all","ĠP ret","Ġp df","ĠL M","ĠSo chi","ĠInter cept","9 20","Ġprofit ability","ĠId le","ĠMac Donald","ĠEst ablishment","um sy","Ġgather ings","ĠN aj","Charl ie","Ġas cent","ĠProt ector","Ġal gebra","Ġbi os","for ums","EL S","Introdu ced","Ġ3 35","Ġastron omy","Cont ribut","ĠPol ic","Pl atform","Ġcontain ment","w rap","Ġcoron ary","ĠJ elly","man ager","Ġheart breaking","c air","ĠChe ro","c gi","Med ical","ĠAccount ability","! !\"","oph ile","Ġpsych otic","ĠRest rict","Ġequ itable","iss ues","Ġ19 05","ĠN ek","c ised","ĠTr acking","Ġo zone","Ġcook er","ros is","Ġre open","Ġinf inity","ĠPharm aceutical","ens ional","Att empt","ĠR ory","Mar co","Ġawa its","H OW","t reated","Ġbol st","Ġreve red","Ġp ods","opp ers","00 10","Ġampl itude","ric an","SP ONSORED","Ġtrou sers","Ġhal ves","ĠK aine","ĠCut ler","ĠA UTH","Ġsplend id","Ġprevent ive","ĠDud ley","if acts","umin ati","ĠY in","Ġad mon","ĠV ag","Ġin verted","Ġhast ily","ĠH ague","L yn","Ġled ger","Ġastron omical","get ting","Ġcirc a","ĠC ic","ĠTenn is","Lim ited","Ġd ru","ĠBY U","Ġtrave llers","Ġp ane","ĠInt ro","Ġpatient ly","Ġa iding","Ġlo os","ĠT ough","Ġ29 3","Ġconsum es","Source File","Ġ\"\" \"","Ġbond ing","Ġtil ted","Ġmenstru al","ĠCel estial","UL AR","Plug in","Ġrisk ing","N az","ĠRiy adh","Ġacc redited","Ġsk irm","é Ľ","Ġexam iner","Ġmess ing","Ġnear ing","ĠC hern","ĠBeck ham","Ġsw apped","Ġgo ose","K ay","Ġlo fty","ĠWal let","Ġ[ '","Ġap ocalypse","Ġb amboo","ĠSP ACE","ĠEl ena","Ġ30 6","ac ons","Ġtight ened","Ġadolesc ence","Ġrain y","Ġvandal ism","ĠNew town","Ġcon ject","c akes","Ġche ated","Ġmoder ators","par ams","E FF","Ġdece it","ĠST L","ĠTanz ania","ĠR I","Ġ19 23","ĠEx ile","the l","Ġthe olog","Ġquir ky","ĠIr vine","Ġneed y","or is","U m","K a","Ġmail box","3 22","Ġb os","ĠPet ra","K ING","Ġenlarg ed","O ften","Ġbad ass","Ġ3 43","ĠPl aces","ĠC AD","Ġpr istine","Ġinterven ing","d irection","Ġl az","ĠD SM","Ġproject ing","ĠF unk","ag og","pay ment","n ov","Ġch atter","AR B","Ġexam inations","ĠHouse hold","ĠG us","F ord","4 14","B oss","Ġmy stic","Ġle aps","ĠB av","ul z","b udget","Foot ball","Ġsubsid ized","Ġfirst hand","Ġcoinc ide","oc ular","Con n","ĠColl abor","Ġfool s","am ura","ah ar","r ists","Ġsw ollen","Ġexp ended","ĠP au","s up","Ġsp ar","Ġkey note","s uff","Ġunequ al","Ġprogress ing","str ings","ĠGamer gate","Dis ney","ĠEle ven","om nia","Ġscript ed","Ġear ners","bro ther","ĠEn abled","æ ³","Ġlar vae","ĠL OC","m ess","Wil son","ĠTem plate","success fully","Ġparam ount","Ġcamoufl age","Ġbind s","ĠQu iet","ĠSh utterstock","r ush","Ġmasc ot","fort une","ĠCol t","ĠBe yon","hab i","Ġha irc","Ġ26 7","ĠDe us","Ġtw itch","Ġconcent rating","Ġn ipples","c ible","Ġg ir","N Z","M ath","n ih","Requ ired","Ġp onder","ĠS AN","Ġwedd ings","Ġl oneliness","N ES","ĠMah jong","69 5","add le","ĠGar ner","ĠC OUR","Br idge","Ġsp ree","ĠCald well","Ġbri bery","Ġ���� ����","plug ins","Ġr acket","Ġchamp agne","vers ible","V ote","Ġmod ifiers","May or","6 80","Ġassemb lies","ĠS ultan","ĠN ing","ĠLad ies","Ġsulf ur","Ġor bs","Ġ---- -","____ ___","ĠJournal ism","Ġes ports","Ġl ush","Ġh ue","Ġspect ral","H onest","ãĥ ı","Ġbus hes","Ġrein forcement","Ġre opened","ĠWhe els","ĠM org","rie ving","Ġaux iliary","Ġj Query","ĠB AT","tes que","Ġver tex","p ure","f rey","ãĤ º","d os","Ġty ph","Ġc ull","Ġe q","Ġdec on","Ġtoss ing","Ġdispar ate","ĠBr igham","print f","led ged","Ġsu nd","Ġco zy","Ġhepat itis","per forming","Ġav al","ĠG G","f uture","Ġpet ertodd","ĠKos ovo","Ġmagn ets","Al ready","ĠEd ison","ĠCe res","ĠRA ID","Ġbrill iance","57 6","Ġder ives","Ġhypert ension","ĠÎ Ķ","Ġlamb da","Ġfl air","Ġmission aries","Ġrap es","ĠSt arter","ĠMon ths","Ġdef y","Ġseism ic","ĠR aphael","Ġeuro zone","65 6","z sche","Ġscr atched","Ġb ows","ĠLenn on","ĠGa ia","Ġdri pping","f acts","A le","Ġfrog s","ĠBre ast","ogene ity","ĠProsecut or","Ġampl ified","ĠHod g","ĠF n","Th ousands","ĠNI H","ĠMonitor ing","FT WARE","ĠPri ebus","ĠG rowing","hun ter","Ġdiagn ose","ĠM ald","ĠL R","Ġcrown ed","Ġburst ing","Ġdiss olution","j avascript","Ġuseful ness","ĠExec ution",": (","ĠIv ory","a ah","Ġpersecut ed","viol ence","ist as","ĠCr ate","Ġimpuls es","ĠSp ani","ed es","Hand le","ĠZ erg","think able","Last ly","Ġspont aneously","Ġinconven ient","Ġdismiss ing","Ġpl otted","Ġeight y","Ġ7 37","r ish","ĠThor nton","ath am","Ġsit com","V en","Rec ipe","t el","l und","Ġcle ars","ĠSas uke","Ġ25 8","Ġopt ing","Ġen raged","est hetic","ĠA e","uch s","Pre p","Fl ow","Ġrun off","ĠE ating","ĠG iles","ĠAct ing","res ources","ib aba","Ġr pm","Ġske wed","ĠBl anc","ĠS akuya","Ġhot ter","Ġ19 24","op ian","ck o","Ġcr umbling","Ġcapt ains","ĠAppropri ations","le aders","dro pping","an uts","Ġrevers ing","ĠP ose","ĠS ek","Sc ot","ĠIde a","c ise","ĠSloven ia","Ġ3 17","Do ctor","Ġcro cod","ald i","Se a","ĠFar rell","Ġmerc enaries","ĠR NC","ĠGu ess","Ġp acing","M achine","Streamer Bot","ĠChar ity","Ġ29 8","Ġcann ons","ĠTob y","TPP StreamerBot","ĠPass ion","cf g","Th om","Ġbad ges","ĠBern stein",". âĢĵ","ĠP OP","ĠCon j","Ġinitial ization","Ġbiod iversity","D ub","Ġfeud al","Ġdisclaim er","Ġc row","Ġign ition","ar f","S HA","Ġk Hz","h azard","ĠArt ists","oe uv","67 9","ĠRud y","N ine","ĠRam adan","å ½","itt o","Ġadren aline","C ert","Ġsmell ed","Ġimp unity","Ġag endas","ĠRe born","ĠCon cent","ĠSe ems","Ġo mega","ĠDust in","Ġback er","ĠSau ce","ĠBoy le","W IN","Ġsp ins","Ġpa uses","u pt","Ġshred ded","Ġstra pped","ĠCor ruption","Ġscr atches","Ġn i","Ġatt ire","ĠS AF","Factory Reloaded","ĠI PS","Ġ( %","Ġsem inar","f ocus","c ivil","Ġ18 60","int osh","Ġcontin ual","Ġabbre vi","ĠS ok","oc obo","X M","Ġfr antic","Ġunavoid able","Ġar tery","Ġannot ations","b ath","Cl imate","Ġd ors","ĠSl ide","co ord","ĠRel oad","ĠL DL","ĠLove craft","Ġunim agin","Ġresemb led","Ġbarr acks","n p","Ġsurrog ate","Ġcategor ized","ãĤ ©","Ġvacc inated","Ġdrain age","Ġind ist","ĠWhats App","Ġ18 70","oler ance","inv oke","am orph","Ġrecon nect","Ġem anc","Ġblind ness","Ġ12 80","intern et","c ollar","Ġalt ru","Ġab yss","ĠT RI","65 7","Ġinf used","HE AD","Ġforest ry","ĠWood y","ĠC i","w i","s am","78 4","hol iday","Ġmog ul","ĠF ees","ĠD EN","In ternal","ur bed","f usc","at om","ĠIll usion","Ġpoll ed","Ġfl ap","Ġco ax","L GBT","An aly","ĠSect ions","ĠCalif orn","em n","Ġh ither","ĠN IGHT","Ġn ailed","ĠPip eline","39 1","o of","ĠPr imal","vere nd","Ġsl ashing","Ġret ri","avi our","Ġdepart ing","g il","IS C","Ġmid way","Ġultras ound","Ġbeh aving","ĠT ara","class es","V irtual","ĠColon ial","Ġstri pping","Ġorchestr ated","ĠGra ves","45 2","ĠIron ically","ĠWrit ers","Ġl ends","ĠMan z","Ġra ven","Ġoxid ative","Ġ26 6","EL F","act ually","asc ar","D raft","Ġfavour able","Ġhumili ating","Ġf idelity","ĠH of","ĠX uan","49 6","Ġlay ered","at is","79 0","Ġpay check","it on","K ar","ĠVM ware","ĠFar mer","Ġserv ic","gl omer","Ġsl ump","ĠFab ric","ĠD OC","est ing","Ġreass ure","Ġph yl","v olt","it ory","R ules","Ġoxid ation","Ġpri zed","Ġmist ress","ĠDj ango","WAR N","å ij","Ġenc ode","ĠFeed back","Ġstupid ity","I an","ĠYugoslav ia","× ¨","ac l","UT E","19 77","Ġqual ifies","Ġpuls es","pret ty","Ġfro ze","Ġs s","Iter ator","Ġur gently","Ġm ailed","ĠCh am","Ġsust aining","Ġbas il","Ġpupp ies","il ant","ĠP LEASE","l ap","ace ous","F ear","ĠMaster y","aut omatic","ĠT AG","Ġant im","ag les","47 3","fram es","Ġwh ispers","ĠWho ever","Ġbra very","ĠUK IP","ract ions","\"\" \"","Ġt ame","Ġpart ed","every thing","CON T","Ġind ebted","Ġadd r","re k","IR ED","Ġem inent","cl inton","Ġo usted","Ġreview er","Ġmelt down","Ġre arr","ĠY ao","the real","aby te","Ġst umbling","Ġbat ches","Ġ25 9","Ġcontrace ptive","Ġprost itute","ens is","De cl","ĠSt rikes","M ilitary","ĠO ath","v acc","pp ings","05 2","Ġpart Name","amp ing","Rep orts","K I","CH R","Ġsubt ly","sw ers","Bl ake","us ual","Ġcontest ants","Ġcart ridges","ĠGRE AT","Ġbl ush","ĠâĢ º","47 2","Ġreason ed","ãĥ ¤","paralle led","Ġd yn","ag ate","Ġnight ly","å Ĩ","55 6","Ġsem antic","ĠAdv oc","Ġ !!","Ġdisag rees","ĠB W","V eh","Ġharm ing","Ġembr aces","Ġstri ves","Ġin land","ĠK ard","Ġhe ats","ĠGin ny","ut an","ern aut","yl ene","ĠE lev","J D","Ġh ars","ĠStar r","Ġsk ysc","Ġcollabor ators","Us ually","Ġrev olutions","ĠSTAT S","Ġdism antle","Ġconfident ly","Ġkin etic","Al i","Ġpercent ile","Ġextract ing","ill ian","est ead","Ġphysic ists","ĠMarsh al","Ġfell owship","Ġd ashed","ĠU R","ĠSi oux","ĠComp act","am ide","P ython","ĠLe igh","ĠPharm ac","ist rates","her ical","Ġf ue","ĠE min","Ġ( {","ĠNeighbor hood","Ġdisrupt ing","ĠD up","Ġg land","ĠSe v","ĠMar ian","arg on","ĠD und","Ġ< !--","Ġstr and","Ġstadium s","z os","Ġpsych osis","ĠR ack","Ġbrilliant ly","ï¸ ı","Ġsubmer ged","ĠInst it","ĠCh ow","Ġc ages","ĠH ats","ĠU rs","Ġdil uted","us at","ien ne","ĠMembers hip","ĠBur k","Ġ ie","Ġarche type","D rug","ult on","ĠSp ock","ĠMcK ay","ĠDep end","F eatured","S oc","19 78","ĠB ere","Ġrelent lessly","Ġcripp ling","Ġar thritis","çĶ Ł","ĠTrop ical","ĠBul g","ĠCher yl","Ġadm irable","Ġsub title","Over ride","Ġorig inating","ĠC CP","Ġsw ore","ĠSo le","ĠDis orders","3 29","Ġprocess ion","Ġref urb","Ġimm ersed","requ ently","Ġskept ics","Ġcer amic","m itter","en stein","b elt","ĠT IT","b idden","Ġf ir","m ist","> ]","Ġwe ave","ĠParad ox","Ġentr usted","ĠBarcl ays","Ġnovel ist","og ie","80 6","Ġnin ety","Ġdisag reements","@@@@ @@@@","ĠAus chwitz","c ars","ĠL ET","t ub","arant ine","P OS","Ġback story","Ġcheer ful","ĠR ag","ek a","bi ased","Ġinexper ienced","ak ra","ĠW itt","t an","Ġrap ist","Ġplate au","ch al","ĠInqu is","exp ression","Ġc ipher","Ġsh aving","add en","re ly","( \\","ism a","ĠReg ulatory","CH AR","ily n","N VIDIA","G U","Ġmur m","la us","Christ opher","Ġcontract ual","ĠPro xy","ĠJa ime","ĠMethod ist","Ġstew ards","st a","per ia","Ġphys iology","Ġbump ed","Ġf ructose","Austral ian","ĠMet allic","ĠMas querade","ar b","Ġprom ul","Ġdown fall","Ġbut cher","Ġb our","ĠIN FORMATION","ĠB is","pect s","ad ena","Ġcontempl ating","ar oo","cent ered","ĠPe aks","Us ed","Ġmod em","Ġg enders","Ġ8 000","37 1","Ġm aternity","ĠR az","Ġrock ing","Ġhandgun s","ĠD ACA","Aut om","ĠN ile","Ġtum ult","ĠBenef it","ĠAppro ach","works hop","ĠLe aving","G er","inst ead","Ġvibr ations","Ġrep ositories","49 7","ĠA unt","ĠJ ub","ĠExp edition","Al pha","Ġs ans","Ġoverd ue","Ġoverc rowd","Ġlegisl atures","Ġp aternal","ĠLeon ardo","Ġexp ressive","Ġdistract ions","Ġsil enced","tr ust","Ġb iking","Ġ5 60","Ġpropri et","Ġimp osition","Ġcon glomer","Ġ= ================================================================","ĠTe aching","ĠY ose","int ensive","T own","Ġtroll ing","ĠGr ac","ĠAS US","Y o","Ġspecial s","ĠNep h","ĠGod zilla","Dat abase","ĠHe gel","Ġ27 2","19 76","ĠGl oria","Ġdis emb","ĠInvestig ations","ĠB ane","ag ements","St range","Ġtre asury","ĠPl ays","Ġundes irable","Ġwid ening","Ġverb ally","Ġinf ancy","Ġcut ter","f ml","Ġ21 00","prot otype","f ine","Ġdec riminal","Ġdysfunction al","Ġbes ie","ĠErn st","z eb","Ġnort heastern","Ġa ust","por ate","ĠMar lins","Ġsegreg ated","ew orld","ĠMa her","Ġtra verse","Ġmon astery","ur gy","G ear","s and","Com pl","ĠE MP","Ġpl ent","ĠMer cer","Ġ27 6","TA BLE","Config uration","H undreds","Ġpr ic","Ġcollabor ating","ĠPar amount","ĠCumm ings","Ġ( <","Ġrecord er","Ġfl ats","Ġ4 16","wh ose","Font Size","ĠOr bit","Y R","Ġwr ists","Ġb akery",") }","ĠB ounty","ĠLanc aster","Ġend ings","acc ording","ĠSal am","e asy","75 5","ĠBur r","ĠBarn ett","onom ous","Un ion","Ġpreced ence","ĠScholars hip","ĠU X","Ġroll out","Ġbo on","al m","ĠCan ter","æ µ","Ġround ing","Ġcl ad","Ġv ap","ĠF eatured","is ations","Ġ5 40","pol ice","Ġunsett ling","Ġdr ifting","ĠLum ia","ĠObama Care","ĠF avor","Hy per","ĠRoth schild","ĠMil iband","an aly","ĠJul iet","H u","Ġrec alling","a head","69 6","Ġunf avorable","Ġd ances","O x","Ġleg ality","Ġ40 3","rom ancer","Ġinqu ire","ĠM oves","\\ \">","ĠVari ant","ĠMess iah","ĠL CS","ĠBah á","75 6","Ġeyeb row","Ġ ¥","ĠMc F","ĠFort y","M as","Ġpan icked","Ġtransform ations","q q","Ġrev olves","ring e","ĠA i","ax e","Ġon ward","ĠC FR","ĠB are","log in","Ġliqu ids","Ġde comp","second ary","il an","ĠCon vert","ami ya","Ġprosecut ing","Ġâī ¡","ĠYork ers","ĠByr ne","sl ow","aw ei","J ean","Ġ26 9","ĠSky dragon","Ġ é","ĠNicarag ua","ĠHuck abee","ĠHigh ly","Ġamph ib","ĠPast or","ĠL ets","Ġbl urred","Ġvisc eral","ĠC BO","Ġcollabor ated","z ig","Leg al","Ġapart heid","Ġbr id","Ġpres et","ĠD ET","ĠAM A","× Ķ","arch ing","auc uses","build er","Ġpo etic","Ġem ulator","ĠMole cular","Ġhon oring","ise um","Ġtract or","ĠCl uster","ĠCal m","ared evil","Ġsidew alks","Ġviol in","Ġgeneral ized","ĠAle c","Ġemb argo","Ġfast ball","ĠHT TPS","ĠL ack","ĠCh ill","ri ver","C hel","ĠSw arm","ĠLev ine","ro ying","L aunch","Ġkick er","Ġadd itive","ĠDe als","W idget","cont aining","Ġescal ate","ĠOP EN","Ġtwe aked","Ġst ash","Ġsp arks","ĠEs sex","ĠE cc","Ġconv ict","Ġblog ging","I ER","ĠH L","Ġmurd erers","75 9","ĠH ib","Ġde pl","ĠJ ord","S ac","Ġdis sect","ĠHow e","os her","Ġcustom izable","ĠFran z","Ġat ro","Ä ĩ","Ġ000 4","Ġout post","R oss","Ġglyph osate","ĠHast ings","ĠBE FORE","Ġsh ove","o pped","ĠSc ala","Ġam ulet","an ian","Ġexacerb ated","Ġe ater","47 1","UM E","Ġpul p","izont al","ĠZ am","ĠAT I","imm une","aby tes","Ġunnecess arily","ĠC AT","ĠAx is","Ġvisual ize","à ī","ĠRad ical","f m","Doc uments","ĠFor rest","Ġcontext ual","ĠSy mbol","Ġtent ative","ĠDO ES","ĠGood s","Ġintermitt ent","} :","medi ated","Ġridic ule","Ġathe ism","Ġpath ogens","ĠM um","Ġre introdu","Ġ30 7","i HUD","Ġflash light","Ġsw earing","Ġp engu","B u","Ġrot ated","ĠCr ane","Ġ() );","Ġfashion able","Ġendors ing","46 3",") [","Ġingest ion","Ġcook s","Ġ9 50","ot omy","ĠIm am","Ġk a","Ġte aser","ĠGhost s","ĠãĤ µ","19 69","Ï ĥ","ub by","Ġconver ter","zan ne","end e","ĠPre par","ĠNic kel","ĠChim era","h im","ĠTyr ann","ĠSabb ath","ĠNich ols","Ġra pt","ih ar","Ġshe lling","Ġillum inate","Ġdent ist","ut or","ĠInteg ration","Ġwh ims","ĠLiter ary","Be aut","Ġp archment","ag ara","Br and","Ġder og","â̦ )","ĠNor se","Ġunw itting","Ġc uc","Ġborder line","Ġupset ting","Ġrec ourse","Ġd raped","ĠRad ar","Ġcold er","ĠPep si","im inary","], [","65 8","V i","ĠF rem","ĠP es","Ġveter inary","ĠT ED","ĠEp idem","n ova","k id","Ġdev out","o ct","j ad","M oh","ĠP AY","Ġge ometric","Ġ3 23","Ġcircum ference","ich ick","19 75","ĠY uri","ĠSh all","ĠH over","un in","S pr","Ġg raft","ĠHapp iness","Ġdisadvant ages","att acks","Ġhub s","ĠStar Craft","é ĸ","Ġgall eries","ĠKor ra","Ġgrocer ies","ĠGors uch","Ġrap ists","Ġfun gi","ĠTyph oon","V ector","ĠEm press","b attle","4 68","Ġparas ite","ĠBom ber","S G","ex ist","ĠP f","Ġun se","Ġsurge ons","B irth","ĠUn sure","ĠPrint ed","ĠBehavior al","ĠA ster","Pak istan","Ġun ethical","Ġs v","ĠIo T","Ġlay outs","P ain","Ġconst ants","ĠL W","ĠB ake","Ġtow els","Ġdeterior ation","ĠBol ivia","Ġblind ed","ĠW arden","ĠMist ress","Ġon stage","Ġcl ans","ĠB EST","19 60","Ġant ique","Ġrhet orical","ĠPer cy","ĠRw anda",", .","B ruce","Ġtra umat","ĠParliament ary","Ġfoot note","id ia","ĠLear ned","se eking","gen ic","Ġdim ensional","H ide","èĢ ħ","Ġintrig ue","in se","Ġle ases","Ġapp rentices","w ashing","Ġ19 26","V ILLE","Ġsw oop","s cl","Ġbed rooms","on ics","ĠCr unch","comp atible","Ġincap ac","ĠYemen i","ash tra","z hou","d anger","Ġmanifest ations","ĠDem ons","AA F","Secret ary","ACT ED","L OD","Ġam y","ra per","eth nic","4 17","Ġpos itives","Ġ27 3","ĠRefuge es","Ġus b","ĠV ald","odd y","ĠMahm oud","As ia","Ġskull s","ĠEx odus","ĠComp et","ĠL IC","ĠM ansion","ĠA me","Ġconsolid ate","storm s","ont ent","99 6","Ġcl en","Ġm ummy","fl at","75 8","ĠV OL","oter ic","n en","ĠMin ute","S ov","Ġfin er","R h","ly cer","Ġreinforce ments","ĠJohann es","ĠGall agher","Ġgym n","S uddenly","Ġext ortion","k r","i ator","T a","Ġhippocamp us","N PR","ĠComput ing","Ġsquare ly","Ġmod elling","ĠFor ums","ĠL isp","ĠKrish na","Ġ3 24","Ġr ushes","Ġens ued","Ġcre eping","on te","n ai","il ater","ĠHorn ets","Ġob livious","IN ST","55 9","Ġjeopard y","Ġdistingu ishing","j ured","Ġbeg s","sim ilar","ph ot","5 30","ĠPark way","Ġs inks","ĠHearth stone","ib ur","ĠBat on","Av oid","Ġd ancer","Ġmag istrate","ary n","Ġdisturb ances","ĠRom ero","Ġpar aph","Ġmis chief","âĸ ĵ","ĠSh aria","Ġur inary","r oute","iv as","f itted","Ġeject ed","ĠAl buquerque","Ġ4 70","Ġirrit ated","ĠZ ip","ĠB iol","à į","Ġden ounce","Ġbin aries","ĠVer se","Ġopp os","ĠKend rick","ĠG PL","Ġsp ew","ĠEl ijah","ĠE as","Ġdr ifted","so far","Ġannoy ance","ĠB ET","47 4","ĠSt rongh","it ates","ĠCogn itive","oph one","ĠIdent ification","ocr ine","connect ion","Ġbox er","ĠAS D","ĠAre as","Y ang","t ch","ull ah","Ġdece ive","Comb at","ep isode","cre te","W itness","Ġcondol ences","ht ar","Ġhe als","Ġbuck ets","ĠLA W","B lu","Ġsl ab","ĠOR DER","oc l","att on","ĠSteven son","ĠG inger","ĠFriend ly","ĠVander bilt","sp irit","ig l","ĠReg arding","ĠPR OG","Ġse aling","start ing","Ġcard inal","ĠV ec","ĠBe ir","Ġmillisec onds","we ak","per se","Ġster ile","ĠCont emporary","ĠPh ant","ĠCl o","Ġout p","Ġex iled","Ġ27 7","Ġself ie","Ġman ic","Ġn ano","ter ms","Alex ander","Ġres olves","Ġmillenn ia","Ġexpl odes","Ġconst ellation","Ġadul tery","m otion","D OC","Ġbroad casters","Ġkinderg arten","ĠMay weather","ĠE co","ich o","Ġ28 7","l aun","Ġm ute","Ġdisc reet","Ġpres chool","Ġpre empt","De lete","ĠFre ed","P i","H K","Ġblock er","ĠC umber","Ġw rought","d ating","Ġins urer","Ġquot as","Ġpre ached","Ġev iction","ĠReg ina","ĠP ens","Ġsevent een","ĠN ass","D ick","Ġfold s","Ġd otted","ĠA ad","Un iversal","Ġp izz","ĠG uru","Ġso ils","Ġno vice","ĠNe ander","Ġst ool","Ġdeton ated","ĠPik achu","ĠMass ive","IV ER","ĠAb del","Ġsubdu ed","Ġtall est","Ġprec arious","Ġa y","r ification","ĠOb j","c ale","Ġun question","cul osis","ad as","igr ated","D ays","Ġque ens","ĠGaz ette","ĠCol our","ĠBow man","ĠJ J","ï ve","Ġdomin ates","Stud ent","Ġm u","Ġback log","ĠElect ro","Tr uth","48 3","Ġcond ensed","r ules","ĠCons piracy","Ġacron ym","hand led","ĠMat te","j ri","ĠImp ossible","l ude","cre ation","Ġwar med","ĠSl ave","Ġmis led","Ġfer ment","ĠK ah","ink i","ke leton","cy l","ĠKar in","Hun ter","Reg ister","ĠSur rey","Ġst ares","ĠW idth","ĠN ay","ĠSk i","Ġblack list","uck et","Ġexp ulsion","im et","Ġret weet","vant age","Fe ature","Ġtro opers","Ġhom ers","9 69","Ġconting ency","ĠW TC","ĠBrew er","fore ign","W are","S olar","Ġund ue","RE C","ulner able","path ic","ĠBo ise","Ġ3 22","Ġarous ed","ĠY ing","ä¸ į","uel ess","Ġp as","Ġmor p","Ġfl oral","Ex press","ud ging","k B","ĠGr anted","Ø ¯","ĠMich a","ĠGoth ic","ĠSPEC IAL","ĠRic ardo","F ran","Ġadminister ing","6 20","por a","Ġ ®","Ġcomprom ises","Ġb itten","Ac cept","Th irty","Ð ²","Ġmater ially","ĠTer r","ig matic","ch ains","Ġdo ve","stad t","Mar vel","FA ULT","Ġwind shield","Ġ3 36","ad ier","Ġsw apping","Ġflaw less","ĠPred ator","ĠMiche le","Ġprop ulsion","ĠPsych ic","Ġassign ing","Ġfabric ation","Ġbar ley","l ust","Ġtow ering","Ġalter cation","ĠBent ley","Sp here","Ġtun a","ĠClass es","Fre edom","un er","L ady","v oice","Ġcool est","or r","Ġpal p","$ {","Ġhyster ia","ĠMet atron","p ants","Ġspawn ing","Exper ts","ĠInvest ors","ĠAn archy","Ġshr unk","ĠVict im","Ġ28 9","Ġec stasy","ĠB inding","58 5","ĠMel ody","57 8","ot ally","ĠE tsy","lig a","Ġapplaud ed","Ġswe ating","Ġredist ributed","Ġpop corn","Ġsem inal","f ur","ĠNeuro science","R and","ĠO st","ĠMadd en","ĠIncre asing","ĠDaw kins","ĠSub way","Ġar sen","cons erv","B UR","Ġsp iked","ĠLy ft","ĠImper ium","ĠDrop box","Ġfav oured","Ġencomp asses","gh ost","Ġins pires","Ġbur geoning","ĠY oshi","ĠVert ical","ĠAud itor","Ġint ending","Ġfilib uster","Bl oom","f ac","ĠCav s","ign ing","Ġcowork ers","ĠBarb arian","rem ember","FL AG","Ġaudit ory","ason ry","Col lege","Ġmut ed","gem ony","ob in","ĠPsych o","9 68","Ġlav ish","Ġhierarch ical","ĠDr one","ou k","Ġcripp led","ĠMax im","Sl ot","Ġqu iz","ĠV id","if ling","Ġarchae ologists","Ġabandon ment","d ial","le on","ĠF as","T ed","Ġr aspberry","Ġmaneu vers","Ġbehavi ours","Ġins ure","Ġrem od","Sw itch","h oe","Ġsp aced","Ġafford ability","ĠF ern","not ation","ĠBal anced","Ġoccup ies","en vironment","Ġneck lace","Ġsed an","F U","ĠBrav o","Ġab users","ĠAn ita","met adata","ĠG ithub","ait o","ĠF aster","ĠWass erman","ĠF lesh","Ġth orn","r arily","ĠMer ry","w ine","Ġpopul ace","ĠL ann","Ġrepair ing","Ġpsy che","Ġmod ulation","aw aru","âĢĭ âĢĭ","ari j","Ġdecor ations","Ġapolog ise","ĠG arg","app ly","Ġgive away","ĠFl an","ĠWy att","U ber","Ġauthor ised","ĠMor al","HAHA HAHA","activ ate","Ġtorped o","ĠF AR","Ġam assed","ĠA ram","ark in","ĠVict ims","st ab","Ġo m","ĠE CO","Ġopio ids","Ġpurpose ly","ĠV est","Ġer g","at an","ĠSur gery","Ġcorrect ing","ĠOrt iz","ĠBe et","Ġrev oke","Ġfre eway","ĠH iggins","F ail","ĠFar ms","ĠAT P","h ound","Ġp oking","ĠCommun ists","mon ster","iment ary","Ġunlock ing","Ġunf it","we ed","en ario","at ical","ĠEnlight enment","ĠN G","ĠComp ensation","de en","ĠWid ow","ĠCind y","ĠAfter wards","Ġ6 000","ikh ail","ag ically","Ġrat ified","Ġcasual ty","H OME","p sey","f ee","Ġspark ling","Ġd é","Ġconcert ed","C atal","Ġcomp lying","ĠA res","ĠD ent","Sh ut","Ġsk im","ad minist","Ġhost ilities","ĠG ins","Ġ6 08","Ġm uddy","ĠMc Int","ĠDec ay","5 25","Ġconspic uous","ĠEx posure","Ġresc ind","Ġwear able","Ġ3 28","our met","ah s","ĠRob ots","Ġe clips","inst ance","ĠRE PORT","ĠApp l","0 30","ĠSk ies","01 00","Ġfall acy","S ocket","ĠRece iver","Ġsol ves","ĠButter fly","ĠSho pping","ĠFI RE","65 4","Med ic","Ġsing ers","ĠNeed less","'' ''","isher s","ĠD ive","58 8","Ġselect ively","Ġcl umsy","88 9","Ġpurch aser","ear ned","ard y","Ġbenef iting","eng lish","Ġyield ing","ĠP our","Ġspin ach","Ġdel ve","ĠC rom","6 10","Ġexport ing","ĠMA KE","Ġ26 3","Ġg rop","Ġenv oy","ĠInqu iry","ĠLu igi","d ry","ĠT uring","Thumbnail Image","ĠVar iety","Ġfac et","Ġfl uffy","Ġexcerpt s","Ġsh orth","ĠOl sen","CL UD","Ġrel iant","ĠUN C","T our","Ġbat hing","Comp any","Ġglobal ization","P red","ĠMalf oy","Ġh oc","j am","craft ed","ĠBond s","ĠKiss inger","Eng land","Ġorder ly","cat entry","Ġ26 1","Ġexch anging","ĠInt ent","ĠAmend ments","D OM","Ġst out","³³³³³³³³ ³³³³³³³³","ĠAir bus","Ġ27 8","hy de","P oll","Item ThumbnailImage","Ġlooph oles","ĠPill ar","Ġexpl or","St retch","A part","Ġun married","Lim it","ĠTransform ers","Ġintellect ually","unct ure","18 00","Ġd arn","B razil","Ġleft over","ber us","f red","Mine craft","3 26","ĠForm s","Ġproof s","ĠDes igned","Ġindex es","ĠSupp ose","EM S","ĠL oving","ĠBon nie","im ating","OT US","Ġconduct or","Ġbehav ed","ĠF ren","Ġsy nerg","Ġmillenn ium","Ġcater ing","ĠL auder","W r","ĠY iannopoulos","ĠAT F","Ġensl aved","Ġawaken ed","D VD","ĠED ITION","ĠConc ert","ĠChall enger","ĠH aku","umer ic","Ġdep recated","ĠSH AR","4 12","Ġdy stop","Ġtremb ling","Ġdread ed","ĠSp ac","p adding","Re pl","ĠG arrison","M ini","Ġun paralleled","am ar","URR ENT","w reck","c ertain","t al","ĠC LS","app ings","Ġsens ed","Ġf encing","ĠPas o","ĠDes k","Ġsc off","Ġcontem plate","ĠL iga","l iquid","75 7","Ġapp rentice","ĠUCH IJ","5 70","ĠTh ousand","ĠIll um","Ġchampion ed","ãĤ Į","Ġelect ors","Ġ3 98","ĠH ancock","round ed","ĠJ OHN","Ġuns atisf","Ġqual ifier","ĠGad get","EN E","Ġdead liest","ĠPl ants","Ġ ions","Ġacc ents","Ġtwe aking","Ġsh aved","F REE","ĠCh aser","Again st","9 60","Ġmeth amphetamine","Ġnormal ized","Ġ$ \\","ĠPre cision","ĠGu am","Ġch oked","ĠX II","ĠCast ing","Tor rent","Ġscal p","ĠJagu ar","w it","Ġsem ic","ix ie","ĠG ould","Ġconf ines","N usra","ĠL on","ĠJ ugg","y cle","ĠCod ec","E gypt","Ġrest rain","ĠAl iens","Ġch oking","ĠD unk","ĠBell a","ab c","Ġsl ang","Ġneuro trans","s av","Ġempower ment","â ĨĴ","Ġclim bers","ĠM im","ĠF ra","ros se","Cap ital","ĠCth ulhu","Inter face","Ġprof icient","ĠIN TO","Ġ3 18","ront al","5 80","ĠDes pair","K enn","Ġscrim mage","ĠCo at","as ions","Ġwall paper","ĠJ ol","Ġresurg ence","Ġant iv","ĠB alls","² ¾","Ġbuff ers","Ġsub system","ĠSt ellar","ĠL ung","A IDS","Ġerad icate","Ġblat antly","Ġbehav es","ĠN un","Ġant ics","ex port","DE V","w b","Ġph p","ĠInteg rity","Ġexplore r","Ġrev olving","auth ored","g ans","Ġbas k","Ġas ynchronous","å į","TH ING","69 8","G ene","ĠR acer","ĠN ico","iss ued","Ġser mon","p ossibly","Ġsize of","Ġentrepreneur ial","ox in","ĠMin erva","Ġpl atoon","n os","ri ks","A UT","ĠAval anche","ĠDes c","ij 士","ĠP oc","Ġconf erred","Î »","Ġpat ched","F BI","66 2","Ġfract ures","Ġdetect s","Ġded icate","Ġconstitu ent","Ġcos mos","W T","Ġswe ats","Ġspr ung","b ara","s olid","Ġuns us","Ġbul ky","ĠPhilipp e","ĠFen rir","Ġtherap ists","ore al","^^ ^^","Ġtotal ed","Ġboo ze","ĠR PC","Prosecut ors","Ġdis eng","ĠSh ared","Ġmotor cycles","Ġinvent ions","Ġlett uce","ĠMer ge","ĠJ C","Ġspiritual ity","ĠWAR NING","Ġunl ucky","ĠT ess","Ġtong ues","ĠD UI","T umblr","Ġle ans","Ġinv aders","Ġcan opy","ĠHur ricanes","ĠB ret","ĠAP PLIC","id ine","ick le","Reg arding","Ġve ggies","Ġe jac","ju ven","F ish","D EM","ĠD ino","Th row","ĠCheck ing","be ard","( &","Ġj ails","Ġh r","trans fer","iv ating","Ġfle ets","ĠIm ag","ĠMc Donnell","Ġsnipp et","Is a","ĠCh att","ĠSt ain","ĠSet FontSize","ĠO y","ĠMathemat ics","49 4","Ġelectro ly","ĠG ott","ĠBr as","B OOK","ĠF inger","d ump","Ġmut ants","Ġrent als","Ġinter tw","Ġc reek","ail a","Bro ther","ĠDisc ord","pe e","raw ler","Ġcar p","Ġ27 9","ãĤ· ãĥ£","rel ations","Ġcontr asts","Col umn","Ġrec onnaissance","Ġun know","Ġl ooting","Ġregul ates","Ġopt imum","ĠChero kee","ĠA ry","Lat est","Ġroad side","Ġd anced","ĠUnic orn","A cknowled","Ġuncont roll","ĠM US","at io","ch ance","ha ven","VAL UE","Ġfavour ites","Ġceremon ial","b inary","pe ed","wood s","EM P","Ġv ascular","Ġcontempl ated","Ġbar ren","ĠL IST","Y ellow","ospons ors","Ġwhisk y","ĠM amm","ĠDeV os","min imum","H ung","44 2","P ic","ĠSnap dragon","77 6","Ġcar ving","Ġund ecided","Ġadvantage ous","Ġpal ms","ĠA Q","Ġst arch","L oop","Ġpadd le","Ġfl aming","ĠHor izons","An imation","bo ost","Ġprob abilities","ĠM ish","Ġex odus","ĠEditor ial","Ġfung us","Ġdissent ing","ĠDel icious","rog ram","ĠD yn","d isk","t om","Ġfab rics","ĠC ove","ĠB ans","Ġsoft en","ĠCON S","Ġin eligible","Ġestim ating","ĠLex ington","pract ice","of i","Ġshe dding","ĠN ope","Ġbreat hed","ĠCorinth ians","y ne","ek i","B ull","Ġatt aching","reens hots","Ġanaly se","ĠK appa","Ġuns ustainable","Ġinter pol","ank y","he mer","Ġprot agonists","Ġform atted","ĠBry ce","ĠAch illes","ĠAb edin","sh ock","Ġb um","b os","qu a","ĠW arn","q t","ĠDi abetes","8 64","ĠIn visible","Ġvan ish","Ġtrans mitting","Ġmur ky","ĠFe i","Ġawa ited","ĠJur assic","umm ies","Ġmen acing","g all","C ath","B uilt","ild o","ĠV otes","Ġon t","Ġmun itions","ĠFre em","ÃŃ n","Ġdec ency","lo pp","ie ved","ĠG ord","Ġun thinkable","ĠNews week","Ġ3 21","He at","Ġpresent er","ji ang","Ġpl ank","ĠAval on","Ġben z","ĠR out","Ġslam ming","ĠD ai","ou ter","ĠCook ie","ĠAlic ia","ge y","Ġvan ity","Ġow l","á µ","t ested","ĠAw akens","Ġcan v","Ġblind ly","ĠRid ley","ĠEm ails","Requ ires","ĠSer bian","ograp hed","if rame","eter ia","Ġaltern ating","qu iet","Ġsoc iology","ĠUn lock","ĠCommun ism","Ġo ps","Ġatt ribution","Ġab duction","ĠAb ram","Ġsidel ined","ĠB OOK","Ġref ining","ĠFe eling","ĠOs lo","ĠPru itt","r ack","ang ible","Ġcaut iously","ĠM ARK","eed s","M ouse","ĠStep h","ĠP air","S ab","99 7","ĠBa al","B ec","Ġcomm a","ĠP all","ĠG ael","Ġmisunder stand","ĠP esh","Order able","Ġdis mal","ĠSh iny","% \"","Ġreal istically","Ġpat io","ĠG w","ĠVirt ue","Ġexhaust ing","wh atever","oph ys","y ip","4 18","Ad just","ĠWa iting","ess on","ĠMaz da","ĠDo zens","Ġstream lined","Ġincompet ence","ĠM eth","Ġeth os","ON ES","Ġincent iv","Ġgr itty","ĠBut cher","Head er","Ġexp onential","à Ł","Ġcorrel ate","Ġcons ensual","s ounding","R ing","Orig in","Ġcon clusive","fe et","ac ly","ĠF ernandez","Buy able","Ġd ucks","aunt lets","Ġel ong","Ġ28 6","Ġsim ul","G as","ĠK irst","Ġprot r","ĠRob o","ĠAo E","op ol","Ġpsych ologically","sp in","ilater ally","ĠCon rad","W ave","44 1","ĠAd vertisement","ĠHarm on","ĠOri ental","is Special","Ġpresum ptive","Ġw il","ĠK ier","ne a","Ġp pm","Ġhar bour","ĠW ired","comp any","Ġcor oner","atur days","ĠP roud","ĠN EXT","ĠFl ake","val ued","ce iver","Ġfra ught","Ġc asing","Ġrun away","Ġg in","ĠLaure nt","ĠHar lem","ĠCur iosity","qu ished","Ġneuro science","ĠH ulu","Ġborrow er","Ġpetition er","ĠCo oldown","W ARD","Ġinv oking","conf idence","For ward","Ġst s","pop ulation","Delivery Date","Fil m","ĠC ov","quick Ship","quickShip Available","prim ary","isSpecial Orderable","inventory Quantity","channel Availability","BO X","ĠMulti player","ĠJen ner","77 8","ĠM d","Ġ~ /.","M N","Ġchild ish","Ġantioxid ant","ĠChrom ebook","Ġ27 4","Ġscreen play","Ġadvent urous","ĠRelations hip","respons ive","ming ton","Ġcorner stone","ĠF ey","F IR","Ġrook ies","ĠF eaturing","Ġorig inate","Ġelectro des","ant es","Ġscript ures","Ġgl ued","Ġdiscont ent","Ġaff licted","lay out","B rave","Ġm osa","ĠQuant ity","ĠH ik","w inner","H ours","Ġent ail","ĠCell s","olog ue","Ġv il","Ġpre acher","Ġdecor ative","d ifferent","Ġprejud ices","ĠSm oking","ĠNotting ham","so Type","Ġrhyth ms","ĠAl ph","bl ast","Ste el","ĠDaniel le","Ġstr ife","Ġrem atch","so DeliveryDate","ĠF ork","t rip","ol ulu","hes es","C G","ĠPOLIT ICO","ost a","ĠDr ift","é¾įå ¥","é¾įå¥ ij士","Ġvet ting","ĠJin ping","ĠRec ession","Min or","ĠF raud","enf ranch","Ġconven ed","ĠNA ACP","ĠMill ions","ĠFarm ing","ĠW oo","ĠFl are","rit o","imm igrant","Ġvac ancy","ĠHE AD","ĠV aj","eg al","ĠV igil","Stud y","Ġru ining","Ġr acks","Ġhe ater","ĠRand olph","ĠBr ush","ĠT ir","Ø ¨","Ġc ov","% ]","Ġrecount s","ĠO PT","ĠM elt","Ġtr uce","Ġcas inos","Ġcrus ade","Ġcarn age","Ġstri pe","ĠK yl","Text ures","Ġ6 98","Ġpro clamation","Ġgood ies","Ġ........ ..","pro claimed","P olit","Ġtop ical","Ġspecial ize","ĠA min","g m","Ġanch ored","Ġbear ings","s ample","ĠHigh land","ĠAut ism","Ġmerc enary","Ġinterview er","L ER","ĠSom ers","Ġembry o","ĠAss y","Ġ28 1","ĠEd iting","ĠCh osen","6 60","Ġp ci","ĠThunder bolt","BI LL","Ġchuck led","jri wal","h of","Ġearth ly","() {","ind ependence","Ġdisp ers","ĠV endor","ĠG areth","Ġp als","P enn","ĠSub mit","ic um","Th u","Ġcl andestine","Ġcann ibal","ĠCl erk","E Stream","gal itarian","âĻ ¥","g ew","Ġhor rend","ĠL ov","ĠRe action","ocr in","Class ic","Ġecho ing","Ġdiscl osing","ĠIns ight","og un","ĠInc arn","upload s","pp erc","guy en","Ġ19 01","ĠB ars","68 7","Ġb ribes","ĠFres no","ur at","ĠRe ese","Ġintr usive","Ġgri pping","ĠBlue print","ĠR asm","un ia","man aged","ĠHeb do","Ġ3 45","Ġdec oding","Ġpo ets","Ġj aws","ĠF IGHT","am eless","ĠMead ows","ĠHar baugh","Inter view","ĠH osp","ĠB RA","Ġdelet ion","m ob","W alker","ĠMoon light","ĠJ ed","ĠSoph ia","Ġus ur","Ġfortun ately","ĠPut ting","ĠF old","Ġsan itation","Ġpart isans","IS ON","B ow","ĠCON C","ĠRed uced","ĠS utton","Ġtouch screen","Ġembry os","âĢ¢âĢ¢ âĢ¢âĢ¢","ĠK rug","com bat","ĠPet roleum","Ġam d","ĠCos mos","Ġpresc ribing","Ġconform ity","ours es","Ġplent iful","Ġdis illusion","ĠEc ology","itt al","Ġf anc","Ġassass inated","regn ancy","Ġperenn ial","ĠBul lets","Ġst ale","Ġc ached","ĠJud ith","ĠDise ases","All en","Ġl as","Ġsh ards","ĠSu arez","ĠFriend ship","inter face","ĠSupp orters","add ons","46 2","ĠIm ran","ĠW im","Ġnew found","ĠM b","An imal","Ġd arling","and e","Ġrh y","ĠTw isted","pos al","yn ski","Var ious","× ľ","ĠK iw","uy omi","Ġwell being","ĠL au","an os","Ġunm ist","Ġmac OS","Ġrest room","ĠOl iv","ĠAir ways","Ġtimet able","9 80","Ġrad ios","v oy","ias co","Ġcloud y","ĠDraw ing","Any thing","Sy ria","ĠH ert","st aking","Ġun checked","Ġb razen","ĠN RS","69 7","onom ic","est ablish","Ġl eng","Ġdi agonal","ĠF ior","L air","ĠSt ard","Ġdef icient","jo ining","be am","Ġomn ip","Ġbl ender","Ġsun rise","Mo ore","ĠF ault","ĠCost ume","ĠM ub","Fl ags","an se","Ġpay out","ĠGovern ors","ĠD illon","ĠBan ana","N ar","Ġtra iled","Ġimperial ist","um ann","ats uki","4 35","ĠRoad s","Ġsl ur","ĠIde ally","Ġt renches","C trl","Ġmir rored","ĠZ el","ĠC rest","Comp at","ĠRoll s","sc rib","ĠTra ils","omet ers","w inter","Ġimm ortality","il ated","Ġcontrad icts","un iversal","ill ions","ĠM ama","opt im","AT URE","Ġge o","et ter","ĠCar lo","4 24","Ġcanon ical","ĠStrongh old","n ear","Ġperf ume","Ġorche stra","od iac","Ġup he","Ġreign ing","vers ive","Ġc aucuses","ĠD EM","Ġinsult ed","Ġ---- --","ĠCr ush","Ġroot ing","ĠWra ith","Ġwh ore","Ġto fu","C md","ĠB ree","Ġ$ _","Ġr ive","ĠAd vertising","Ġw att","ĠH O","Ġpersu asive","ĠParam eters","Ġobserv ational","ĠN CT","ĠMo j","ĠSal on","Ġtr unc","Ġexqu isite","ĠMar a","Ġpo op","ĠAN N","Ex c","ĠWonder ful","ĠT aco","Ġhome owner","ĠSmith sonian","orpor ated","mm mm","Ġlo af","ĠYam ato","ĠInd o","Ġcl inging","á s","Ġimm utable","h ub","Or ange","Ġfingert ips","ĠWood en","ĠK idd","ĠJ PM","ĠDam n","C ow","c odes","48 2","Ġiniti ating","ĠEl k","ĠCut ting","Ġabsent ee","ĠV ance","ĠLil ith","G UI","Ġobsc ured","Ġdwar ves","ĠCh op","ĠB oko","Val ues","Ġmult imedia","Ġbrew ed","Reg ular","CRIP TION","ĠMort al","Ġa pex","Ġtravel er","Ġbo ils","Ġspray ing","Rep resent","ĠStars hip","4 28","Ġdisappro val","Ġshadow y","Ġlament ed","ĠRe place","ĠFran ç","67 7","d or","Ġunst oppable","Ġcoh orts","gy n","ĠClass ics","ĠAm ph","Ġsl uggish","ĠAdd iction","ĠPad res","Ġins cription","Ġin human","min us","ĠJere miah","at ars","Ter ror","ĠT os","ĠSh arma","ast a","c atch","Ġpl umbing","ĠTim bers","Sh ar","H al","ĠO sc","Ġcou pling","hum ans","Ġsp onge","Ġid ols","ĠSp a","ĠAdv ocate","ĠBe ats","lu a","Ġtick ing","Ġload er","ĠG ron","8 10","Ġstim ulated","Ġside bar","ĠManufact urer","ore And","19 73","Ġpra ises","ĠFl ores","dis able","ĠElect rical","ra ise","E th","Ġmigr ated","Ġlect urer","K ids","ĠCa vern","Ġk ettle","Ġgly c","ĠMand ela","ĠF ully","å§ «","FIN EST","Ġsquee zing","ĠRy der","amp oo","oreAnd Online","Inst oreAndOnline","Buyable InstoreAndOnline","Ġcommem orate","ĠRamp age","Aust in","ĠSh roud","ĠRu ins","9 15","ĠK H","Ġwater front","ĠE SC","b aby","ĠC out","ĠEm blem","Ġequival ents","49 2","Un ique","ĠNiet zsche","brow ser","Ġim itation","ĠWere wolf","ĠKir in","ac as","' ,\"","Ġà ¾","Review ed","Ġc unt","Ġvo ic","ĠLen ovo","Ġbond ed","48 1","Ġinhib itors","Ġendeav ors","ĠHav ana","ĠSt out","ĠJ olly","A ctor","*/ (","Ġoccur rences","ĠT ens","Incre ased","ĠACT ION","Ġ ãĢĮ","ĠRank ings","ĠB reat","Ġ30 9","D ou","Ġimpact ing","ĠDuc hess","pre fix","Q B","Ġsummon ing","Ġbest owed","ĠKe pler","ĠPOW ER","c ube","ĠK its","ĠG rip","Ġop ium","Ġrep utable","t oc","ich ael","ĠR ipple","Ġcaf é","ĠZ oom","ĠBur ma","Ġwa ive","Ġst alls","Ġdem eanor","inc erity","Ġfluor ide","ĠSH OULD","Par is","Ġlong ing","Ġpl at","Ġgross ly","Ġbull s","Ġshowc asing","ex pected","ĠG addafi","engine ering","Re peat","ĠK ut","Ġconce ivable","Ġtrim med","osc ope","ĠCand idate","ĠT ears","rol og","Lew is","S UP","Ġroad map","Ġsal iva","Ġtrump et","Jim my","Ġmirac ulous","Ġcolon ization","Ġam put","ĠGN OME","ate ch","D ifferent","ĠE LE","ĠGovern ments","ĠA head","ãħĭ ãħĭ","word press","L IB","ĠIn clude","ĠDor othy","0 45","ĠColomb ian","Ġle ased","88 4","Ġde grading","ĠDa isy","i ations","Ġbapt ized","Ġsurn ame","co x","Ġblink ed","ãĥ ¢","Ġpoll en","Ġder mat","Ġre gex","ĠNich olson","ĠE ater","ç ľ","rad or","Ġnarrow er","Ġhur ricanes","Ġhalluc inations","r idden","ISS ION","ĠFire fly","Ġattain ment","Ġnom inate","Ġav ocado","ĠM eredith","Ġt s","Ġreve rence","Ġe uph","Ġcr ates","ĠT EXT","Ġ4 43","Ġ3 19","J SON","iqu ette","Ġshort stop","ic key","Ġpro pelled","Ġap i","ĠTh ieves","77 9","Ġovers aw","Ġcol i","ĠNic ola","Ġover cl","ik awa","ĠC yr","Ġ38 4","78 9","ĠAll ows","10 27","Det roit","TR Y","set up","ĠSocial ism","Sov iet","s usp","ĠAP R","ĠShut down","Ġal uminium","zb ek","ĠL over","GGGG GGGG","Ġdemocr acies","Ġ19 08","ĠMer rill","ĠFranco is","gd ala","Ġtraff ickers","ĠT il","ĠGo at","Ġsp ed","ĠRes erv","Ġpro d","55 2","Ġc ac","ĠUn iv","ĠSch we","Ġsw irling","ĠWild erness","ĠEgg s","Ġsadd ened","Ġarch aic","H yd","Ġexcess ively","B RE","Ġaer ospace","ĠVo ices","Cra ig","Ġign ited","In itially","ĠMc A","Ġhand set","Ġreform ing","Ġfrust rations","ĠDead pool","ĠBel ichick","ract or","ĠRagnar ok","ĠD rupal","ĠApp roximately","19 20","ĠHub ble","arm or","ĠSar as","ĠJon as","Ġnostalg ic","Ġfeas ibility","Sah aran","Ġorb iting","Ġ9 70","R u","Ġsh in","ĠInvestig ators","Ġinconsist encies","ĠP AN","B G","Ġgraz ing","Ġdetect ors","ĠStart up","ĠFun ny","ĠNa omi","Consider ing","Ġh og","ut f","ce mic","Ġfort ified","ĠFun ctions","Ġcod ec","nut rition","H at","\" !","micro soft","55 8","ĠTh in","ĠA CE","Al ias","ĠO PS","p apers","P K","ãĢ İ","Ġimpro bable","N orthern","equ al","Ġlook out","Ġty res","ĠMod ified","ĠK op","Abs olutely","Ġbuild up","sil ver","Ġaud i","Ġgro tesque","ĠSab er","ĠPres byter","ON Y","Ġglac iers","ĠSho als","ĠK ass","ĠH RC","ĠNic ol","ĠL unch","ĠF oss","âĸ Ĵ","AD RA","ĠOne Plus","o ing","ground s","Ġincident al","Ġdatas ets","68 9","ĠClarks on","Ġassemb ling","ĠCorrect ions","Ġdrink ers","Ġqual ifiers","Ġle ash","Ġunf ounded","ĠH undred","Ġkick off","T i","Ġrecon cil","ĠGr ants","ĠCompl iance","ĠDexter ity","Ġ19 06","w arn","D allas","Max imum","n ard","av ia","be aut","ens itivity","tr ace","Ġpione ers","ĠF ract","ãĢ ı","Ġpre cept","Ġgloss y","ĠI EEE","Ac ross","Ġ6 80","S leep","che on","Ġsatir ical","ĠMin otaur","ĠCla ude","Ġr é","ape go","Ġcar rot","ĠSem in","ino a","Ġz o","Ind ependent","Ġdiagn oses","ĠC ue","M AR","Ġrend ition","ĠK ik","Ġpath ology","Ġselect s","Link edIn","Ġass ay","ĠD res","Ġtext ual","post ed","IT AL","ĠM aul","N eal","Ġinter connected","Ġerr atic","ĠVir us","Ġ5 30","Ġenvironmental ists","ĠP helps","Ġeng agements","ĠIN ST","Ġeconom ical","nox ious","Ġg earing","izz y","Ġfavor ably","ĠMcG ill","T erm","Ġh anged","Ġball park","ĠRe yes","Ġbe ware","ĠP sal","ĠMass acre","q i","Ġin accessible","acly sm","Ġfr ay","ill ac","Ġbitter ly","ĠCert ification","Mich igan","Ġir respective","al ore","Em pty","Ġendorse ments","Ġund et","f g","equ ipped","Ġmerc iless","ĠC ust","Ġimm ature","Ġvou cher","ĠBlack well","Ñ ı","h awk","dis ciplinary","ile e","ĠMak oto","ĠD ude","ãĥĩ ãĤ£","Y ears","Ġin ver","Ġsh aman","ĠY ong","ip el","ell en","ĠCath y","br ids","Ġs arc","65 1","N ear","Ġground work","Ġam az","Ġ4 15","ĠHunting ton","hew s","ĠB ung","Ġarbit rarily","ĠW it","ĠAl berto","Ġdis qualified","best os","46 1","Ġp c","Ġ28 4","ro bat","Rob in","Ġh ugs","ĠTrans ition","ĠOcc asionally","Ġ3 26","ĠWh ilst","ĠLe y","Ġspaces hip","cs v","Ġun successfully","ĠA u","le ck","ĠWing ed","ĠGrizz lies",". �","Ġne arer","ĠSorce ress","ĠInd igo","El se","8 40","let es","Co ach","Ġup bringing","ĠK es","Ġseparat ist","Ġrac ists","Ġch ained","Ġabst inence","lear ning","Ġrein stated","Ġsymm etry","Ġremind ers","ĠChe vy","Ġm ont","Ġexempl ary","ĠT OR","Z X","Ġqual itative","ĠSt amp","ĠSav annah","ĠRoss i","Ġp aed","Ġdispens aries","ĠWall s","ĠCh ronic","Ġcompliment ary","ĠBeir ut","Ġ+ ---","igs list","Ġcrypt ographic","mas ters","ĠCap itals","Ġmax imal","Ġent ropy","Point s","Ġcombat ants","l ip","ĠGl ob","ĠB MC","ph ase","th ank","HT TP","Ġcomm uter","Ġ\\( \\",".. /","ĠReg ener","ĠDO I","ĠActiv ision","Ġsl it","os al","RE M","Ġch ants","Y u","Ke ys","Bre xit","ĠFor ced","Ari zona","Ġsquad ron","IS O","ĠMal one","Ġ3 38","Ġcontrast ing","Ġt idal","Ġlib el","Ġimpl anted","Ġupro ar","ĠC ater","Ġpropos itions","M anchester","ĠEuro s","it amin","G il","ĠEl ven","ĠSe ek","ĠB ai","Ġredevelop ment","ĠTown s","ĠL ub","! \",","al on","K rist","Ġmeas urable","Ġimagin able","Ġapost les","Y N","7 60","Ġster oid","Ġspecific ity","ĠL ocated","ĠBeck er","ĠE du","ĠDiet ary","uts ch","ĠMar ilyn","Ġbl ister","ĠM EP","ĠK oz","ĠC MS","y ahoo","ĠCar ney","Ġbo asting","ĠC aleb","By te","read s","ad en","Pro blem","ĠWood ward","S we","S up","ĠK GB","Set up","Ġtac it","Ġret ribution","Ġd ues","ĠM ü",". ?","ä¸ Ń","p ots","Ġcame o","ĠP AL","educ ation","A my","like ly","g ling","Ġconstitution ally","ĠHam m","ĠSpe ak","Ġwid gets","br ate","Ġcra ppy","ĠI ter","Ġanticip ating","ĠB out","P ixel","ĠY ep","ĠLaur ie","Ġh ut","Ġbullet in","ĠSal vation","Ġch ats","ear able","Honest ly","AL TH","onse qu","c ult","isco very","ovy ch","Ġse lves","ĠSat oshi","S ounds","Ġconver gence","ĠRosen berg","19 74","Ġnas al","Ġfull est","Ġfer ocious","x us","ist e","AM S","Ġlobb ied","Ġso othing","ĠGun n","t oday","0 24","Ġinspir ational","ĠN BN","p b","g ewater","or ah","all owed","ĠCol iseum","Ġspecial izing","Ġinsane ly","ĠT ape","del ay","Ġt arn","ĠP ound","Ġmel anch","Ġdeploy ments","il and","Ġless en","Ġfur ry","ĠUE FA","Ġblood shed","ĠMe ier","ither ing","Ġhe irs","ĠJ aw","ax ter","ĠPublic ations","Ġal ters","int ention","ĠWinc hester","d etermination","ĠLif etime","th in","Mon ster","7 80","Ġapprox imation","Ġsuper markets","ĠSecond s","or os","h uge","Ġb ribe","ĠLIM ITED","un ed","Ġmis interpret","ĠIn jury","Ġ3 67","Ġthreshold s","ĠCarn ival","Ġgastro intestinal","Ġguid eline","Ġde ceived","f eatures","Ġpurported ly","ĠRon nie","ĠNew t","Ġsp acious","as us","Ġsuperhero es","ĠCyn thia","le gged","k amp","ch io","Ġth umbnail","ĠShir ley","ill ation","Ġshe ds","ĠZ y","E PA","Ġdam s","Ġy awn","n ah","ĠPe ggy","ĠE rie","ĠJu ventus","ĠF ountain","r x","don ald","al bum","ĠComp rehensive","Ġc aching","ĠU z","ulner ability","ĠPrinc iple","ĠJ ian","ing ers","cast s","ĠOs iris","ch art","t ile","ĠTiff any","ĠPatt on","ĠWh ip","Ġovers ized","J e","ĠCind erella","ĠB orders","ĠDa esh","M ah","Ġdog ma","Ġcommun ists","v u","Coun cil","Ġfresh water","Ġw ounding","Ġdeb acle","Ġyoung ster","Ġthread ed","ĠB ots","ĠSav ings","ãģ Ĥ","ol ing","oh o","Ġillum ination","M RI","Ġlo osen","tr ump","ag ency","ur ion","Ġmoment arily","ĠCh un","ĠBud apest","ĠAl ley","D isk","Ġaston ished","ĠCon quer","ĠAccount ing","h aving","ĠWe in","ĠAl right","Ġrev olver","Ġdel usion","Ġrelic s","Ġad herent","qu ant","Ġhand made","or io","Ġcomb ating","c oded","Ġquad ru","re th","N ik","ĠTrib al","ĠMyster ious","Ġin hal","ĠWin ning","ĠClass ification","ch anged","Ġun ab","Ġsc orn","icip ated","w l","ond uctor","Ġrein forcing","ĠChild hood","an ova","Ġadventure r","Ġdoctor al","ĠStrateg ies","Ġengulf ed","ĠEnc ounter","Ġl ashes","Crit ical","ric ular","ĠU TF","oci ation","check ing","ĠConsult ing","Run time","per iod","ĠAs gard","Ġdist illed","ĠPas adena","ĠD ying","ĠCOUN TY","Ġgran ite","Ġsm ack","Ġparach ute","ĠS UR","Virgin ia","ĠF urious","78 7","ĠO kin","Ġcam el","ĠM bps","19 72","ĠCh ao","ĠC yan","j oice","ef er","ĠW rap","ĠDeb ate","S eg","Ġfore arm","ĠIgn ore","Ġtim estamp","Ġprob ing","ĠNo on","ĠGra il","f en","Ġdorm ant","ĠFirst ly","ĠE ighth","ĠH UN","ĠDes ire","or as","Girl s","ĠDes mond","z ar","am ines","O AD","exec ute","Ġbo obs","ĠAT L","_ (","Chel sea","Ġmasturb ation","ĠCo C","Ġdestroy er","ĠCh omsky","Ġsc atter","ĠAss ets","79 6","ĠC argo","Ġrecept ive","ĠSc ope","Ġmarket ers","Ġlaun chers","Ġax le","ĠSE A","se q","ĠM off","f inding","ĠGib bs","Georg ia","extreme ly","N J","Ġlab orers","st als","Ġmed iation","ĠH edge","at own","Ġi od","des pite","v ill","J ane","ex istence","Ġcoinc ided","ĠUt ilities","ĠChe ap","Ġlog istical","Ġcul mination","ĠNic otine","p ak","F older","Ġrod ents","st uff","Ġlaw fully","Ġreper to","io ch","j j","Dial ogue","HH HH","lic tion","Look s","Ġ29 7","Ġtur rets","ĠAb andon","Ġinc ess","ĠTraff ord","Ġcur led","Ġprefer ring","Ġprivat ization","Ġir resist","ĠP anda","ĠSh ake","ĠMc Gr","ãĥ Ħ","und ers","Ġdiscrim inated","Ġbart ender","I LE","Atl antic","Ġprop ensity","ĠW iz","ĠG im","con ference","Ġrein forces","G h","w agon","Ġe erie","F al","Ġhug ged","rac ist","R IC","F u","Ġf iller","ĠSt ub","Ġeng raved","ĠWrest le","Ġimagin ative","ĠPe er","ĠFact ors","an us","ĠDrac ula","mon itor","Ġrou ters","ib ia","ĠBoo lean","end ale","ĠSl aughter","ĠSh ack","R FC","ĠSpiel berg","S ax","ĠPH OTO","ĠCl over","ĠR ae","Dep ending","ĠMem or","ar am","Ġpier ced","Ġcur tains","v ale","ĠInqu isition","ĠP oke","Ġforecast ing","Ġcompl ains","S ense","ĠHer mes","isc overed","Ġb ible","ĠMor ph","Ġg erm","78 5","D ON","Ġcon gen","Ġcr ane","ĠD PR","Ġrespect fully","R oom","ĠN aw","ĠDal ai","re ason","ĠAng us","Educ ation","ĠTitan ic","Ë ľ","Ġo val","un ited","Ġthird s","Ġmoist ur","ĠC PC","M iami","Ġtent acles","ĠPol aris","ex c","ex clusive","ĠPra irie","Ġcol ossal","ĠBl end","sur prisingly","ÃŃ s","Ġindo ctr","Ġbas al","ĠMP EG","und o","Spl it","Develop ment","Ġlan tern","19 71","Ġprov ocation","Ġang uish","ĠB ind","ĠLe ia","duc ers","ipp y","conserv ancy","Ġinitial ize","ĠTw ice","ĠSu k","Ġpred ic","Ġdi ploma","Ġsoc iop","Ing redients","Ġhamm ered","ĠIr ma","Q aida","Ġglim ps","ĠB ian","Ġst acking","Ġf end","gov track","Ġun n","dem ocratic","ig ree","Ġ5 80","Ġ29 4","Ġstraw berry","ID ER","Ġcher ished","ĠH ots","Ġinfer red","Ġ8 08","ĠS ocrates","O regon","ĠR oses","ĠFO IA","Ġins ensitive","Ġ40 8","Recomm end","ĠSh ine","Ġpain staking","UG E","ĠHell er","ĠEnter prises","I OR","ad j","N RS","L G","Ġalien ated","Ġacknowled gement","ĠA UD","ĠRen eg","Ġvou chers","Ġ9 60","Ġm oot","ĠDim ensions","Ġc abbage","B right","g at","ĠK lu","Ġlat ent","Ġz e","ĠM eng","Ġdis perse","Ġpand emonium","H Q","Ġvirt uous","ĠLoc ations","ee per","prov ided","Ġse ams","ĠW T","iz o","PR OV","Ġtit anium","Ġrecol lection","Ġcr an","Ġ7 80","ĠN F","49 1","64 2","p acking","59 8","text ure","Sp ider","fre edom","cipl ed","ĠTAM ADRA","âĻ ¦","aut hent","ĠW ANT","r ified","Ġr ites","Ġuter us","k iss","Ġâī ¤","Ġsk illet","Ġdis enfranch","ĠGa al","Comp an","Ġage ing","gu ide","B alt","Ġiter ator","Ġdiscretion ary","t ips","Ġprim ates","ĠTechn ique","ĠPay ments","az el","ĠR OCK","stant ial","0 60","Ġd mg","ĠJack ets","ĠPlay off","Ġnurs ery","ĠSy mb","art on","Ġannex ation","Color ado","Ġco ils","ĠSh oes","âĦ¢ :","ĠRo z","COM PLE","ĠEve rest","ĠTri umph","J oy","G rid","à ¼","process or","ĠPros per","ĠSever us","ĠSelect ed","r g","ĠTay yip","St ra","Ġski ing","Ġ? )","Ġpe g","Tes la","Ġtime frame","Ġmaster mind","ĠN B","scient ific","ĠSh it","gener ic","IN TER","N UM","Ġst roll","ĠEn ix","ĠM MR","ĠE MS","m ovie","Ĥ ª","Ġminim izing","idd ling","Ġilleg itimate","Ġprot otyp","Ġpremature ly","Ġmanual s","obb ies","ĠCass idy","D EC","des ktop","Ġaer os","Ġscreen ings","Ġdeb ilitating","ĠGr ind","nature conservancy","Ġf ades","ter mination","assets adobe","F actor","Ġdefinitive ly","P oké","ap ult","ĠLaf ayette","C orn","ĠCor al","Ġstagn ant","T ue","Ġdissatisf action","G ender","Ġkid neys","ĠG ow","ĠDef eat","ĠAsh ton","Ġcart els","Ġfore closure","ĠExpl ore","stre ngth","ot in","Ġveterin arian","Ġf umble","Ġpar ap","ĠSt rait","r ils","Ġpr ick","ĠBerm uda","ĠAm munition","skin ned","Ġab ound","ĠB raz","Ġshar per","ĠAsc ension","Ġ9 78","Ġpreview s","Ġcommun ion","ĠX Y","Ġph ony","Ġnewcom er","Ġ3 32",".\" ,\"","Ġredist ribution","Prot ect","ĠSo f","K al","Ġlip stick","w orst","Ġtang led","Ġretrospect ive","int eger","Ġvolunte ering","Ġ19 07","Ġ --------------------","ic hen","Ġunve iling","Ġsen seless","Ġfisher ies","\\ -","Ġh inges","Ġcalcul us","My th","Ġund efeated","Ġoptim izations","Ġdep ress","Ġbill board","ĠY ad","ĠPy ramid","Is n","I de","Ġleg ion","ĠK ramer","ent anyl","Ġpenet rating","ĠHaw th","ĠPR ODUCT","ĠGer ard","ĠP act","ĠIn cluding","ĠEl ias","ĠEl aine","vis ual","Ġhum ming","Ġcond esc","ĠF asc","ä¸ Ĭ","Ġe galitarian","Ġdev s","ĠD ahl","O ps","D H","ĠB ounce","id ated","ald o","Ġrepublic an","Ġh amb","ĠS ett","ograph ies","CH APTER","Ġtrans sexual","Ġsky rocket","ans wer","Ġmark up","Ø ª","Ġhero ine","Comp are","ĠT av","Be ast","Ġsuccess ors","Ġna ïve","ĠBuck ley","st ress","me at","Ġdownload able","Ġindex ed","Ġsc aff","ĠL ump","ĠHom o","Stud io","In sp","Ġr acked","far ious","ĠPet ty","Ex ternal","Ġ19 09","W ars","com mit","put ers","Ġun ob","ĠEr r","ĠE G","ĠAl am","ĠSiber ia","ĠAtmosp heric","IS TER","ĠSatan ic","trans lation","ĠL oud","tra umatic","l ique","Ġreson ate","ĠWel ch","Ġspark ing","ĠT OM","t one","Ġout l","Ġhandc uffed","ĠSer ie","8 01","Ġland marks","ĠRee ves","Ġsoft ened","Ġdazz ling","ĠW anted","month s","Mag ikarp","Ġunt reated","ĠBed ford","M i","ĠDynam o","O re","79 5","Ġwrong ful","Ġl ured","Ġcort isol","Ġve x","d rawn","ile t","Download ha","ĠF action","Ġlab yrinth","Ġhij acked","w aters","er ick","Ġsuper iors","ĠRow ling","ĠGu inness","Ġt d","99 2","Ġune arthed","Ġcentr if","Ġsham eless","P od","ĠF ib","Ġ icing","Ġpredict or","Ġ29 2","fore station","con struct","C and","@ #","Ġag itated","Ġre pr","OV A","Ġkn itting","ĠLim a","Ġf odder","68 4","ĠPerson a","k l","7 01","Ġbreak up","á ¸","Ġapp alled","Ġantidepress ants","ĠSus sex","Har ris","ĠTher mal","ee ee","U pload","Ġg ulf","Ġdoor step","ĠSh ank","L U","ĠM EN","ĠP ond","s orry","Ġmis fortune","n ance","Ġb ona","M ut","Ġde graded","ĠL OG","ĠN ess","an imal","Ġa version","und own","Ġsupplement ed","ĠC ups","Ġ50 4","Ġdep rive","ĠSpark le","Å Ĥ","ĠMed itation","auth ors","ĠSab an","ĠN aked","air d","ĠMand arin","ĠScript ures","ĠPerson nel","ĠMahar ashtra","Ġ19 03","ĠP ai","ĠMir age","omb at","Access ory","Ġfrag mented","T ogether","Ġbelie vable","ĠGl adiator","al igned","ĠSl ug","M AT","Ġconvert ible","ĠBour bon","amer on","ĠRe hab","nt ax","Ġpowd ered","pill ar","Ġsm oker","ĠMans on","ĠB F","5 11","ĠGood ell","ĠD AR","m ud","g art","Ġob edient","ĠTrans mission","ĠDon ation","8 80","Ġbother ing","Material s","ãĤ ±","dest roy","Ġfore going","Ġanarch ism","ĠK ry","ice ps","Ġl ittered","ĠSch iff","Ġanecd otal","un its","Ġf ian","ĠSt im","ĠS OME","ĠInv aders","Ġbehaviour al","ĠVent ures","Ġsub lime","Ġfru ition","ĠPen alty","Ġcorros ion","¶ ħ","Ġlik ened","Ġbesie ged","ween ey","ĠCre ep","Ġlinem en","mult i","ic ably","ud der","Ġvital ity","Ġshort fall","ĠP ants","ap ist","H idden","ĠDro ps","med ical","Ġpron unciation","ĠN RL","Ġinsight ful","J V","ĠBe ard","ĠCh ou","Ġchar ms","Ġb ins","Ġamb assadors","ĠS aturdays","Ġinhib itor","ĠFr anch","6 01","', '","ĠCon or","art ney","ĠX peria","g rave","be es","ĠProtest ants","Ġso aking","ĠM andal","Ġph ased","Ġ6 60","Ġsc ams","Ġbuzz ing","ĠItal ians","ĠLoren zo","ĠJ A","Ġhes itated","Ġcl iffs","ĠG OT","ingu ishable","Ġk o","Ġinter ruption","Z ip","Lear ning","Ġundersc ores","ĠBl ink","K u","57 9","ĠAut ob","I RE","Ġwater ing","Ġpast ry","8 20","Ġvision ary","ĠTempl ar","awa ited","Ġpist on","Ġant id","current ly","Ġp ard","Ġw aging","Ġnob ility","ĠY us","Ġinject ing","f aith","ĠP ASS","å º","Ġret ake","ĠPR OC","Ġcat hedral","b ash","Ġwrest lers","Ġpartner ing","Ġn oses","Ġ3 58","Trans form","am en","Ġb outs","ĠId eal","ĠConstant in","Ġse p","ĠMon arch","att en","ĠPe oples","mod ified","Ġmor atorium","Ġpen chant","Ġoffensive ly","Ġprox ies","ok ane","ĠTaiwan ese","ĠP oo","ĠH OME","us ional","Ġver bs","ĠO man","vis ory","Ġpersu asion","Ġmult it","Ġsc issors","G ay","ow ay","oph ysical","l us","gn u","Ġap ocalyptic","Ġabsurd ity","Ġplay book","Ġautobi ography","I UM","Ġsne aking","ĠSim ulation","pp s","ell ery","Plan et","Ġright fully","Ġn iece","ĠN EC","ĠIP O","ĠDis closure","lean or","ous y","ST ER","Ġ28 2","Cru z","Ch all","64 3","ĠSurv ive","ĠF atal","ĠAm id","ap o","We apons","D EN","7 70","ĠGreen wald","Ġlin en","al os","Ġpollut ants","ĠPCI e","k at","Ġp aw","ĠK raft","C hem","ĠTermin ator","Ġre incarn","Ġ] [","ĠSe eds","Ġsilhou ette","ĠSt ores","Ġgro oming","ĠD irection","ĠIs abel","ĠBr idges","ðŁ ij","E ED","ĠM orsi","Ġval ves","ĠRank ed","ĠPh arma","ĠOrgan izations","Ġpenet rated","ĠRod ham","ĠProt oss","Ġove rest","Ġex asper","ĠT J","Ġ 000000","Ġtrick le","Ġbour bon","WH O","Ġw retched","Ġmicrosc opic","Ġcheck list","Ġad orned","R oyal","Ad minist","ĠRet irement","ĠHig hest","We ather","ile ge","Ġincre ments","ĠC osponsors","Ġmas se","ĠS inn","r f","Ġh ordes","as sembly","75 4","ĠNat asha","ĠTY PE","ĠGEN ERAL","Ġarr anging","Ġ40 7","l ator","Ġg lean","Ġdisc redited","Ġclin icians","UN E","Ġachie ves","ĠEm erson","com plex","= [","Ġprincip ally","Ġfra il","p icked","Ġthan king","Ġre cl","ĠL AST","Ġsupp ressing","il ic","Ġantidepress ant","ĠLis bon","Ġth or","Ġsp a","Ġking doms","ĠPear ce","em o","Ġpl ung","Ġdiv est","Ġ ********************************","b is","osp els","ad r","Sp irit","hall a","P ink","end ez","Ġresurrect ed","esc ape","ĠRosen stein","Ġge ological","Ġnecess ities","Ġcarn iv","ĠE lys","ĠBar ney","Ġ29 6","dig y","ST ON","D OWN","Ġmil estones","Ġk er","Ġdismant ling","Ġre prim","Ġcross ings","19 45","Ġpatri archy","Ġblasp hemy","Ġ3 59","met ry","ĠOb esity","ĠDiff erences","bl ocking","ãĥķ ãĤ¡","ich ita","ĠSab ha","ph alt","ĠCol o","ual a","effic ients","ĠMed ina","con sole","55 7","ĠHann ibal","ĠHab it","ĠF ever","Ġthen ce","Ġsyn agogue","Ġessential s","Ġw ink","ĠTr ader","ID A","ĠSp oiler","ĠIceland ic","ĠHay ward","Ġpe ac","Ġmal ice","Ġflash back","Ġth w","Ġlay offs","L iquid","Ġtro oper","Ġh inge","ĠRead ers","Ph ill","ĠB auer","Cre ated","Ġaud its","ac compan","Ġunsus pecting","ier a","6666 6666","Ġbro ch","Ġapprehend ed","ĠM alk","cer ning","ĠCod ex","O VER","M arsh","ĠD eng","ĠExp ression","Ġdisrespect ful","Ġasc ending","t ests","ĠPlaint iff","ster y","ĠAl ibaba","din and","ĠDem psey","Applic ations","mor al","Ġthrough put","Ġquar rel","Ġm ills","Ġhe mor","ĠC ASE","terror ist","st im","ifest yle","ro zen","CE PT","Ar k","u ci","lect ic","Ġirrit ating","she ets","A y","Ġrede emed","Ġhorn y","ĠTe ach","ĠS ear","dem ocracy","4 65","ĠRest ore","Ġstand by","ĠP is","iff in","Ġsleep y","Ġextr ater","Ġcompl iments","Fram eworks","Ġinstall s","Ġb anging","sur face","found land","Ġmetaph ysical","Ġ28 3","oul s","dev ices","Ar gs","ĠSac rifice","ĠMcC orm","es on","Cons ervative","ĠM ikhail","see ing","is ively","ĠRo oms","ĠGener ic","Ġenthusi astically","Ġgri pped","Ġcomed ic","ĠElectric ity","Ġgu errilla","Ġdec oration","ĠPerspect ive","Ġconsult ations","Ġun amb","Ġplag iar","Ġmagic ian","Ġe rection","ĠTour ism","or ied","ro xy","11 00","T am","Ī è","Î ³","× ª","ĠPred ators","Nit rome","Ġtelesc opes","project s","Ġun protected","Ġst ocked","ĠEnt reprene","nex pected","Ġwast ewater","V ill","Ġint imately","Ġi Cloud","ĠConst able","Ġspo of","Ġne farious","Ġfin s","Ġcens or","ĠMod es","ĠEs per","ar bon","Ġinter sections","Ġlaud ed","Ġphys i","Ġgener ously","ĠThe Nitrome","ĠTheNitrome Fan","Ġar isen","ĠÙ Ī","Ġg lands","ĠPav ilion","ĠGu pta","Ġuniform ly","Ġr amps","ri et","ĠWH EN","ĠVan essa","Ġrout ed","Ġlim p","ĠC PI","p ter","int uitive","Ġv aping","Ġexperiment ed","ĠOlymp us","ĠAm on","Ġsight ing","Ġinfiltr ate","ĠGentle man","Ġsign ings","ĠMe ow","ĠNav igation","che cks","4 33","Ġel apsed","ĠBulg arian","esp ie","ĠS OM","d uring","Ġsp ills","anc a","ĠPly mouth","M AL","Ġdomest ically","ĠWater gate","ĠF AM","k illed","ed ited","ĠYour self","Ġsynchron ization","ĠPract ices","ST EP","Ġgen omes","ĠQ R","not ice","Ġloc ating","z in","Ġ3 29","al cohol","Ġk itten","V o","Ġr inse","Ġgrapp le","ĠSc rew","ĠD ul","A IR","Ġle asing","ĠCaf é","Ġro ses","ĠRes pect","Ġmis lead","Ġperfect ed","Ġnud ity","Ġnon partisan","ĠCons umption","Report ing","Ġnu ances","Ġdeduct ible","ĠSh ots","Ġ3 77","Ġæ ľ","ano oga","Ben ef","ĠB am","ĠS amp","if ix","Ġgal van","ĠMed als","rad ius","Ġno bles","Ġe aves","igr ate","K T","ĠHar bour","u ers","Ġrisk ed","re q","Ġneuro t","get table","ain a","Rom ney","Ġunder pin","Ġlo ft","ĠSub committee","ĠMong ol","b iz","Ġmanif ests","ass isted","ĠG aga","Ġsy nergy","Ġreligious ly","ĠPre f","ĠG erry","T AG","ĠCho i","4 66","beh ind","ĠO u","Gold Magikarp","Ġhemor rh","R iver","Ġtend on","Ġinj ure","ĠF iona","Ġp ag","Ġag itation","|| ||","ur an","ĠE SA","Ġest eem","Ġdod ging","Ġ4 12","r ss","Ġce ases","ex cluding","Ġint akes","Ġinsert s","Ġemb old","ĠO ral","up uncture","4 11","ĠUn ified","ĠDe le","Ġfurn ace","ĠCoy otes","ĠBr ach","L abor","Ġhand shake","Ġbru ises","Gr ade","éĹ ĺ","ĠGram my","ile en","St ates","ĠScandinav ian","ĠKard ash","8 66","Ġeffort lessly","ĠDI RECT","ĠTH EN","ĠMe i","ert ation","19 68","Ġgro in","w itch","Requ irements","98 5","Ġroof s","Ġest ates","ĠH F","Ġha ha","Ġdense ly","ĠO CT","Ġpl astics","Ġincident ally","ĠTr acks","ĠTax es","Ġch anted","Ġforce ful","ĠBie ber","ĠK ahn","K ent","ĠC ot","lic ts","F ed","Ġhide ous","ĠVer d","ĠSynd icate","ĠIl legal","J et","ĠD AV","re asonable","c rew","Ġfundamental ist","Ġtruth ful","ĠJ ing","Ġl il","Ġdown ed","Ġen chanted","ĠPolic ies","ĠMcM aster","ĠH are","ides how","Ġpar ams","en cers","gorith m","Ġallow ances","Ġturb ulent","Ġcomplex ities","ĠK T","Ġ3 37","ĠGen etic","F UN","D oug","t ick","Ġg igs","ument hal","Ġpatriarch al","Ġcal c",", ...","Ġc out","ĠGu an","Ġpath ological","ĠR ivals","Ġunder rated","Ġflu orescent","ĠJ iu","arna ev","ĠQu an","Ġ4 29","Ġ à¨","M ario","Con struct","ĠC itation","ĠR acial","ĠR SA","ĠF idel","Ġ3 95","Person ally","C ause","à »","rad ical","in en","Ġvehement ly","ĠPap a","Ġintern ship","Ġfl akes","ĠRe ck","Luck ily","B ra","20 20","rav ings","R N","W onder","Ser iously","Ġre usable","Ġpoll uted","ĠP eng","le igh","ind le","Ġcircuit ry","ĠMad onna","ĠB ART","Res idents","att ribute","Phil adelphia","Cl ub","Ġplan ner","Ġfr antically","Ġfaith fully","ĠTerrit ories","ĠL AT","ĠAnders en","an u","ĠP ARK","ĠS ora","i age","ĠPlay offs","ĠG CC","4 27","Ġab norm","ĠL ever","Ġdisob edience","As ync","ĠShe a","V ert","Ġsk irts","ĠSaw yer","x p","Ġwors ening","Ġsc apego","ĠAng le","oth al","Ġtro ve","ĠSt y","ĠN guyen","mar ine","ide on","Dep ths","Bl og","ĠIll uminati","Ġtract s","Ġorgan ise","Ġo str","F s","Ġlever aging","ĠD aredevil","as ar","Ġl ang","Ġex termin","urs ions","ĠRom o","ãĤ¤ ãĥĪ","Ġcont ended","Ġencounter ing","ĠTable t","ĠAltern ate","sk ill","Ġswe ets","Ġco hesive","cap acity","Ġrep ud","Ġl izard","ro o","Ġpilgr ims","ĠR uff","ĠInstr ument","ĠLog o","uit ous","E H","Ġsales man","Ġank les","L ed","ĠPat ty","ud os","Own er","Ġdiscrep ancies","k j","M U","Ġuncond itional","Dragon Magazine","i ard","O ak","ĠConvers ation","be er","ĠOs aka","D elta","us ky","Ġsecret ion","Ġpl aza","Ġm ing","Ġde pletion","ĠM ous","ĠI TS","ĠH imal","ĠFle ming","Ġcyt ok","ĠH ick","Ġbat ters","ĠInt ellectual","6 75","é r","IS ION","ĠQu entin","ĠCh apters","ih adi","Ġco aster","WAY S","ĠL izard","ĠY or","and ering","S kin","ha ust","ab by","Ġportray ing","Ġwield ed","d ash","Ġprop onent","Ġr ipple","Ġgrap hene","Ġfly er","Ġrec urrent","Ġdev ils","Ġwater fall","æĺ ¯","go o","Text Color","Ġtam pering","IV ES","TR UMP","ĠAb el","ĠS AL","ĠHend ricks","ĠLu cius","b ots","Ġ40 96","IST ORY","Gu est","ĠN X","in ant","Ben z","ĠLoad ed","ĠCle ver","t reatment","Ġta vern","Ġ3 39","ĠT NT","ific antly","Tem perature","F el","Ġunder world","ĠJud ges","Ġ< +","Ġst ump","Ġoccup ancy","Ġab er","ĠF inder",") \",","ĠN unes","res et","in et","ect omy","Ġwell ness","ĠP eb","quart ered","and an","Ġneg atives","ĠTh iel","ĠCl ip","ĠL TD","Ġbl ight","Ġreperto ire","K yle","Ġqu er","ĠC es","Ġha pl","98 9","ĠTh ames","isc opal","Des k","ivari ate","ĠEx cellence","found ation","Ġâ ĩ","X i","Ġmyster iously","esty les","Ġper ish","ĠEng els","ĠDE AD","09 0","}} }","ĠUn real","Ġrest less","ID ES","orth odox","ĠInter mediate","Ġdin ners","ĠTr out","ĠSe ym","ĠHall s","og ged","Ġtraged ies","Ġdid nt","67 6","Ġail ments","Ġobserv able","ĠV ide","ad apt","ĠD usk","Ġprofessional ism","ĠPres cott","ĠInd ies","p ox","ĠMe hran","W ide","Ġend emic","ĠPar an","B ird","Ġped als","ĠI U","ĠAdam ant","ĠH urt","Ġcorrel ates","urd en","Ġspons oring","cl imate","ĠUnivers ities","ĠK not","enn es","ĠDam ian","ĠAx el","S port","Ġbar b","ĠS no","sh own","ste en","ud ence","Ġnon violent","Ġhom ophobia","Ġbiom ass","ĠDet ail","Ġsrf N","ĠT une","accompan ied","I ENCE","Al bert","ĠMong o","z x","ĠCer berus","or bit","c ens","Ġsl ay","SH ARE","H Y","Ġb rawl","ĠPro be","Ġnonex istent","ĠClare nce","ĠBlack burn","Ġport als","ĠR ita","ĠRem ain","ĠLe vant","Ġtrick ed","ĠF erry","aver ing","ĠStraw berry","ĠAn swers","Ġhorrend ous","ĠA man","Supp lement","ĠT oad","Ġpe eled","Ġman oeuv","ĠU zbek","mond s","ĠH ector","Ġ40 2","pe es","fix es","Ġd j","Ġres umes","Ġaccount ant","Ġadvers ity","Ġham pered","ĠL arson","Ġd oping","part s","H ur","Ġbe arded","Ġy r","ĠPlug in","å¥ ³","Ġ/ **","rol ley","Ġwaters hed","ĠSub mission","if lower","AS C","Ġcho ir","Ġsculpt ures","m A","incre asing","ai i","Ġsne akers","Ġconfront s","ĠEle phant","ĠEl ixir","Ġrec al","ĠT TL","w idget","ĠW ax","ĠGr ayson","Ġha irst","Ġhumili ated","ĠWAR N","app iness","ĠT TC","F uel","Ġpol io","Ġcomplex es","Ġbab e","ĠX IV","P F","). [","P arts","Ġ4 35","M eg","ĠY ards","ĠAL P","Ġy ells","Ġprin ces","Ġbull ies","ĠCapital ism","ex empt","FA Q","ĠSp onge","ĠAl a","Ġpleas antly","Ġbu f","Ġden ote","Ġunp ublished","Ġkne eling","asc a","Ġl apse","al ien","99 4","Ġrefere es","ĠLaw yers","S anta","Ġpuzz ling","ĠProm etheus","ĠPh araoh","ĠDel ay","Ġfacilit ates","ĠC ES","Ġjew els","Ġbook let","ond ing","Ġpolar ization","ĠMor an","ĠSal ad","ĠS OS","ĠAdv ice","PH OTOS","IC AN","iat ures","ex press","ĠWonder land","ĠC ODE","ĠCL ASS","9 75","Ġg rep","ĠD iesel","ĠGl ac","! ?\"","Ġr m","o ine","disc rimination","ĠN urse","m allow","Ġv ortex","ĠCons ortium","Ġlarge Download","stra ight","augh lin","G rad","Ġpublic ized","ĠW aves","ĠRed d","Ġfest ivities","ĠM ane","ar ov","Ġfleet ing","ĠDr unk","ug en","C ele","Ġchromos omes","ĠD OT","-+-+ -+-+","Ġbus iest","ĠBe aver","Sy rian","ĠK yr","k as","ĠCross Ref","19 50","76 01","Ġrepe aling","ĠWin ners","ĠMac ro","ĠD OD","bl ance","S ort","64 1","Ġmet re","ĠD irk","Ġgo ggles","Ġdraw backs","Ġcomplain ant","Ġauthor izing","Ġantit rust","oper ated","Ġm ah","Ġexagger ation","Am azing","ĠSer aph","Ġha ze","w ow","Ġextingu ished","Ġcan yon","ĠB osh","Ġv ents","Ġsc rape","Cor rect","4 26","Ġav g","Dem and","ĠâĪ ¼","Ġmicrobi ota","\"} ],\"","ĠSt ev","B io","ĠPlan es","Ġsuggest ive","Ġdec ipher","ĠRefuge e","ĠKe jriwal","ĠGreen peace","Ġdecl ass","ĠSound ers","Ġth o","Ġdec rypt","Ġbr ushing","ĠJane iro","ip op","S i","8 77","ĠGeoff rey","Ġc pu","ĠHaz el","Ġview points","Ġcris py","ĠNot ification","Ġsold er","ĠMod est","ĠHem isphere","Ġcass ette","in cludes","Ġident ifiers","ĠC ALL","in cent","T odd","ĠSwe ep","Ġ3 34","b oss","Ġsm ir","gin x","Ġtown ship","Ġg rieving","ĠMos que","Net flix","AS ED","ĠMillenn ials","oc om","19 67","Ġbold ly","s leep","Ġes che","arij uana","Ġsw irl","ĠPen al","Ġneglig ent","ĠStephen son","K ER","ĠZ oro","ris is","Ġlocal ization","ĠSeym our","ĠAng lic","red itation","prot ection","ĠPa ige","Ġo mit","ĠR ousse","ĠT ub","Ġinv itations","t ty","Ġm oss","ph ysical","C redits","Ġan archy","Ġchild care","Ġl ull","ĠM ek","ĠL anguages","lat est","ĠSan ford","Ġus ability","Ġdiff use","ĠD ATA","Ġsp rites","ĠVeget a","ĠProm otion","ãĥ¼ ãĤ¯","rict ing","z ee","Tur kish","ĠTD s","pro ven","57 1","Ġsmug glers","707 10","Ġreform ed","ĠLo is","Ġun fl","ĠWITH OUT","ĠReturn ing","ann ie","ĠTom as","Fr anc","ĠProf it","ĠSER V","ĠR umble","ik uman","es an","Ġt esters","Ġgad get","Ġbrace let","ĠF SA","comp onent","Ġparamed ics","Ġj an","ĠRem em","ĠSk inner","Ġl ov","ĠQu ake","rom a","Ġfl ask","Pr inc","Ġover power","Ġlod ging","ĠK KK","ret te","Ġabsor bs","w rote","Ġ ,\"","K ings","ĠH ail","ĠFall ing","xt ap","ĠHel ena","ire ns","L arry","Ġpamph let","ĠC PR","G ro","ĠHirosh ima","Ġhol istic","\". [","Ġdet achment","Ġas pire","Ġcompl icit","ĠGreen wood","Ġresp awn","ĠSt upid","ĠFin ished","f al","b ass","Ġab hor","Ġmock ery","ĠFe ast","VID EO","Ġcon sec","ĠHung ry","P ull","ĠH ust","it ance","? ãĢį",") --","ĠPar allel","con v","4 69","ha ar","w ant","P aper","m ins","ĠTor o","ĠTR UMP","ĠR ai","D W","ĠW icked","ĠL ep","Ġfun ky","Ġdetrim ent","ios is","ache v","Ġde grade","im ilation","Ġret ard","Ġfrag mentation","Ġcow boy","ĠY PG","ĠH AL","Parent s","ĠS ieg","ĠStra uss","ĠRub ber","× IJ","Fr ag","Ġp t","Ġoption ally","ĠZ IP","ĠTrans cript","ĠD well","88 2","M erc","ĠM OT","ãĥ¯ ãĥ³","Ġhun ts","Ġexec utes","In cludes","Ġacid ic","ĠRespons ibility","ĠD umb","we i","And erson","ĠJas per","ight on","abs olutely","Ad ult","Ġpl under","Mor ning","ĠT ours","ĠD ane","Î º","ĠT EST","ĠG ina","Ġcan ine","aw an","Ġsocial ists","ĠS oda","Ġimp etus","ĠSupplement ary","oli ath","ĠKinn ikuman","mitted ly","second s","Ġorganis ers","Ġdocument aries","Vari able","GRE EN","Ġres orts","Ġbr agging","Ġ3 68","Art ist","w k","bl ers","Un common","ĠRet rieved","Ġhect ares","Ġtox in","r ank","Ġfaith s","ĠG raphic","Ġve c","ĠL IA","Af rican","Ġard ent","end iary","L ake","ĠD OS","cient ious","ĠOk awaru","ĠAll y","ĠTim eline","D ash","ĠI c","contin ue","Ġt idy","Ġinstinct ively","ĠP ossibly","ĠOut door","ĠWould n","Ġl ich","ĠBr ay","ĠA X","Ġà ī","Ġ+ #","\\ '","Direct ory","ab iding","Ġf eral","ic ative","but t","Ġper verse","S alt","Ġwar ped","Ġnin eteen","Ġcabin ets","Ġsrf Attach","ĠSl oan","Ġpower ing","reg ation","F light","se vere","Ġst ren","Ġc og","ap ache","Ġâ Ŀ","Ġcaf eteria","p aces","ĠGrim oire","uton ium","Ġr aining","Ġcir cling","Ġlineback ers","c redit","Ġrep atri","ĠCam den","lic ense","Ġly ric","Ġdescript or","Ġval leys","Ġre q","Ġback stage","ĠPro hibition","ĠK et","Op ening","S ym","æĸ ¹","Ġserv ings","Ġoverse en","Ġaster oids","ĠMod s","ĠSpr inger","ĠCont ainer","è »","ĠM ens","Ġmult im","Ġfire fighter","pe c","Ġchlor ine","Ð ¼","end i","Ġsp aring","Ġpolyg amy","ĠR N","ĠP ell","Ġt igers","Ġflash y","ĠMad ame","S word","Ġpref rontal","Ġpre requisite","uc a","Ġw ifi","Ġmiscon ception","Ġharsh ly","ĠStream ing","ot om","ĠGiul iani","foot ed","Ġtub ing","ind ividual","z ek","n uclear","m ol","Ġright ful","49 3","Ġspecial ization","Ġpassion ately","ĠVel ocity","ĠAv ailability","T enn","Ġl atch","ĠSome body","Ġhel ium","cl aw","Ġdi pping","XX X","Ġinter personal","7 10","Ġsub ter","Ġbi ologists","ĠLight ing","Ġopt ic","Ġden im","end on","ĠC orm","Ġ3 41","ĠC oup","Ġfear less","Ġal ot","ĠCliff ord","ĠRun time","ĠProv ision","up dated","lene ck","Ġneur on","Ġgrad ing","ĠC t","sequ ence","in ia","con cept","Ġro aring","ri val","ĠCaucas ian","Ġmon og","key es","Ġappell ate","Ġlia ison","EStream Frame","ĠPl um","! .","Ġsp herical","Ġper ished","Ġbl ot","Ġben ches","Ġ4 11","Ġpione ered","Ġhur led","Jenn ifer","ĠYose mite","Ch air","Ġreef s","Ġelect or","ĠAnt hem","65 2","Ġun install","Ġimp ede","Ġbl inking","Ġgot o","Dec re","A ren","Ġstabil ization","ĠDis abled","ĠYanuk ovych","Ġoutlaw ed","ĠVent ura","ten ess","Ġplant ation","Ġy acht","ĠHu awei","Ġsol vent","Ġgr acious","Ġcur iously","Ġcapac itor","Ġc x","ĠRef lex","Ph ys","ĠC f","pt in","cons ervative","Ġinv ocation","c our","F N","ĠNew ly","H our","As ian","ĠLe ading","ĠAer ospace","An ne","Ġpre natal","Ġdeterior ating","H CR","ĠNorm andy","ol ini","ĠAm bro","9 10","Ġset backs","ĠT RE","Ġs ig","ĠSc ourge","59 7","79 8","Game play","Ġm sec","M X","Ġprice y","ĠL LP","aker u","Ġover arching","ĠB ale","Ġworld ly","Cl ark","Ġscen ic","Ġdisl iked","ĠCont rolled","T ickets","ĠE W","ab ies","ĠPl enty","Non etheless","Ġart isan","Trans fer","ĠF amous","Ġinf ield","ble y","Ġunres olved","ĠML A","ãĤ Ĥ","Cor rection","Ġdemocr at","ĠMore no","ro cal","il ings","Ġsail or","Ġr ife","h ung","Ġtrop es","Ġsn atched","ĠL IN","ĠB ib","ES A","ĠPre v","ĠCam el","run time","Ġob noxious","4 37","Ġsum mers","Ġunexpl ained","ĠWal ters","cal iber","Ġg ull","ĠEnd urance","ä½ ľ","Ġ3 47","Ir ish","Ġaer obic","Ġcr amped","ĠHon olulu","à ©","us erc","ec ast","AC Y","ĠQu ery","ãĤ¹ ãĥĪ","Bet a","Ġsuscept ibility","ĠSh iv","ĠLim baugh","Ġà ĸ","ĠN XT","ĠM uss","ĠBrit ons","ES CO","EG IN","Ġ% %","Ġsec ession","ĠPat ron","ĠLu a","n aires","ĠJPM organ","us b","ocy te","Ġcouncill ors","ĠLi ang","f arm","Ġnerv ously","Ġattract iveness","ĠK ov","j ump","Pl ot","Ġst ains","ĠStat ue","ĠApost les","he ter","ĠSUP PORT","Ġoverwhel m","Y ES","Ġ29 1","d ensity","Ġtra pping","M it","Ġf ide","ĠPam ela","atl antic","Dam n","Ġp ts","OP A","Ġserv icing","Ġoverfl owing","ul o","ĠE rit","t icket","light ing","ĠH mm","ãĥ¼ ãĥ«","im oto","Ġchuck le","4 23","ãģ ķ","sh ape","Ġque ues","Ġanch ors","ãĤ¼ ãĤ¦ãĤ¹","F er","Ġaw oke","Ġ6 66","h ands","Ġdiver gence","Ġ50 5","T ips","Ġdep ot","Ġske w","ĠDel iver","op ot","Ġdiv ul","ĠE B","uns igned","ĠUn i","X box","Ġfor ks","Ġ7 02","å ¯","Ġpromot ers","ĠV apor","Ġlev ied","sl ot","Ġpig ment","Ġcyl inders","C RE","Ġsn atch","Ġperpet ually","Ġl icking","ĠFe et","ĠKra ken","ĠHold en","ĠCLS ID","m r","Ġproject or","Ġden otes","Ġchap el","ĠTor rent","b ler","R oute","ĠDef endant","ĠPublisher s","ĠM ales","ĠInn ov","ĠAg ility","rit er","ty mology","st ores","L ind","Ġf olly","ĠZur ich","B le","Ġnurt ure","Ġcoast line","uch in","D omin","Ġfri vol","ĠCons olid","res ults","M J","Ġphyl ogen","Ġha uled","ĠW iley","ĠJess ie","ĠPrep are","ĠE ps","Ġtreasure r","I AS","Ġcolon ists","Ġin und","ĠWW F","ĠCon verted","6 000","out side","ĠApp earance","ĠRel ic","ĠM ister","s aw","Ġresult ant","Ġadject ive","ĠLaure l","ĠHind i","b da","Pe ace","Ġreb irth","Ġmembr anes","Ġforward ing","Ġcoll ided","ĠCar olyn","K ansas","5 99","ĠSolid GoldMagikarp","Be ck","Ġstress ing","ĠGo o","ĠCooper ative","Ġf s","ĠAr chie","L iter","ĠK lopp","J erry","Ġfoot wear","War ren","Ġsc ree","h are","Under standing","P ed","Ġanth ology","ĠAnn ounce","M ega","Ġflu ent","Ġbond age","ĠDisc ount","il ial","C art","ĠNight mares","Sh am","ĠB oll","uss ie","H ttp","Atl anta","Ġun recogn","ĠB id","Ġunder grad","Ġforg iving","ĠGl over","AAAA AAAA","4 45","V G","pa io","kill ers","Ġrespons ibly","Ġmobil ize","Ġeffect ed","ĠL umin","Ġk ale","Ġinfring ing","ann ounced","Ġf itt","b atch","ĠT ackle","ĠL ime","ĠAP P","uke mia","Ġrub y","Ġex oner","ĠCas ual","0 70","Ġpel vic","Ġautom ate","ĠK ear","ĠCoast al","Ġcre ed","Ġbored om","ĠSt un","ri ott","Ĥ İ","Ġregener ate","Ġcomed ians","ĠOP ER","Sp ons","id ium","on is","L ocated","05 7","Ġsusp ense","ĠD ating","C ass","Ġneoc ons","ĠShin zo","Ġaw oken","ch rist","ĠMess ages","att led","ĠSpr ay","ĠSp ice","C W","Ġshield ing","ĠG aul","Am id","Ġparam ilitary","Ġmult if","ĠTan ner","il k","Ġgodd amn","g ements","Ġbe friend","m obi","Ġ3 88","fold er","acc a","Ġins in","g ap","N ev","fif th","Ġpsychiat ry","b anks","TH IS","Ġhar b","ac qu","Ġfac ade","ĠPower Point","80 3","Ġbl uff","Sh ares","Ġfavor ing","El izabeth","Ãį Ãį","Ġr anger","77 2","ĠAr che","h ak","ĠGen etics","ĠF EMA","Ġev olves","Ġest e","ĠP ets","ĠM é","ĠInterest ing","ĠCanter bury","ch apter","ĠStar fleet","Sp anish","Ġdraw back","ĠNor wich","9 70","n orth","ag anda","Ġtransform ative","ram ids","bi ology","ad ay","Ġpropag ation","ĠGam ma","ĠDen ise","ĠCalcul ator","ent imes","ĠB ett","Ġapp endix","ĠHD D","AK ING","Ġst igmat","Ġhol ster","Ġord inarily","Ch ance","ĠCont rary","Ġad hesive","Ġgather s","6 12","re au","ony ms","ew ays","Ġindu ces","Ġinterchange able","se m","Wh it","Ġtr ance","Ġincorpor ation","ĠExt ras","Fin ancial","Ġawkward ly","ĠStur geon","ĠH Y","Norm ally","ĠEnd ing","ĠAss ist","enc rypted","Ġsub jug","Ġn os","Ġfan atic","C ub","C U","?\" .","Ġirre versible","å Ĥ","03 1","ĠH AR","sp read","ul ia","= $","Sc ope","L ots","Ġlif estyles","ol on","Ġf eds","Ġcongrat ulate","web kit","Ġindist inguishable","ĠSw ing","Ġcommand ments","qu ila","ab ella","m ethyl","ann abin","Ġo vere","Ġlob ster","ĠQU EST","ĠCONT IN","bern atorial",":::: ::::","ĠTra ve","ĠSam oa","AN I","75 2","Ð ´","userc ontent","ĠMod erate","y eah","ĠK itt","Ġwe e","Ġstuff ing","ĠInter vention","ĠD ign","Ġware houses","ĠF iji","Ġpel lets","Ġtake away","ĠT ABLE","ĠClass ical","col lection","Ġland fall","ĠMus cle","Ġsett les","ĠAD V","Ġ3 44","L aura","Ġf ared","ĠPart ial","4 36","oss ibility","ĠD aly","ĠT arant","ĠFu ji","am l","c ence","55 1","ĠProced ures","ĠO CD","ĠU D","t in","Q UI","ach o","4 38","Ġgl itches","Ġenchant ment","Ġcalcul ates","IR O","ĠH ua","alys es","ĠL ift","um o","Ġle apt","Ġhypothes ized","ĠGust av","it ans","VERS ION","æ ł","Rog er","Ġr and","ĠAd apter","Ġ3 31","ĠPet ition","k ies","M ars","Ġunder cut","ze es","ĠLy ons","ĠDH CP","Miss ing","Ġretire es","Ġins idious","el i","> )",". ãĢį","Ġfinal ists","ĠA ure","Ġacc user","Ġwas tes","ĠY s","ĠL ori","Ġconstitu encies","Ġsupp er","Ġmay hem","or ange","Ġmis placed","Ġmanager ial","Ġex ce","ĠCL I","Ġprim al","ĠL ent","Cry stal","h over","ĠN TS","end um","Ġd w","ĠAl c","n ostic","Ġpres erves","ĠTs arnaev","Ġtri pled","rel ative","Arc ade","k illing","ĠW EEK","ĠH anna","D ust","Com pleted","ģ «","Ġappro ves","ĠSur f","ĠLuther an","ven ants","Ġrobber ies","we ights","soft ware","at ana","ug al","Ġgrav y","ĠC ance","OLOG Y","ly ak","Ton ight","Ġunve il","Ġ19 04","ĠMin ion","ent ious","st ice","pack ages","ĠG EAR","Ġg ol","ĠHutch inson","ĠProf ession","ĠG UN","ĠDiff erence","ĠTsuk uyomi","ĠLes bian","6 70","Ġfug itive","ĠPlan etary","-------------------------------- ------------------------","Ġacc rued","Ġch icks","Ġsto pp","Ġblock ers","C od","Ġcomment ers","ĠSomew here","ĠPhot ographer","the me","Ġmay oral","w u","Ġanten nas","Ġrev amped","ĠSubject s","it é","im ura","Ġentr ances","liter ally","Ġten ets","ĠO MG","ĠMP H","ĠDon key","ĠOff ense","Ġ\" +","Sn ap","ĠAF B","Ġan imate","ĠS od","His panic","Ġinconsist ency","D b","F Y","Ex port","Ġa pe","Ġpear l","ib el","ĠPAC s","Ġ{ \\","Ġact u","ĠHS BC","camp us","Ġpay off","Ġde ities","ĠN ato","ou ple","Ġcens ored","ĠCl ojure","Ġconf ounding","en i","Ġreck on","op he","Ġspot ting","Ġsign ifies","Ġprop el","Ġfest ive","S uggest","Ġpled ging","ĠB erman","Ġrebell ious","Ġovershadow ed","Ġinfiltr ated","j obs","67 2","Ġscal able","Ġdomin ion","ĠNew foundland","ĠMead ow","Ġpart itions","AM I","Ġsupplement ary","str ument","Ġhair y","Ġperpet uate","Ġnuts hell","ĠPot ato","ĠHob bit","Ġcur ses","Flo at","Ġquiet er","Ġfuel ing","Ġcaps ules","ĠL ust","ĠH aunted","Exec utive","Ġchild birth","G re","Ġrad iant","å İ","Ġm alls","Ġin ept","ĠWarrant y","Ġspect ator","E h","t hens","Ġculmin ating","æ ©","ary a","ãĤ ®","ilit arian","ĠOR IG","ĠSp ending","pt ives","ĠS iren","ĠRec ording","ay ne","Ġv im","Ġspr ang","T ang","ĠM FT","mor ning","ĠWe ed","m peg","cess ion","ĠCh ung","7 30","w arning","56 2","handed ly","P oor","P olitics",": #","Ġp ian","Ġfec es","ĠDocument ation","Ġban ished","Ġ3 99","ĠAR C","Ġhe inous","J ake","ĠAm ir","way ne","v re","os henko","Ġnotebook s","Ġfound ational","Ġmarvel ous","ixt ape","Ġwithdraw als","Ġh orde","ĠD habi","is able","ĠK D","Ġcontag ious","ĠD ip","ĠAr rows","Ġpronoun s","Ġmorph ine","ĠB US","68 2","Ġk osher","fin ished","ĠInstr uments","Ġf used","yd en","ĠSal mon","F ab","aff ected","K EN","C ENT","Dom ain","Ġpoke mon","ĠDr inking","G rowing","ĠInvestig ative","ĠA ether","em i","Ġtabl oid","Ġrep ro","ĠNot withstanding","ĠBers erker","Ġdram as","Ġclich é","Ġb ung","ĠU RI","ĠD os","0 44","Ġpast ors","Ġl s","Ġac rylic","aun ts","Ed ward","Ġmajor ities","B ang","Ġfield ing","ĠRepl acement","ĠAl chemy","pp ard","ĠRome o","ĠSan ct","ĠLav rov","ib ble","Inst ruct","Ġimp ractical","ĠPlay boy","ce phal","Ġsw aps","Ġk an","ĠThe o","Ġillust rating","Ġdismant led","ĠTrans gender","ĠG uth","UG H","Ġtriumph ant","Ġencomp ass","Ġbook mark","udd in","j er","Ġpred icate","ES H","Ġwhen ce","ĠAB E","Ġnon profits","Se qu","Ġdi abetic","Ġp end","Ġheart felt","sh i","Ġinter acts","ĠTele com","Ġbombard ment","dep ending","ĠLow ry","ĠAd mission","ĠBl ooming","ust ration","ene gger","B rew","Ġmol ten","ĠNer d","P IN","âĸ Ģ","ave ment","Ġtou red","Ġco efficients","ĠTray von","ans son","Ġsand y","t old","fl ows","Ġpop ulous","ĠT inder","ĠBl iss","R achel","Min imum","Ġcontest ant","ĠRed uce","ĠMor se","ĠGrass ley","ĠClick er","Ġexp r","Ġs incerity","Ġmar qu","Ġelic it","ĠPro position","ĠDemon ic","Ġtac os","G reek","Ġpost war","Ġin sofar","ĠP ork","Ġ35 2","doctor al","walk ing","Ġmid term","ĠSam my","sight ed","ĠTR ANS","ic i","AL D","ĠUS L","ĠF ISA","ĠAm pl","ĠAlex andra","ine lli","Tr ain","Ġsign ify","ĠVers us","Ġob fusc","Ġk h","Ġagg ro","ĠRen ault","Ġ3 48","5 18","ox icity","0 22","ĠTw ist","Ġgoof y","D ynamic","Ġbrief ings","m ight","8 99","Ġderog atory","T ro","Ġfor ging","ĠKor an","ĠMar ried","ĠBuc s","Ġpal ate","ĠCon version","m able","4 13","Ġ( _","Ġs iph","ĠN EO","col lege","Ġmarg inally","Ġfl irt","ĠTra ps","ĠP ace","é »Ĵ","Ġgoalt ender","Ġforb ids","Ġcler ks","ĠT ant","ĠRobb ins","ĠPrint ing","Ġpremie red","Ġmagn ification","ĠT G","ĠR ouse","ĠM ock","odynam ics","Ġpre clude","ism o","ĠPul itzer","Ġaval anche","ĠK odi","rib une","ĠL ena","Elect ric","Ġref inery","Ġend owed","Ġcounsel ors","Ġd olphin","ĠM ith","Ġarm oured","hib ited","Beg in","ĠP W","O il","ĠV or","ĠShar if","ĠFraz ier","est ate","Ġj ams","Pro xy","Ġband its","ĠPresbyter ian","ĠPrem iere","t iny","ĠCru el","Test ing","Ġhom er","ĠV ERS","ĠPro l","ĠDep osit","ĠCoff in","Ġsemin ars","Ġs ql","ĠDef endants","Altern atively","ĠR ats","ç «","ethy st","' >","Ġiss uer","58 9","Ġch aired","ĠAccess ories","man ent","Ġmar row","ĠPrim ordial","C N","Ġlimit less","ĠCarn age","Ġund rafted","q v","IN ESS","on ew","Ġco hesion","98 7","Ġne cks","Ġfootball er","ĠG ER","Ġdetect able","ĠSupport ing","ĠCS V","oc ally","k Hz","Ġund e","Ġsh one","Ġbud ding","tra k","Stand ing","ĠStar craft","ĠKem p","Ben ch","Ġthw arted","ĠGround s","ath i","L isa","Dial og","ĠS X","V ision","Ġingen ious","Ù IJ","Ġfost ering","ĠZ a","ĠIn gram","Ġ\" @","N aturally","6 16","0 35","ĠF AC","H mm","55 4","Ġacceler ator","ĠV end","Ġsun screen","Ġtuber culosis","rav iolet","ĠFunction al","ĠEr rors","ed ar","19 66","ĠSpect re","ĠRec ipes","88 5","ĠM ankind","L iverpool","Ġ| --","Ġsubst itutes","ĠX T","w ired","Ġinc o","ĠAf gh","E va","ic c","S ong","K night","Ġdilig ently","ĠBroad cast","A id","Ġaf ar","ĠH MS","aton in","ĠGr ateful","Ġfire place","ĠOm ni","e uro","ĠF RE","ĠSh ib","ĠDig est","t oggle","Ġheads ets","Ġdiff usion","ĠSqu irrel","ĠF N","Ġdark ened","out her","Ġsleep s","ĠX er","gun s","Ġset ups","Ġpars ed","Ġmamm oth","ĠCur ious","g ob","ĠFitz patrick","ĠEm il","im ov","........ .....","ĠB enny","Second ly","Ġheart y","Ġcons on","st ained","Ġgal actic","cl ave","Ġplummet ed","Ġp ests","Ġsw at","Ġrefer rals","ĠLion el","h oly","Ġunder dog","ĠSl ater","ĠProv ide","ĠAm ar","ress or","å Į","ong a","Ġtim id","Ġp iety","ĠD ek","Ġsur ging","az o","Ġ6 10","Ġdes ks","ĠSp okane","ĠAn field","Ġwars hips","ĠCob ra","Ġar ming","clus ively","ĠBad ge","ag ascar","ĠPR ESS","ĠMcK enzie","ĠFer dinand","burn ing","Af ee","Ġtyr ann","ĠI w","ĠBo one","100 7","ĠRe pt","Ċ Âł","Ġcar avan","ĠD ill","ĠBundes liga","Ch uck","Ġheal er","ãĥ¼ãĥ Ĩ","ĠH obby","Ġneg ate","Ġcrit iques","section al","mop olitan","Ġd x","Ġouts ourcing","ĠC ipher","t ap","Sh arp","Ġup beat","Ġhang ar","Ġcru ising","ĠNi agara","Ġ3 42","ill us","ĠS v","Ġsubt itles","Ġsqu ared","Ġbook store","Ġrevolution aries","ĠCarl ton","ab al","Ut ah","Ġdesp ise","ĠU M","cons ider","aid o","Ġc arts","ĠT urtles","Tr aining","Ġhonor ary"," ¢","Ġtri angles","4 22","Ġreprint ed","Ġgrace ful","ĠMong olia","Ġdisrupt ions","ĠB oh","Ġ3 49","Ġdr ains","Ġcons ulate","Ġb ends","Ġm afia","ur on","ĠF ulton","m isc","Ġren al","Ġin action","ck ing","Ġphot ons","Ġbru ised","ĠC odes","og i","Ġn ests","ĠLove ly","ĠLib re","ĠD aryl","Ġ# ##","S ys",". ,\"","Ġfree zes","est ablishment","and owski","Ġcum bers","ĠSt arg","ĠBom bs","Ġleg ions","Ġhand writing","Ġgr un","ĠC ah","sequ ent","Ġm oth","ĠMS M","Ins ert","F if","Ġmot el","Ġdex ter","ĠB ild","hearted ly","Ġpro pe","ĠText ure","ĠJ unction","ynt hesis","oc ard","ĠVer a","ĠBar th","Ġμ g","Ġl ashed","Ġ35 1","ĠZ amb","ĠSt aples","ĠCort ex","ĠCork er","Ġcontinu um","ĠWR ITE","unt a","rid or","Ġde ems","0 33","ĠG OLD","p as","Ġrep ressive","ãĥĨ ãĤ£","Ġbaff led","Sc ar","Ġc rave","Ġ ______","Ġentrepreneurs hip","ĠDirector ate","Ġ' [","Ġv ines","Ġasc ended","ĠGR OUP","ĠGood bye","Ġdo gged","ãĥ´ ãĤ¡","Man ufact","Ġunimagin able","ri ots","ier rez","Ġrel ativity","ĠCraft ing","ra ught","ud en","c ookie","Ġassass ins","Ġdissatisf ied","ac ci","Ġcondu it","Sp read","ĠR ican","n ice","izz le","Ġsc ares","ĠWH Y","ph ans","5 35","Ġprot racted","ĠKrist en","5 36","ĠSc rib","ĠNe h","Ġtwent ies","Ġpredic ament","Ġhandc uffs","Ġfruit ful","ĠU L","ĠLud wig","Ġatt est","ĠBre aker","Ġbi ologically","ĠDeal er","Ġrenov ations","f w","ess en","Al ice","ĠHen ri","Ġun ilaterally","ĠS idd","h ai","ĠSt retch","S ales","Ġcumbers ome","ĠJ avier","Ġtrend y","Ġrot ting","ĠChall enges","Ġscra ps","Ġfac ets","ĠVer onica","ĠVer ge","ĠS ana","Al ien","ĠR ih","Ġrad ial","ect ar","Ġ6 30","cl i","Mar ie","Ġwild fire","ĠCat o","h ander","Ġwait ress","Ġch ops","ĠS ECTION","Ġblunt ly","ĠCat alog","n ian","stud y","Ġpat rolling","ĠT enth","nex us","ĠN ON","op sy","Ġsc athing","s ie","Ġdeterior ated","V B","Naz is","Ġdep ictions","Ġauthent icated","ĠCon ce","k rit","Ġpromul g","ĠL ONG","U FC","ĠVis itors","ĠRec all","Ġrehab ilit","ĠSL I","Ġglac ier","ĠB ite","Ġ50 3","Ġvom it","Ġfer mented","ĠKh alid","Ġgrad ed","ĠMag icka","ĠIch igo","power ful","ic ators","75 3","Ġsh rew","Ġ35 6","Ġlegal izing","Ġall otted","ĠArch demon","ith ing","igg urat","V OL","Le od","Ġo ily","Ġindu cing","Ġamy gdala","Ġadm ins","ĠAcqu isition","C AN","Ġsche matic","Ġmo an","ĠCamer oon","Ġt ink","Ġmer ry","Ġbutter flies","ĠGo ff","Ġworks pace","ĠCor ona","Ġj avascript","ĠD olphin","ĠCant or","4 64","to e","AP S","ĠAg ing","Ġpadd ed","ĠZ heng","ĠHe ld","Ġest ranged","Ġ7 70",". }","ĠDun ham","Ġsm okes","Ġcap itals","und ai","Sh in","ĠFound ing","Ġent itle","Ġcenter piece","D iscover","Ġthere to","al ert","ĠN ou","ĠAnaly st","l c","F H","FI ELD","ĠP OV","gr ay","Ġar cs","ĠH OT","Ġr s","Ġoblig atory","ĠArchitect s","ĠS ven","ĠF EC","0 200","Christ mas","ĠAlban ia","rat om","58 7","Ġhard ships","Ġaut os","ĠCharg es","Ġap es","Ġ3 76","wal let","Ġintox ication","Ġgobl in","Ġ5 70","++++++++ ++++++++","ĠYel p","ĠMag netic","ĠBr iggs","R ail","Ġspawn s","ĠW iggins","Ġshowc ased","Ġres orted","ub en","Ġwh ipping","Ġim itate","Ġdigest ion","ĠUS PS","ĠG est","Ġye a","ĠT ight","ind al","ic as","` .","C AST","'' ;","ĠF et","opath ic","In valid","Ġregrett ed","Ġbro ccoli","ĠSc ores","e ve","Ġpost ings","Ġaccum ulating","Ġneed less","elf th","Ġmay ors","Ġsc rib","Ġanecd otes","Ġbot ched","ĠRib bon","ĠConstant ine","i uses","ess es","Ġdev ise","Comp ared","Ġp udding","Ġg arg","Ġev oke","79 7","Ġdet ox","9 09","ĠPie ces","ĠMcC artney","Ġmet ast","ĠK rypt","P OR","Ġt ending","ĠMerch ants","Pro of","ĠV arg","ĠPort able","ãĥ¼ãĥĨ ãĤ£","B rain","25 00","Ġfol iage","Ø ¹","Ġment ors","ĠA ires","Ġminimal ist","Ġing ested","ĠTro jan","ĠQ ian","inv olved","0 27","Ġer oded","RA FT","Ġbl urry","M ob","Ġbuff et","ĠFn atic","ae a","KN OWN","ĠIn it","s afety","en um","ACT ION","ĠCrus her","ĠD ates","Ġ ................","c alling","ak ov","Ġvent ured","Ġ5 55","au ga","H art","ĠA ero","M AC","Ġthin ly","Ġar ra","ST ATE","ild e","ĠJac qu","ĠFem ales","Ġthe orem","Ġ3 46","Ġsmart est","ĠPU BLIC","ĠK ron","ĠB its","ĠV essel","ĠTele phone","Ġdec ap","Ġadj unct","ĠS EN","mer ga","Ġred acted","Ġpre historic","Ġexplan atory","ĠRun s","ĠUtt ar","ĠM anny","ĠAUTH OR","ĠUnle ashed","ĠBow ling","be ans","79 3","Ġunivers es","Ġsens it","ĠK ung","re peat","ctr l","Ġp aced","Ġfull er","Cl ock","Ġrec omb","ĠF aul","ĠB unker","Ġpool ed","Ġan a","ĠM outh","LL OW","hum ane","Ġbull do","ĠMicha els","f am","Ġwreck ed","Ġport rays","ĠWh ale","ĠH es","Ġguess es","ĠBrow se","ĠL APD","Ġconsequ ential","ĠInn ocent","ĠD RAG","Ġtrans gress","ĠO aks","Ġtri via","ĠRes on","ĠA DS","-- +","ĠT oll","Ġgrasp ing","ĠTHE M","ĠT ags","ĠCon clusion","Ġpract icable","Ġho op","Ġunintention ally","Ġign ite","ĠM ov","ur ized","le hem","Ter min","Ġcolour ful","ĠLin ear","ĠEll ie","G y","Ġman power","Ġj s","Ġem oji","ĠSHAR ES","_ .","0000 7","Ġsophistic ation","Ġunders core","Ġpract ise","Ġbl ob","op ens","Uk raine","Ke eping","Y C","J R","ult imate","Cl aim","Ġautom obiles","99 3","ste el","Ġpart ing","ĠL ank","... ?","Ġ38 5","Ġremem brance","Ġe ased","Ġcov ari","ĠS ind","Effect ive","Ġdisse mination","ĠMo ose","ĠCl apper","br ates","App ly","Ġinv is","Ġwors ened","âĢĶ -","Ġlegisl ator","ĠL ol","ĠRow e","Ġdealers hip","um ar","id ences","Ġinvestig ates","Ġc ascade","Ġbid der","ĠB EN","Iron ically","Ġpres iding","Ġd ing","Ġcontrad icted","Ġshut s","ĠF IX","Ġ3 66","Dist rict","Ġsin ful","ĠChar isma","o ops","Ġtot ality","Ġrest itution","ĠOpt imus","ĠD ah","Ġcl ueless","urn ed","Ġnut rit","Ġland owners","Ġfl ushed","Ġbroad en","m ie","Ġprint ln","Ġn ig","ĠCorp us","J en","Ġprot o","ĠWik imedia","ĠPal o","C OR","Ġstory lines","Ġevangel icals","ĠDar rell","Ġrot or","ĠH W","sk illed","ery l","Ġbe gg","ĠBl umenthal","Ġwe aving","Ġdown wards","ĠJack et","ĠANG EL","Te chnology","Ġes oteric","alde hyde","Ġfur iously","Ġforeign er","We ak","CH O","ĠH ound","Exper ience","ĠPlay station","ĠM IA","ĠU ng","cl oth","ag all","Ġcal ming","iz ens","St ruct","ĠW itches","ĠCeleb ration","Ġ........ ......","pt roller","ĠTC U","Ġb unny","ãĥ į","ut orial","Ġup scale","ĠSt a","ĠCol ossus","Ġchlor ide","ĠZ ac","ĠRe asons","ĠBrook ings","ĠWH ITE","][ /","ĠL ose","9 05","Ġunders ide","ern els","Ġv ape","do zen","upp et","ĠST OP","mat ical","ĠStat ements","hed dar","P AC","Custom er","Ġmem os","ĠP J","end ars","ĠLim its","l augh","Ġstabil ized","ĠALE C","Y A","Up grade","al am","Ġtechn o","Ġan ew","fore seen","Ġcolleg iate","ĠPy ro","ĠD ism","Ġfront line","Ġammon ia","I U","Qu ite","John ny","ass in","G OP","ĠSt yles","ĠSovere ign","acter ial","5 49","ĠR IP","ĠL ists","Ġ3 64","ĠRece p","s ocket","ĠByr d","ĠCand le","An cient","Ġappell ant","en forcement","ace a","ans ki","Ġold s","88 6","Ġsl urs","Ġem pires","Ġbuck le","Ġalien ation","ĠAber deen","Ġunic orn","Ġoverr iding","ĠL X","pp a","Ġdesp ised","ĠB ugs","ĠB ST","S outhern","5 33","Ġhall mark","ĠPost er","Ġstem med","Ġprincip als","ĠT ECH","ĠSand wich","It aly","Ġche esy","ĠSet TextColor","ĠProt ective","ĠC ohn","J O","apt op","Re ason","Lead er","ĠUnder stand","ĠFr idays","ĠContin uous","Ġcl ipping","ĠR ye","Ġber th","tim er","ann is","re act","Ġbuff alo","ĠPar as","Ġ6 55","Ġpres ided","ĠSun rise","Ġve ts","Ġcl oves","ĠMcC ull","Stre ngth","G AN","Ġill iter","ĠPric ing","l é","Ġresist or","Ġbr un","ĠSuff olk","Ñ ĭ","ĠL iver","Re leased","Ġwhat s","8 60","ĠMe asures","Ġden ouncing","ĠRy zen","Ġsou ven","Ġcareg ivers","ch ini","ĠScar lett","Ġt rough","Cong ratulations","Ġtax is","ĠTrad ition","j it","Ġtable top","Ġhither to","Ġdis information","off ensive","h ra","ĠDISTR ICT","Ġcompl icate","chen ko","ĠRecon struction","Ġpalp able","Ġa usp","Ġ4 28","Ġshowc ases","ĠPublic ation","know ledge","inn on","4 19","Ġretri eval","and ers","Ġref ute","Ġinqu ired","g ur","Ġneg ativity","Ġcons erve","Ġafter life","Ġpres upp","ĠGill espie","Ġm t","ĠD N","T ap","Ġper pend","ĠS my","does n","Ġsp illing","Ġhyp ers","K ate","® ,","ke pt","ĠP owered","Ġj a","ĠK lux","ard e","ab an","Ġ4 44","Ġflatt ened","ĠImprove ments","urg a","ĠK und","Ġins cribed","Ġfac ult","Ġunpre pared","ĠCons umers","Ġsatisf ies","Ġpul monary","Ġinf iltration","Ġex ternally","Ġcongrat ulations","ag han","Ġair liner","Ġfl ung","Ġfly ers","G D","Ġsnipp ets","Ġrec ursive","Ġmaster ing","L ex","Ġovert ly","v g","Ġluck ily","Ġenc ro","ĠLanc et","ĠAbyss al","function al","Ġs ow","Ġsqu id","Ġnar ration","Ġn aughty","ĠHon our","ĠSpart ans","Ġsh atter","ĠTac oma","ĠCal ories","ĠR aces","Sub mit","Ġpurpose fully","w av","ĠY ok","F est","ĠG err","Met ro","Ġit iner","f amous","Ġ\" {","in line","was her","Iss ue","ĠCL IENT","oz o","Vers ions","7 25","ĠGl ock","Ġshield ed","ĠPC R","ENC Y","ĠWe ld","ĠSim pl","Ġredirect ed","ĠK ham","Ġ( >","Ġlab ou","Ġdi apers","ss l","Ġcell ar","organ isms","ore sc","ĠBer ks","did n","Sh ipping","C hest","Ġund one","Ġmillion aire","Ġc ords","ĠYoung er","appropri ately","Ġsequ els","u ve","ant icipated","Ġle wd","ĠSh irt","ĠDmit ry","V eter","Ġsl aying","ĠY ar","Ġcompl ication","I owa","ĠEric a","ĠBL M","g irlfriend","b odied","6 26","19 63","Ġintermedi ary","Ġcons olation","M ask","ĠSi em","ow an","Beg inning","Ġfix me","Ġculmin ated","Ġcon duc","ĠVolunte er","Ġpos itional","Ġgre ets","ĠDefin itions","Ġthink er","Ġingen uity","Ġfresh men","ĠMom ents","Ġ35 7","ate urs","ĠFed Ex","s g","69 4","Ġdwind ling","ĠBO X","sel age","Ġt mp","Ġst en","ĠS ut","Ġneighbourhood s","Ġclass mate","f ledged","Ġleft ists","Ġclim ates","ATH ER","ĠScy the","ul iffe","Ġs ag","Ġho pped","ĠF t","ĠE ck","ĠC K","ĠDo omsday","k ids","Ġgas ped","Ġmon iker","ĠL od","ĠC FL","t ions","r ums","fol ios","Ġm d","Ġunc anny","Ġtrans ports","ĠLab rador","Ġrail ways","Ġappl iance","ĠCTR L","æ Ģ","Pop ulation","ĠConfeder acy","Ġunb earable","Ġdors al","ĠIn form","op ted","ĠK ILL","Mar x","Ġhypoc ritical","q us","ĠN umerous","ĠGeorg ian","ĠAmbro se","ĠL och","Ġgu bernatorial","ĠX eon","ĠSupp orts","ens er","ee ly","ĠAven ger","19 65","Ar my","Ġju xtap","Ġcho pping","ĠSpl ash","ĠS ustainable","ĠFin ch","Ġ18 61","ict ive","at meal","ĠG ohan","Ġlights aber","ĠG PA","ug u","ĠRE PL","vari able","Ġher pes","Ġdesert s","ac iously","Ġsitu ational","week ly","ob l","Ġtext ile","ĠCorn wall","Ġcontrace ptives","ĠA ke","] -","ä¹ ĭ",": ,","ĠW em","ĠB ihar","Ġ' .","Ġbe re","Ġanal ogue","ĠCook ies","Ġtake off","Whe el","Ġmaj estic","Ġcomm uting","0 23","ĠCor pse","ass ment","min i","Ġgor illa","ĠAl as","ere e","Ġacquaint ances","ĠAd vantage","Ġspirit ually","Ġey ed","pm wiki","ĠE nder","Ġtrans lucent","Ġnight time","ĠIM AGES","5 45","ĠK amp","ĠFre ak","Ġ ig","Port land","4 32","ĠM ata","Ġmar ines","Ġh ors","ater asu","ĠAtt ribution","Ġ-------- -","Ġk ins","ĠBEL OW","++ +","Ġre eling","ol ed","Ġcl utter","ĠRel ative","Ġ4 27","B US","Ġa vert","ĠChe ong","ĠA ble","ĠPry or","Develop er","Ġen cyclopedia","ĠUSA F","ĠG arry","Sp ain","Bl ocks","Ġexp osition","ĠGamer Gate","W OR","Ġstockp ile","Ġclot hed","ĠT one","ĠR ue","t umblr","Ġtreacher ous","Ġf rying","Ñ Į","ĠS ph","Ġrest raints","Ġemb odies","ĠG es","S afety","Ġnegoti ators","min ing","ĠAppalach ian","L OS","ĠJenn a","Ġpass ers","ç ĭ","sn ap","Ġshort en","creat or","Ġinn umerable","uther land","67 4","ĠW OM","ĠAs cend","ĠArm ory","ĠTrans action","K ick","Ġsuit case","day Name","Ġwaste ful","mar riage","ĠMcC abe","ite ch","ĠO ss","Cl osure","ĠTreasure r","Ġindec ent","ĠD ull","Ġresid ences","19 59","ĠS ettlement","Ham ilton","Ġself ies","ĠRank ing","ĠBark ley","ĠB ore","ĠW CS","ĠMar itime","ĠH uh","ĠForest ry","Ġcultiv ating","ĠBall ard","Ġg arrison","ĠSD L","9 30","Ġnas cent","Ġirresist ible","Ġaw fully","\\/ \\/","Ġequ ate","Ġanthrop ology","ĠSylv ia","Ġintest ine","Ġinnoc uous","cess ive","ag ra","ĠMet roid","G rant","8 55","ģ ĸ","Ġ\" _","ãĥĥ ãĥī","Ġappra isal","ĠFred dy","04 6","Ġ40 6","Ġ18 30","Ġd ocking","St atic","Ġp ont","ĠVolt age","ĠSt ead","ĠMort gage","ĠJon ah","Y L","CLASS IFIED","Ġas bestos","nik ov","Ġcoll agen","ĠOrb ital","P ocket","7 99","Ġhy brids","inc hes","Ġinv oice","und y","Ġinequ alities","T rend","w ashed","B ALL","Ġluc id","ĠComment ary","Ġw itty","Br andon","Ġbru ising","Ġ6 20","es cent","box ing","P OL","Ġ3 78","R ect","Ġlic ences","ĠMcG ee","p ressed","D anny","Ġj ammed","ord inate","Ġle th","Ġdistingu ishes","ĠYam aha","IL S","ĠH ume","ĠC ategories","Rober ts","Ch art","Ġbeet le","ĠGra veyard","Ġ($ )","o ÄŁ","Ġtw ilight","are lla","á ½","Ġbooth s","ĠH HS","ĠFeld man","Ġexcav ation","Ġphilosoph ies","at ography","ĠGar age","te chnology","Ġunfor gettable","Ġver ifying","Ġsubord inates","E ls","Ġne b","G aming","EN A","ĠAchieve ment","it ters","ĠG abe","Ġd umps","for cer","Ġpo ignant","ĠM BA","ĠHe idi","ime i","Ġm ages","Ġliber ate","Ġcircum cised","ĠMer maid","ĠMat th","t ogether","ĠW ichita","Ġstore front","ĠAd in","V II","Four th","Ġexplore rs","W ER","Not able","Bro ok","m ens","F aith","-------- -","ĠJ ou","¬ ¼","Ġpine apple","Ġam alg","el n","ark able","ĠãĤµ ãĥ¼ãĥĨãĤ£","ĠãĤµãĥ¼ãĥĨãĤ£ ãĥ¯ãĥ³","Ġov arian","ĠE choes","Ġhairc ut","Ġp av","Ġch illed","anas ia","Ġsty led","Ġd ab","ni per","Ġminister ial","ĠD UP","T an","Ġsul ph","ĠD eter","ĠBo hem","od an","Ġeduc ator","â ĵĺ","sp ir","Ch icken","ĠE leanor","Ġqu i","Ġheav iest","Ġgrasp ed","U RA","Ġcro oked","Jess ica","pro blem","Ġpred etermined","Ġman iac","Ġbreath s","ĠLauder dale","Ġh obbies","y z","Cr ime","Ġcharism a","d L","Ġle aping","Ġk ittens","Ang elo","ĠJ ACK","ĠSu zanne","Ġhal ting","ENT ION","Ġswall owing","ĠEarthqu ake","Ġeight eenth","ĠN IC","ĠIN F","ĠCons cious","Ġparticular s","circ le","7 40","Ġbene volent","Ġ7 47","Ġ4 90","Ġr undown","ĠVal erie","ĠB UR","Ġcivil isation","ĠS chn","W B","ot ide","intern ational","Ġj ohn","Ġ19 02","Ġpe anuts","Ġflav ored","k us","Ġro ared","Ġcut off","é £","Ġorn ament","Ġarchitect ures","Ġ3 69","ol or","ĠWild e","ĠC RC","ĠAdjust ed","Ġprov oking","land ish","Ġrational ity","Ġjust ifies","Ġdisp el","Ġa meric","ĠPol es","Ø ©","Ġen vis","ĠD oodle","ä½ ¿","igs aw","auld ron","Techn ical","T een","up hem","ĠX iang","Ġdetract ors","ĠZ i","ĠJournal ists","Ġconduc ive","ĠVolunte ers","Ġs d","Know ing","Ġtrans missions","ĠPL AN","ĠL IB","Ġall uded","Ġob e","Ġd ope","ĠGold stein","Ġwavelength s","ĠDest ination","nd a","ug i","Ġattent ive","ĠLe an","ral tar","Ġman g","mb uds","ak ings","b ender","Ġacc ol","Ġcraw led","N OW","Min nesota","Ġflour ished","ĠZ up","ĠSuper visor","ĠOliv ier","Ex cellent","Ġwid en","D one","Ġw ig","Ġmiscon ceptions","Cor p","W an","Ġvener able","ĠNot ably","ĠKling on","an imate","Bo ost","ĠS AY","miss ing","ibli ography","mel on","Ġpay day","Ø ³","bo le","Ġve iled","ĠAl phabet","It alian","Ġever lasting","ĠR IS","ĠC ree","rom pt","Ġh ating","Ġgrin ning","Ġge ographically","OS H","Ġwe eping","ĠÂłĠÂłĠÂłĠÂł ĠÂłĠÂłĠÂłĠÂł","Ġimpe cc","Let ter","Ġblo ated","PL A","ĠFe in","Ġper sever","Th under","Ġa ur","ĠR L","Ġpit falls","âĸ º","Ġpredomin ant","Ġ5 25","7 18","AP E","7 14","Ġfarm land","ĠQ iao","Ġv iolet","ĠBah amas","Ġinflic ting","ĠE fficiency","Ġhome brew","Ġundert ook","Ġcur ly","ĠHard ing","man ia","59 6","Ġtem pered","Ġhar rowing","ĠP ledge","ĠFranken stein","è ª","M otion","Ġpredict ably","ĠExpl osion","oc using","er d","col o","FF ER","Ġback field","ĠV IDE","ue bl","N arr","ĠArg ument","Ġgen omic","Ġbout ique","Ġbatt ed","ĠB inary","Ġg amb","ĠRh ythm","67 3","Ġa float","ĠOlymp ia","Y ING","Ġend if","is in","Ġwin ters","Ġsc attering","I v","D istance","Ġtr u","ĠCom fort","Ġne xus","Ġair flow","ĠByz antine","p ayers","con i","ĠB etsy","D eal","ĠN ug","ĠContin ent","red ibly","Ġoptim izing","al beit","Ġec static","ĠPro to","ç ·","iv ot","âĸ Ħ","em p","rou nder","Ġcl out","ĠI ST","66 3","ĠDoll ars","ĠD AC","Ġsubsc ribed","Ġrehears al","Ġam ps","ĠSh ang","es m","Ġspr inkle","Ġassail ant","ĠO o","ĠCoin base","T act","Ġret ina","Ġn uns","R ON","att o","Ġj ug","ĠSV G","Ġb ikini","ĠFI LE","ĠFound ers","ep ort","ĠK P","Ġrest ores","ĠTh ick","Ġash ore","Ġappro vals","R ender","M AG","G raham","ĠCort ana","ãĥ³ ãĤ¸","ss h","or ians","ars ity","ĠInsp ired","u pper","Ġsign alling","Ġreb uke","Ġfl ares","Ġdownt ime","Stud ies","Ġstagn ation","ĠSequ ence","Ġgr unt","Ġass ures","ĠPL A","59 2","Ġintra ven","d epend","Sus an","ĠManz iel","Man ia","Cont ract","Ġsl ams","Ġcult ured","Ġcred itor","L IST","ĠH UM","ĠChatt anooga","serv ed","Ġclo aked","ĠF TP","p owder","ĠSt ella","uct ive","Ġcheap ly","ĠMU CH","ĠGalile o","Ġsu ites","spe ech","Ġdeliber ations","ĠCh ips","« ĺ","Bal ance","ĠWyn ne","ĠAk ron","Ass et","Ġhon oured","Ġed ged","Like wise","anim ous","ĠW age","ĠEz ek","ad vertisement","ĠRT X","ĠM AD","Ġmigr ating","ĠS QU","Ġ4 75","Ed ited","Ġshorth and","ĠBas ics","Ġcro tch","ĠEV EN","Ġv m","effic iency","Ġcal ves","ĠF rie","ĠBrill iant","Ġstri kers","Ġrepent ance","Ġarter ies","r l","B ed","h ap","Ġcrypt ography","ĠSab res","Ġ4 14","vi ks","ih ara","aps es","T alking","Ġintertw ined","Ġdoc ks","Ġalle le","ĠArt ifact","ĠH IM","t orn","ç ķ","Ġop acity","ĠE ly","os uke","Ġn ipple","Ġhand written","ĠV K","ĠChamber lain","ĠLa os","ig raph","g row","Ġtr illions","Ġdescend ant","ĠSail or","as uring","Ġce ilings","ĠWare house","f lying","ĠGl ow","Ġn ont","Ġmiscar riage","Ġrig s","Ġmin istries","Ġelabor ated","Ġdel usional","ĠHum ane","Ġ3 79","n ets","Ġblack out","add ers","Ġn p","ĠT ire","ro sc","Ġsub div","Ġlink age","Ġchron ological","ĠHER O","Ġres ettlement","ĠVin yl","Ġpast oral","ĠMob il","ĠBar bar","Co oldown","ĠF ritz","c riminal","re pe","Ġbell ig","ĠBre ed","Ġ4 18","Ġsem blance","ij k","Ġcur tail","Ġclin ch","cont ained","ĠProm pt","ast on","Ġw i","Ġpursu its","5 15","ĠGl oss","Ġfl ips","Ġcoup ons","Ġcl oning","ĠLike ly","Rem oved","ĠQu artz","r ices","ĠSpe ars","Ġp ious","Ġdep reciation","ĠD are","oun ces","am az","O nt","Ġp innacle","d ocker","0 26","ĠW yr","ĠPro per","Ë Ī","n il","By tes","Ġseek er","t rial","Ġunf olds","ĠMar se","Ġextravag ant","ĠSurviv ors","RED ACTED","ĠSpeed way","ĠCra igslist","sub mit","ĠGener ations","Ġup holding","Ġblood stream","ĠMiss ions","ĠL awn","Ġlim bo","ene i","H uh","ĠWild cats","pre p","ĠMark us","ĠFor bidden","rit ic","IN O","Ġexhib iting","requ ent","ch uk","Ġhabit ual","ĠComp atibility","Dr ag","RIP T","uj ah","GR OUND","Ġdelinqu ent","Ġburn er","Ġcontempor aries","Ġgimm ick","load s","Ġno zzle","p odcast","ĠW ak","ĠStat en","ĠK uh","ãģ ĵ","inter rupted","Ġinv incible","ĠBurn ett","cig arette","ĠPeb ble","ĠTem porary","ĠMar ino","58 2","Ġwast eland","ident ly","T x","Ġr ite","ĠPan asonic","ĠM iddles","ĠHort on","ae us","Ġc uring","Ġm ats","Ġadj ourn","Ġfears ome","pe z","bo ats","Ġpro pell","Ġconflic ted","ĠAng er","Ġinsurg ent","K arl","Ġco ales","Ġsouth western","Ġdis su","ĠO vert","******** ****","Ġbox ed","ĠBr une","aa a","Ġgard ening","ĠEng el","tr acks","Ġpur ified","Ġplace holder","ĠL ikes","Ġd an","G ab","Ġe ct","ĠF aw","ĠEl iot","Ġ' ,","otrop ic","ĠRu in","hed on","Ġca ul","Ġa ft","ĠCad illac","gh a","ass ian","ud eb","ĠT ick","Ġadjust s","AR GET","5 37","isc he","ant y","ĠFried rich","ĠBl izz","ĠA OL","Camp aign","Ġmamm al","ĠVe il","ĠK ev","ĠMaur it","ĠDam ien","N ation","E astern","Ġ{ :","Ġ= ================================","Ġstereotyp ical","Ġatt ic","ĠCy borg","requ ire","Ġaward ing","ĠPap ua","bt n","b ent","B oo","Ġ( =","ĠX ander","ĠSomers et","Ġcatch y","Ġcert ify","STR UCT","Ġit al","Ġt ides","ĠBr ands","G ray","comp etitive","Ġcur ator","ĠD G","omin ium","ĠGM Os","ci ating","ĠCarm en","ow ard","Balt imore","Ġr gb","C u","Ġwip es","spe ll","IT NESS","Ġsummar izes","ĠRe vis","Ġwhistlebl owers","ĠBre ach","Ġcro chet","k os","ews ki","Ġrep et","Ġcrim son","ĠKar achi","read able","dim ension","ĠI gor","ild ed","ĠZ ed","ĠKe ane","ĠCos metic","DE P","Ġretreat ing","ĠU A","ens ical","Ġd usk","ĠDick ens","Ġaren as","ĠPass age","level s","Ġcur v","P ope","Ġch ores","ĠEl ise","ĠComp ass","b ub","Ġmamm alian","ĠSans krit","ĠAN C","ĠCr ack","Q ual","L aun","amp unk","Ġlearn ers","Ġglam orous","Ġfur the","erm ott","c and","Gener ic","Ġnarr ated","Ġdisorder ly","ĠTrans actions","ĠDet ention","ĠR oku","Ä į","Ġunder statement","ĠS aur","ĠRodrig o","ĠAS AP","S in","Ġre joice","Method s","Ġelectro de","Ġworsh ipped","Ġid i","ĠPhys icians","Ġpop up","Ġde ft","ĠRem oval","ĠBu enos","ver bs","Ġfun k","ush a","rict ion","ore a","ĠBang alore","ĠKen obi","zz i","Ġnorm ative","Ġgobl ins","Ġcaf es","ĠUN CLASSIFIED","ĠF ired","S IGN","Ġs clerosis","ĠV oter","ĠSon ny","ĠExt end","ĠEV s","Ar senal","Ġp si","Ġwid est","ĠT us","Ġlo oms","Ġjust ifying","ĠGr anger","è ¯","Ref er","58 3","Ġflour ishing","ab re","Ġr ave","ĠCont ra","Ġ18 98","Add s","Ġf ul","ĠCo oke","some one","= #","67 1","Ġy ak","Ġar te","ĠMis cellaneous","ĠDet ection","ĠCl ancy","â ģ","ass ies","Ġval iant","ĠFemin ist","cor ruption","V el","P ear","Ġsucc inct","Ġquick est","k w","Ġsp itting","ĠL ibraries","åħ ī","ant z","D ad","ĠSpec ifications","rup ulous","and r","RES ULTS","Ġsnow ball","Ġpred is","ĠB axter","ĠNurs ing","ĠCh aff","s we","Ġout age","Ġnest ing","Ġnotor iety","tr igger","on ite","j on","Ġf ou","ook ed","ĠCelebr ity","re ality","Ġfat ig","Ġhug ging","Ġbother s","ĠPan zer","ĠCh andra","fig ured","Ġvol ts","ĠCloud s","Ġfee ble","ĠCur ve","ĠAs us","78 6","abs or","ĠV ICE","ĠH ess","Ġmanufact ures","Ġgri zz","ĠPower ful","ac id","Ġsub sections","ĠKrug man","ĠAl ps","is u","Ġsequ est","ĠUlt ron","ĠT inker","ĠGo ose","Ġmism atch","Att orney","Ġmorph ology","ĠSix ers","ut tered","ĠE LECT","gr an","Rus sell","ĠG SL","Ġfort night","Ġ. )","Ġapost le","pr one","el ist","Unt itled","ĠIm plementation","ist ors","Ġtank er","Ġpl ush","Ġattend ants","ĠT ik","ĠGreen wich","ĠY on","ĠSP L","cell s","unt led","S olution","ĠQu é","Ġvac ated","Ġupt ick","ĠMer idian","æ ĥ","ĠDr ill","9 25","58 4","Ġrenov ated","ĠKub rick","zy k","Ġl ousy","pp el","ohyd rate","ĠI zzy","lesi astical","CC C","ĠAj ax","Ġad apters","ĠPetra eus","Ġaffirm ation","ĠST OR","le ms","ad oes","ĠConstantin ople","Ġp onies","Ġl ighthouse","Ġadherent s","ĠBre es","omorph ic","Fight ing","Ġpl aster","ĠP VC","ĠOb st","Ġdear ly","ĠTo oth","icks on","Ġsh aming","P lex","A gg","Ġâ̦ \"","Ġsub reddits","Ġpige on","ĠResident ial","ĠPass ing","Ġl um","ĠP ension","Ġpessim istic","Ġ4 32","z inski","c ade","0 75","Ġapolog ised","iy ah","Put ting","Ġgloom y","ĠLy me","=-=-=-=- =-=-=-=-","ĠT ome","ĠPsych iatric","ĠH IT","c ms","ap olog","Ġbreak er","Ġdeep en","Ġtheor ist","ĠHigh lands","Ġb aker","Ġst aples","Ġinterf ered","ĠAb ortion","jo ined","ch u","Ġform ulate","Ġvacc inations","Ġban ter","phe us","Ġoutfield er","ĠM eter","Ġ# ####","Ġ18 95","Ġnarrow ing","ĠST ORY","f p","ĠC ST","ign ore","Ġproclaim ing","ĠR U","ĠB ALL","yn a","65 3","Ġpos it","P RE","59 4","ĠRegist rar","ĠPil grim","ic io","Ġpre tt","Ġlif eless","Ġ__ _","Ne igh","ĠCh urches","orn o","Ġor cs","Ġkind red","ĠAud it","Ġmillenn ial","ĠPers ia","g ravity","ĠDis ability","ĠD ARK","W s","od on","Ġgrand daughter","ĠBro oke","ĠA DA","ER A","Ġpick ups","ĠWil kinson","ĠSh ards","ĠN K","Ġexp el","ĠKis lyak","Ġj argon","Ġpolar ized","ian e","Pub lisher","Ġreb utt","Ġapprehens ion","ĠK essler","Ġpr ism","F UL","19 64","ĠL oll","ä ¿","le thal","Å Ł","Ġg hetto","Ġb oulder","ĠSlow ly","ĠOsc ars","ĠInst ruction","ĠUl tr","ĠM oe","N ich","ĠP ATH","( *","ĠRE LEASE","un ing","rou se","en eg","Ġre imb","ĠDet ected","Do S","Ġster ling","Ġaggreg ation","ĠLone ly","ĠAtt end","hig her","Ġairst rike","ks on","SE LECT","Ġdef lation","ĠHer rera","C ole","rit ch","Ġadvis able","F ax","Ġwork around","Ġp id","mort em","ers en","Ġtyp o","Ġal um","78 2","ĠJam al","script s","Ġcapt ives","ĠPres ence","ĠLie berman","angel o","Ġalcohol ism","ass i","Ġrec ite","Ġgap ing","Ġbask ets","ĠG ou","Brow ser","ne au","Ġcorrect ive","und a","sc oring","ĠX D","Ġfil ament","Ġdeep ening","ĠStain less","Int eger","Ġbu ggy","Ġten ancy","ĠMub arak","Ġt uple","ĠD roid","ĠS itting","Ġforfe it","ĠRasm ussen","ixt ies","es i","ĠKim mel","Ġmetic ulously","Ġap opt","ĠS eller","08 8","ec ake","hem atically","T N","Ġmind less","Ġdig s","ĠAcc ord","ons ense","em ing","br ace","Ġe Book","ĠDist ribut","ĠInvest ments","w t","] ),","beh avior","56 3","Ġbl inding","ĠPro testers","top ia","Ġreb orn","ĠKel vin","ĠDo ver","ĠD airy","ĠOut s","Ġ[ /","Ï Ģ","b p","ĠVan ity","ĠRec ap","ĠHOU SE","ĠF ACE","Ġ4 22","69 2","ĠAnt ioch","cook ed","Ġcoll ide","Ġa pr","Ġsle eper","ĠJar vis","Ġalternative ly","ĠLe aves","ĠM aw","Ġantiqu ity","ĠAdin ida","Ġab user","Poké mon","Ġass orted","ĠRev ision","ĠP iano","ĠG ideon","O cean","Ġsal on","Ġbust ling","ogn itive","ĠRah man","Ġwa iter","Ġpres ets","ĠO sh","ĠG HC","oper ator","Ġrept iles","Ġ4 13","ĠG arr","ĠCh ak","Ġhas hes","Ġfail ings","Ġfolk lore","Ġab l","ĠC ena","ĠMac Arthur","ĠCOUR T","Ġperipher y","app ers","Ġreck oned","ĠInf lu","ĠC ET","Ġ3 72","ĠDefin itive","ass ault","4 21","Ġreservoir s","Ġd ives","ĠCo il","DA Q","Ġvivid ly","ĠR J","ĠBel lev","Ġec lectic","ĠShow down","ĠK M","ip ed","reet ings","ĠAs uka","L iberal","ĠÏ Ħ","Ġbystand ers","ĠGood win","uk ong","S it","ĠT rem","Ġcrim inally","ĠCirc us","ch rome","88 7","Ġnan op","ĠOb i","ĠL OW","o gh","ĠAuth ors","ob yl","Ur ban","Ġt i","ĠWe ir","t rap","ag y","Ġparent heses","Ġout numbered","Ġcounter productive","ĠTob ias","ub is","P arser","ST AR","Ġsyn aptic","ĠG ears","Ġh iber","Ġdebunk ed","Ġex alted","aw atts","H OU","Ch urch","ĠPix ie","ĠU ri","ĠForm ation","ĠPred iction","C EO","Ġthro tt","ĠBrit ann","ĠMad agascar","ë ĭ","Ġbill boards","ĠRPG s","ĠBe es","complete ly","F IL","Ġdoes nt","ĠGreen berg","re ys","Ġsl ing","Ġempt ied","ĠPix ar","ĠDh arma","l uck","ingu ished","Ġend ot","Ġbab ys","05 9","che st","r ats","Ġr idden","Ġbeet les","Ġillum inating","Ġfict itious","ĠProv incial","Ġ7 68","Ġshe pherd","ĠR ender","Ġ18 96","C rew","Ġmold ed","ĠXia omi","ĠSp iral","Ġdel im","Ġorgan ising","Ġho ops","ĠBe i","z hen","Ġfuck in","Ġdec ad","Ġun biased","am my","sw ing","Ġsmugg led","Ġk ios","ĠP ERSON","ĠInquis itor","Ġsnow y","Ġscrap ing","ĠBurg ess","P tr","ag ame","R W","Ġdro id","ĠL ys","ĠCass andra","Jac ob","Ġ35 4","Ġpast ure","Ġfr anc","ĠScot ch","ĠEnd s","ĠI GF","def inition","Ġhyster ical","ĠBrown e","77 1","Ġmobil ization","æ ķ","iqu eness","Th or","Ġspear headed","Ġembro iled","Ġconject ure","jud icial","Ch oice","Ġpaper back","P ir","Ġrec overs","ĠSur ge","ĠSh ogun","ĠPed iatrics","ãģ ł","Ġsweep s","ĠLabor atories","ĠP acks","al us","add in","Ġhead lights","g ra","Ev idence","COL OR","Ad min","Ĭ ±","Ġconco ct","s ufficient","Ġun marked","Ġrich ness","Ġdiss ertation","Ġseason ing","Ġg ib","ĠM ages","un ctions","ĠN id","che at","ĠTM Z","c itizens","ĠCatholic ism","n b","Ġdisemb ark","ĠPROG RAM","a ques","Ty ler","Or g","ĠSl ay","ĠN ero","ĠTown send","IN TON","te le","Ġmes mer","9 01","Ġfire ball","ev idence","aff iliated","ĠFrench man","ĠAugust a","0 21","Ġs led","Ġre used","ĠImmun ity","Ġwrest le","assemb led","Mar ia","Ġgun shots","ĠBarb ie","Ġcannabin oids","ĠTo ast","ĠK inder","IR D","Ġre juven","Ġg ore","Ġrupt ure","Ġbre aching","ĠCart oon","Ġ4 55","ĠPale o","6 14","Ġspe ars","ĠAm es","ab us","Mad ison","GR OUP","Ġab orted","y ah","Ġfel on","Ġcaus ation","Ġprep aid","Ġp itted","op lan","ĠShel ley","ĠRus so","ĠP agan","Ġwill fully","ĠCan aver","und rum","ĠSal ary","ĠAr paio","read er","ĠR ational","ĠOver se","ĠCa uses","Ġ* .","Ġw ob","Ke ith","ĠCons ent","man ac","77 3","6 23","Ġfate ful","et imes","Ġspir ited","ĠD ys","Ġhe gemony","Ġboy cot","ĠEn rique","em outh","Ġtim elines","ĠSah ara","ĠRel ax","ĠQuin cy","ĠLess ons","ĠE QU","SE A","N K","ĠCost co","Incre ase","Ġmotiv ating","ĠCh ong","am aru","ĠDiv ide","Ġped igree","ĠTasman ia","ĠPrel ude","L as","9 40","57 4","Ġch au","ĠSp iegel","un ic","-- >","ĠPhil ips","ĠKaf ka","Ġuphe aval","Ġsent imental","Ġsa x","ĠAk ira","ser ial","Mat rix","Ġelect ing","Ġcomment er","ĠNeb ula","ple ts","ĠNad u","ĠAd ren","Ġen shr","ĠR AND","fin ancial","ĠCly de","uther ford","Ġsign age","Ġde line","Ġphosph ate","rovers ial","f ascist","ĠV all","ĠBeth lehem","Ġfor s","Ġeng lish","S olid","N ature","Ġv a","ĠGu ests","Ġtant al","Ġauto immune",";;;;;;;; ;;;;","ĠTot ally","ĠO v","Ġdef ences","ĠCoc onut","Ġtranqu il","Ġpl oy","Ġflav ours","ĠFl ask","ãĤ¨ ãĥ«","ĠWest on","ĠVol vo","8 70","Ġmicro phones","ver bal","R PG","Ġi ii","; }","0 28","Ġhead lined","Ġprim ed","Ġho ard","ĠSh ad","ĠEN TER","Ġtri angular","Ġcap it","l ik","ĠAn cients","Ġl ash","Ġconv ol","Ġcolon el","en emy","G ra","Ġpub s","ut ters","Ġassign s","ĠPen et","ĠMon strous","ĠBow en","il ver","H aunted","ĠD ing","start ed","pl in","Ġcontamin ants","ĠDO E","ff en","ĠTechn ician","R y","Ġrob bers","Ġhot line","ĠGuard iola","ĠKau fman","row er","ĠDres den","ĠAl pine","E lf","Ġf mt","ĠS ard","urs es","g pu","Un ix","Ġunequiv ocally","ĠCitizens hip","qu ad","m ire","ĠS weeney","B attery","6 15","Ġpanc akes","Ġo ats","M aps","ĠCont rast","mbuds man","ĠE PS","Ġsub committee","Ġsour cing","Ġs izing","ĠBuff er","ĠMand atory","Ġmoder ates","ĠPattern s","ĠCh ocobo","ĠZ an","ĠSTAT ES","ĠJud ging","ĠIn her","* :","Ġb il","ĠY en","Ġexh ilar","oll ower","z ers","Ġsn ug","max imum","Ġdesp icable","ĠP ACK","ĠAn nex","Ġsarcast ic","Ġlate x","Ġt amp","ĠS ao","b ah","ĠRe verend","ĠChin atown","ĠA UT","d ocumented","ĠGA BA","ĠCan aan","ĠÙ ħ","Ġgovern s","pre v","E sc","ĠEst imates","OS P","Ġendeav our","ĠCl osing","omet ime","every one","Ġwor sen","Ġsc anners","Ġdev iations","ĠRobot ics","ĠCom pton","Ġsorce rer","Ġend ogenous","Ġem ulation","ĠPier cing","ĠA ph","ĠS ocket","Ġb ould","ĠO U","ĠBorder lands","Ġ18 63","G ordon","ĠW TO","Ġrestrict s","Ġmosa ic","Ġmel odies","ç Ħ","T ar","Ġdis son","ĠProv ides","Ġ ......","b ek","F IX","Ġbro om","ans hip","Do ctors","Ġner ds","ĠReg ions","na issance","Ġmet e","Ġcre pt","pl ings","Ġgirlfriend s","kn it","ig ent","ow e","Ġus hered","ĠB az","M obil","4 34","ĠPres ents","orig in","Ġins omnia","ĠA ux","4 39","ĠCh ili","irs ch","G AME","Ġgest ation","alg ia","rom ising","$ ,","c row","ĠIn spection","at omic","Rel ations","J OHN","rom an","ĠClock work","ĠBak r","m one","M ET","Ġthirst y","Ġb c","Ġfacult ies","R um","Ġnu ance","ĠD arius","ple ting","fter s","etch up","Reg istration","ĠK E","R ah","Ġpref erential","ĠL ash","ĠH H","Val id","ĠN AV","Ġstar ve","ĠG ong","z ynski","ĠAct ress","Ġw ik","Ġun accompanied","lv l","Br ide","AD S","ĠCommand o","ĠVaugh n","Wal let","Ġho pping","ĠV ie","Ġcave ats","Ġal as","if led","ab use","66 1","Ġib n","Ġg ul","Ġrob bing","t il","IL A","Ġmit igating","Ġapt ly","Ġty rant","Ġmid day","ĠGil more","ĠDe cker","Ġ§ §","part ial","Ex actly","Ġphen otype","Ġ[+ ]","ĠP lex","ĠI ps","vers ions","Ġe book","Ġch ic","g ross","\":\" \"},{\"","ĠSur prisingly","M organ","Ġresid ues","ĠConf ederation","in feld","Ġl yr","mod erate","Ġperpend icular","V K","Ġsynchron ized","Ġrefres hed","Ġad ore","ĠTor ment","ol ina","Ġ26 00","Item Tracker","Ġp ies","ĠF AT","ĠR HP","0 48","ĠRES P","ĠB J","all ows","P and","Ġunw elcome","ĠV oc","ĠBast ard","ĠO W","ĠL AR","ĠHeal er","Environment al","ĠKen yan","ĠTr ance","ĠP ats","Ġali ases","ĠGar field","Ġcampaign er","Ġadvance ments","ĠOkin awa","ĠC oh","ows ky","Ġstar ved","Ġsize able","Ġ: -)","Ġm RNA","Ġsusp ensions","ist ar","Scot land","Pr in","-------------------------------- ----------------","Ġ50 2","Ġteasp oons","Ġ10 50","Ġcoerc ive","ĠMason ic","edd ed","ĠPass enger","Ġl att","Ġbr aces","ĠSt eal","ĠNY T","ĠK ats","ĠCel est","ae z","T u","ĠCoul ter","ðŁ ĺ","Fl ickr","ĠWil mington","ith s","++ ;","Ġv ending","Ġneg ro","ĠPh i","ĠYellow stone","Call back","Ġsh ampoo","ĠSh ades","w at","Ġsuper human","Ġridic uled","Ġhol iest","om bo","Ġintern s","Ġh one","ĠPar agu","UR I","Ġd angling","ãĤ »","so v","ict ional","av ailability","Ġrev ocation","Ġd ow","in ic","ĠTHE IR","Ġis o","Ġout ings","ĠLeth al","Ġ) ))","Ġinacc ur","Ġout landish","Ġan us","let ico","id on","l ol","Ġun regulated","Ġsuccumb ed","Ġc uff","ĠWast eland","let al","Ġsub str","Ġcoff ers","Ġautom akers","ov i","ĠX ue","ĠDayton a","Ġjar ring","Ġf umes","Ġdisband ed","z ik","itt on","Ġstriking ly","Ġsp ores","Ad apter",".) :","ĠLynd on","ival ry","Ġor ally","Ġtumult uous","Ġdisple asure","Ġcon es","or rect","Ġappe ase","Ġder by","ĠTrip oli","ĠAl ess","Ġp oked","ĠGu ilty","v P","En ough","Ġorig inals","6 99","Ġrabb i","Ġproverb ial","Ġpostp one","el ope","ĠMist y","Ġstaff ed","ĠUn employment","redit ary","Ġdilig ent","re comm","me asures","as in","8 25","Ġpond s","Ġmm ol","ĠS AR","ĠC ARE","Ġ3 71","Ġclen ched","ĠCors air","Ġcaric ature","z n","att ach","ĠSch ro","spe ak","p ainted","ĠS uc","ĠE NT","Ġcell ul","ĠP aid","di agn","WH ERE","Ġtext ed","B arn","Ġret racted","ĠRe ferred","S av","Ġup keep","Ġwork places","ĠTok ens","Ġampl ify","cl inical","Ġmult ic","mber g","Ġconvol uted","Reg ion","5 65","ĠTop ic","Ġsn ail","Ġsal ine","Ġins urrection","ĠPet r","f orts","B AT","ĠNav ajo","Ġrud imentary","ĠLak sh","OND ON","Me asure","Ġtransform er","ĠGodd ard","Ġcoinc ides","ir in","R ex","ĠB ok","qu it","Ġshotgun s","Ġprolet arian","Ġsc orp","ĠAd a","5 14","Ġsl ander","record ed","Ġemb ell","ris ome","Ġapolog izing","ĠMul cair","ĠGib raltar","Cl a","Ġall ot","ĠAtt ention","Ġ4 33","le ave","Ġwh ine","ĠIss a","ĠFa ust","ĠBar ron","hen y","Ġvictim ized","J ews","Ġnurt uring","ett el","W inged","ĠSub tle","Ġflavor ful","ĠRep s","eng ed","call back","Ġdirection al","Ġcl asp","ĠDirect ions","plan et","icult ure","Hel per","ic ion","ac ia","Ġç ¥ŀ","Ġsur ges","Ġcan oe","ĠPrem iership","be en","Ġdef ied","ĠTro oper","Ġtrip od","Ġgas p","ĠE uph","ĠAd s","vern ight","high ly","R ole","Ġent angled","ĠZe it","6 18","ĠRust y","Ġhaven s","ĠVaugh an","HA EL","ĠSER VICE","/ ,","Ġstr icken","Ġdel usions","Ġb is","ĠH af","Ġgrat ification","Ġent icing","UN CH","Ad ams","ĠOL ED","ĠBeet le","Ġ18 99","ĠSO FTWARE","ateg or","V L","ĠTot em","ĠG ators","AT URES","Ġimped ance","Reg istered","ĠC ary","ĠAer ial","on ne","en ium","Ġd red","ĠBe g","Ġconcurrent ly","Ġsuper power","ĠX an","j ew","imes ter","ĠDick inson","âĶ ģ","F la","Ġp ree","ĠRoll ins","© ¶æ","Ġden omination","ĠL ana","5 16","Ġinc iting","sc ribed","j uries","ĠWond ers","app roximately","Ġsusp ending","Ġmountain ous","ĠL augh","oid al","N s","Det ect",") =","ĠL uthor","ĠSchwarz enegger","ĠMull er","ĠDev i","ec ycle","J ar","6 13","ĠL ongh","B ah","ĠSP ORTS","n w","Ġref inement","Ġwater ways","Ġd iner","Bl ade","68 3","F ac","Ġinitial s","Ġro g","Ġparan ormal","B UT","Ġ[ (","ĠSw anson","ĠM esh","âĸ ¬","Impro ve","ĠRad iation","ĠEst her","ĠE sk","ĠA ly","ik y","Ġir rad","ĠBuck ingham","Ġref ill","Ġ. _","Re pe","CON CLUS","Ġdifferent iated","Ġchi rop","ĠAt kins","Pat tern","Ġexc ise","Ġcab al","N SA","ĠST A","ĠS IL","ĠPar aly","Ġr ye","ĠHow ell","ĠCount down","ness es","alys ed","Ġres ize","ãĤ ½","Ġbudget ary","ĠStr as","w ang","Ġap iece","Ġprecinct s","Ġpe ach","Ġsky line","Ġ35 3","pop ular","App earances","ĠMechan ics","ĠDev Online","S ullivan","Z en","Ġp u","op olis","5 44","Ġde form","Ġcounter act","ĠL ange","Ġ4 17","Con sole","77 4","Ġnodd ing","Ġpopul ism","Ġhe p","Ġcoun selling","compl iance","U FF","Ġunden iably","Ġrail ing","ĠHor owitz","ĠSim one","ĠBung ie","Ġa k","ĠTal ks","x ff","fl ake","Cr ash","Ġsweat y","Ġban quet","ĠOFF IC","Ġinvent ive","Ġastron omer","ĠStam ford","ĠSc are","ĠGRE EN","olic ited","Ġr usher","Ġcent rist","ight ing","Ġsub class","Ġdis av","Ġdef und","ĠN anto","oci ate","m ast","Ġpac if","Ġm end","e ers","imm igration","ESS ION","Ġnumber ing","Ġlaugh able","ĠEnd ed","v iation","em ark","P itt","Ġmetic ulous","ĠL F","Ġcongrat ulated","ĠBir ch","Ġsway ed","Ġsemif inals","Ġhum ankind","m atter","ĠEqu ip","opa usal","S aid","ĠLay out","Ġvo icing","Ġth ug","Ġporn ographic","I PS","Ġmo aning","Ġgriev ance","Ġconf essions","esc al","TEXT URE","Aut hent","os aurus","P urchase","Ġreleg ation","al ter","ĠÂł Âł","Ġr iddled","Ġo gre","ĠLow ell","Occ up","E at","ĠHy der","ĠAdvis er","Com merce","H unt","ĠOr th","ĠComp etitive","ĠCL A","CD C","Ġsal ads","F le","Ġindustrial ized","` ,","ĠO WN","Ġbec k","ĠPart icularly","oub t","Ġm M","ĠHuss ain","ĠChen nai","Ġ9 20","Ġappoint ing","ĠCull en",",,,, ,,,,","Ġp ores","ver ified","Ġbi ochemical","em ate","Ġcoward ly","ĠHels inki","ĠEthiop ian","S OURCE","ER C","est ro","Ġbi otech","ĠS our","Ġbrew er","Bloom berg","Ġintens ify","Gl ass","an co","ĠF DR","gre SQL","ĠF ires","©¶æ ¥µ","ec o","100 1","ĠHom eless","Ġinstant aneous","ĠH aste","ig el","D iamond","Ġp aving","Ġland fill","Ġd ads","h oun",": ]","Ġinc endiary","ĠLiving ston","ĠHil bert","ĠChe cks","st yles","in ators","ĠCl ive","ph rine","Ġchimpan zees","Ġp all","ĠJ M","ĠAad haar","ð Ŀ","Ġachie vable","dis abled","P ET","OOOO OOOO","M ot","Ġint angible","Ġbal let","ĠWe bs","ĠEst imated","Effect s","Ġb ailed","Josh ua","Ġturb ulence","Ġoccup ant","ĠDay light","Ġ36 1","me et","Ġstat ically","Ġon look","Ġk i","il legal","Ġvel vet","Ġdehyd ration","Ġacqu ies","ĠRe z","ak ura","ĠU pton","at ro","Ġincomp rehensible","Ġback door","ĠRh ino","7 27","Ġmath s",") +","Ġhe resy","Ġd f","ĠRoc he","ĠL ydia","Ġpanc reat","re ply","arre ll","Ġsolicit ation","Ġcirc adian","BI P","Ġfor ay","Ġcrypt ic","iz u","ime o","ĠTom ato","ĠH oms","ex amination","Ġqu arry","ĠVal iant","ĠJer icho","ĠIN CLUD","Ġ18 40","5 19","Ġres ists","Ġsnap shots","ĠSp ur","ĠAnt iqu","Log in","Ġbest selling","Ġant ic","ĠS utherland","ãĤ¢ ãĥ«","Ġ~ /","ĠP arm","è ĥ","P ages","int ensity","Ġimm obil","Ġ18 65","zz o","Ġn ifty","Ġf entanyl","ĠPres ervation","op hen","Ġd arts","ĠD inosaur","po inters","ĠR ite","s uggest","aware ness","ĠSher idan","Ġst ances","Ġsor cery","Ġper jury","ĠNik ola","ie ver","Ġf iance","ĠJordan ian","ĠBall oon","Ġn ab","Ġk b","Ġhuman ities","ĠTan aka","hill ary","Ġconsult ancy","ĠZ ub","Ġrem ission","Ġconf id","CH Q","ĠF ug","Ġimpro vis","Y ep","/ _","Ġunwilling ness","Ġport folios","05 5","ĠInstruct or","aim an","Ġclaim ants","M bps","ĠBy e","re ceived","T weet","Ġind emn","ri z","am ara","N at","Ġeval uates","ĠL ur","ep ad","FO X","ĠTh ro","Ġrust y","Ġbed rock","ĠOp rah","J B","Ġmanip ulative","Ġwill ful","Ġrel apse","Ġext ant","The me","S ensor","ĠSt ability","go vern","Ġpo ppy","Ġkn ack","Ġins ulated","ĠT ile","ĠExt rem","Ġunt old","Ġconver ge","Ġref uel","ig roup","Ġdistort ions","Ġrav aged","Ġmechan ically","ĠRe illy","ĠN ose","ĠIncarn ation","ĠBeck y","abb ling","Ġt aco","Ġr ake","Ġmelanch oly","Ġillust rious","ĠDart mouth","Gu ide","ĠR azer","ĠBen z","Ult imate","ĠSur prise","Ġpage ant","off er","Who ever","Ġw iser","Ġchem ist","ĠHE LL","ĠBul k","Ġpl utonium","ĠCO VER","Ö ¼","f ailed","Ġtire lessly","Ġinf ertility","ĠTr ident","ĠShow time","ĠC iv","V ice","requ ires","itt ance","Ġun controlled","interest ing","56 1","Ġinnov ate","ateg ic","L ie","ĠS elling","U l","Ġsav ior","ĠT osh","Ġsw ast","P ASS","Ġr ink","Ġcard io","ĠI ro","ud i","Ġv antage","Ġv ans","ĠNi ño","+ =","Ġpropag ate","< ?","Ġmethod ological","204 39","Ġtrig lycer","Ġing rained","ĠAn notations","arr anted","6 17","ĠS odium","ĠA AC","techn ical","mult ipl","Ġ3 73","å ĭ","Ġdec isively","Ġboost ers","Ġdessert s","ĠGren ade","Ġtest ifying","ĠSc ully","ID s","Ġlock down","ĠSc her","ĠR é","ĠWhit man","ĠRams ay","rem ote","Ġh ikers","ĠHy undai","Ġcons cientious","Ġcler ics","ĠSiber ian","ut i","is bury","Ġrel ayed","Ġqu artz","ĠC BI","seek ers","ull a","Ġweld ing","ĠSh al","ble acher","T ai","ĠSam son","Ġt umble","ĠInvest or","Ġsub contract","ĠShin ra","ow icz","j andro","d ad","Ġtermin ating","ĠNe ural","ä» £","Ġleak age","ĠMid lands","ĠCaucas us","í ķ","c it","ll an","iv ably","ĠAlb ion","Ġ4 57","Ġregist rations","Ġcomr ade","Ġclip board","0 47","Ġdiscour aging","ĠO ops","Ad apt","Ġem path","n v","ĠPR OT","ĠDon n","ĠP ax","ĠB ayer","t is","Squ are","Ġfoot prints","part icip","ĠChile an","B rend","ind ucing","M agn","Ġclub house","ĠMagn um","Ġenc amp","ĠEth nic","uch a","ere y","Ġw atered","ĠCal ais","Ġcomplex ion","Ġsect s","Ġren ters","Ġbr as","oÄŁ an","Time out","Man agement","Ġinf ographic","P okemon","Cl ar","Ġloc ality","Ġfl ora","as el","P ont","Ġpop ulate","ĠO ng","Ġsubs istence","Ġa uctions","ĠMcA uliffe","ĠL OOK","br inger","Ġtit an","Ġmanif old","ĠâĹ ı","Ġcalibr ated","Ġcal iphate","ĠSH E","ĠCommission ers","ce ivable","j c","W inner","5 24","Ġcond one","Other wise","Ġp iling","Ġem body","ĠCrime an","ut ics","ĠEx hibition","Ġ4 26","e ering","Ġv ying","ĠH UGE","* =-","Ġprin cipled","à ¦","Ġquir ks","ĠEdit ors","put ing","G ES","ĠF TA","ठ¾","add on","ĠH AM","ĠFrie za","W oman",". $","Ġc rib","ĠHer od","Ġtim ers","ĠSp aces","ĠMac intosh","at aka","Ġgl ide","Ġsmell ing","ĠB AL","Ġun su","Ġcond os","Ġbicy cl","ĠRev ival","55 3","Ġjugg ling","H ug","ĠKardash ian","ĠBalk ans","mult iple","Ġnutrit ious","oc ry","19 00","Ġinteg rates","Ġad joining","ĠF older","roll ment","ven ient","Ġu ber","y i","Ġwh iff","ĠJu ven","ĠB orough","net te","Ġb ilingual","ĠSp arks","ph thal","man ufact","Ġt outing","ĠPH I","Ke efe","Rew ard","Ġinf all","ĠTem per","typ ically","ĠNik ol","Ġregular s","Ġpseud onym","Ġexhib itions","Ġbl aster","Ġ40 9","w arming","Ġrever ber","Ġrecip rocal","Ġ6 70","ip ient","b ett","ĠBe gins","Ġit ching","ĠPh ar","Ass uming","Ġem itting","ĠML G","Ġbirth place","Ġt aunt","ĠL uffy","ĠAm it","Ġcir cled","ĠN ost","enn ett","Ġde forestation","ĠHist orically","ĠEvery day","Ġovert ake","79 2","Ġn un","ĠLuc ia","Ġaccompan ies","ĠSe eking","ĠTr ash","an ism","R ogue","Ġnorth western","ĠSupplement al","ĠNY U","ĠF RI","ĠSat isf","x es","5 17","Ġreass ured","Ġspor adic","Ġ7 01","Ġmed ial","Ġcannabin oid","Ġbarbar ic","Ġep is","ĠExplos ive","ĠD ough","Ġuns olved","Support ed","Ġacknowled gment","sp awn","Ġkit chens","Ġ- =","talk ing","ic ist","ĠPeg asus","ĠPS U","Ġphot on","ĠAuthent ication","R G","@# &","76 2","ĠCl air","Ġdi aper","Ġbr ist","ĠProsecut ors","ĠJ em","6 28","ĠEvery where","ĠJean ne","equ ality","ãĥ© ãĥ³","object s","ĠPel icans","Ġ39 2","Ġbl u","b ys","ĠA go","Ġinstruction al","Ġdiscrim inating","ĠTR AN","ĠCorn el","ag os","Ġty re","Ġas piration","ĠBrid gewater","\": -","! \".","ĠEn s","ĠCoc o","P ie","Ġdet ach","ĠC ouch","Ġphys ique","ĠOccup ations","osc opic","en ough","B uzz","App earance","Y P","Ġrac er","Ġcompl icity","r pm","T oy","Ġinterrupt s","ĠCat alyst","Ġut ilitarian","imp act","Ġsp aghetti","Ġp orous","Ġeste emed","Ġinc iner","ĠI OC","7 48","Ġesp resso","ĠSm ile","abil ia","6 35","Ġmathematic ian","Ġ4 24","ĠK L","ĠH IP","Ġover heard","ĠT ud","ĠT ec","Ġqu izz","Ġfl attering","Ġcon n","âĢ İ","Ġatt aches","ĠR OS","ĠAC S","Ġt cp","ĠSh ame","sk ip","res pected","ĠTrin idad","gr ain","Ġfooth old","ĠUnch arted","ĠJul io","z l","av ored","ĠAn xiety","er rors","ĠCent auri","its ch","D addy","Ġclutch ing","ĠIm plement","ĠGut ierrez","Ġ7 60","Ġtele portation","end ra","Ġrevers ible","st ros","Ad venture","08 3","Ġliber ating","Ġas phalt","ĠSp end","AR DS","im sy","PR ES","ĠEmer ging","Ġwild fires","Ġtechn ologically","Ġem its","ĠART ICLE","Ġirregular ities","Ġcher ish","çī Ī","Ġst ink","ĠR ost","Econom ic","Ġcough ing","ĠMcC ann","pro perties","ilant ro","Ġreneg oti","Trans lation","Ġin quest","ĠGra pe","oot ers","gu i","ĠSwords man","ace ae","h itting","Ġr c","Ġexert ed","ĠS AP","it ent","Ġperil ous","Ġobsc urity","Ġassass inate","Ġab original","Ġresc uing","ĠSh attered","lock ing","all ion","Ch anging","ĠHar rington","ĠB ord","ĠAfgh ans","Jam ie","aret z","ĠAugust us","Ġ38 6","8 30","Ġj og","ok ingly","Tr igger","ĠH OR","Stat istics","Ġviewers hip","Ġadd itives","h ur","Ġmaxim izing","ĠR ove","ĠLou ie","ĠBuck et","ĠCHR IST","ou sel","Ġstre aks","ir ted","Ġt ert","Ġcolonial ism","Ġbur ying","y k","Cond ition","ĠDPR K","By Id","75 1","âĹ ¼","Ġwor risome","Ġvoc ational","sl ice","Ġsa ils","ĠCorrection al","95 4","Ġt ul","K id","l uster","Ġfam ilial","ĠSp it","ĠEp iscopal","Specific ally","ĠVol cano","run s","q s","Ġve tted","Ġcram med","t rop","here r","Thank fully","Ġper cussion","Ġor anges","Ġround up","Ġ4 99","x ious","Char acters","ĠZion ism","ĠR ao","ÃĽ ÃĽ","W F","Ġunintention al","ONE Y","Gr ab","Com mercial","Ġglut amate","ĠMcK enna","ru ciating","ning ton","ih u","Ch an","ĠSw ap","Ġleaf lets","Ġfunction ally","er ous","F arm","Ġcal oric","ĠLiter ally","con cert","Ġshe nan","Ġrep aid","ey es","Ġbas hing","ĠG orge","Ġcollabor ations","Ġun account","itch ie","Ġteam work","pp elin","Ġpip ing","Ġmin ced","Ġd iam","ri eg","Ġmasc ara","Ġsuck er","ĠMo ons","App s","ĠPe ck","Ġper v","ĠFl oat","o ley","ĠN ish","im ize","Ġarom atic","u in","end ish","! /","ĠB icycle","ĠAS IC","ile ged","ĠQuad ro","ios yn","Ġlock out","ĠW ink","SP EC","Attempt s","Ġseed ed","red o","ias is","Ġsn ag","ãĥķ ãĤ©","ãĤ ¶","Ġground ing","Ġrelie ver","Ġfrivol ous","ĠG ifts","ĠF aces","Es pecially","Ġmicrobi ome","im ag","ĠSch l","ĠP les","ĠBle ach","ĠIr win","ĠE aton","ĠDisc iple","Ġmultipl ication","Ġcoer ced","Ġ4 19","st h","E vil","B omb","Ġex orc","Ġstag gered","L ESS","Ġinert ia","ĠED IT","Ġgo b","Tr aditional","Ġclass y","Lear y","ĠP AGE","yr s","Ġtrans porter","Ġmat ured","Ġhij ab","Ġbi ome","Where as","Ġex termination","ĠT ues","ĠT akeru","ĠAud rey","er ial","ĠAd en","aff les","Ġnarciss istic","ĠB aird","UT F","I re","ĠCon nie","Ch amp","Ġwhis pering","ĠH att","D K","Ġdis infect","Ġdeduct ed","Ġpart ake","Ġdown grade","ĠEs ports","ĠContin uing","Ġdemocr atically","icro bial","itt a","Ġlim estone","Ġexempt ed","ĠFren zy","H erm","7 28","Ġfled gling","Met a","765 61","69 3","% :","w ake","5 26","ĠDis cipline","Ġvirgin ity","ĠLeg ions","ĠFrank ie","int ent","Ġrest rooms","ĠRou ter","da q","Ġobjection able","âĨ ij","w ark","ĠRah ul","g ain","activ ation","abs olute","ĠAccess ed","Ġ24 00","ogg les","Ġsecond ly","ĠDEF ENSE","Ġpost age","wra pper","sh arp","7 29","Ġcommun icates","Ġadd on","ĠMil itia","H ong","Ġsl umped","ĠJP EG","ĠI car","ad ish","68 1","Ġmaj esty","ĠWolf gang","ĠEl astic","u per","Ġv iz","Ġunconscious ly","ĠST D","ĠS ass","Ġflower ing","ĠHel ic","ĠDra per","ĠAm ateur","Ġman ure","Ġdis ingen","ĠLe i","br ing","9 49","Ġinhib ited","Ġhead quartered","Ġen igmatic","�� �","Ġred ress","R H","Ġratt led","Ġd iction","l io","ĠT BA","ĠSN AP","C alling","Ġfasc ists","ĠD ove","iew icz","0 36","Ġco asts","ĠR ect","Ġ) ]","L ot","6 29","ĠS EM","ĠPeters en","ĠExpl ain","ĠBo ards","ĠBe zos","ĠJ ournals","Ġ20 24","p arser","Ġmist rust","Ġgr ate","ĠL ocked","bo a","S aint","g aming","Ġvow el","in ately","bl ow","All ah","Ġun matched","Ġb ordering","ĠExp end","n r","Or acle","rou ch","Ġcont iguous","ac us","Ġdist raught","58 1","Ġanat omical","O X","ap ixel","8 33","ĠPL US","Ġres usc","Ġab iding","57 3","Ġvac ancies","Em ily","Ġhyp othal","ĠWer ner","ĠWe e","ĠDJ s","5 13","Ġwitch craft","Ġac upuncture","ent ary","benef it","Product s","ĠP SP","ĠMP G","ĠJ inn","ĠJ arrett","Ġ4 45","ĠIm aging","ĠP yth","Fin ish","Ġte x","Ġjuven iles","Ġhero ism","Ġdoubt less","ĠA ki","ĠT end","ĠPatri arch","Ġbit ters","ĠTele communications","it atively","ag na","Ġr g","ĠS OLD","Ġcomp ulsion","ĠN asa","ĠKath ryn","Ġmillion aires","Ġintrins ically","Ġbolst ered","time out","fl o","Ġtut or","p our","Stat ement","Ġ{ *","ĠRud olph","ĠKimber ly","rog ens","adi q","] +","Ġindign ation","Ġfract uring","ĠRe leases","ĠGr ain","pro tein","L ago","Ġvac ations","Ġboot ed","ĠTH REE","ĠH G","oresc ence","Ġt f","Ġso ar","iosyn cr","Ġgl ances","ĠSp oon","ĠJ ury","ĠCow boy","Ġcreat ively","Hig her","Ġsolic itor","Ġhaw k","ac io","89 6","Ġsuperf lu","Ġbombs hell","ct ure","Ġbroker age","Ġraid ing","Ġf rench","Ġang led","Trans action","ĠGen ocide","u pe","ĠHait ian","57 2","! :","Ġunwitting ly","iter ator","sc roll","Ġtall ied","Ġbi omedical","ĠC ARD","Ġe uphem","Ġbrain storm","a quin","K o","Mic helle","ĠR unes","ĠBall istic","ud ers","Ġmod esty","ĠiP ads","ĠEzek iel","Y E","Ġstars hip","Ġpower fully","Ġper l","ĠSh ade","ĠQu art","ĠE EG","Ġfisher man","OS ED","ĠTyp ical","df x","Ġmes hes","Ġet ched","worth iness","Ġtopp led","Ġ3 96","or ius","We iss","Ġmy sql","ĠVal halla","Ù Ĵ","le asing","Ġrec omp","rap nel","S el","04 3","Ġder ailed","ĠGu ides","IR T","Ġde human","ĠBritt any","\" ))","Ġex claim","Ġb alk","Ġ8 40","CLA IM","int el","L AB","Ġpe gged","Ġast roph","sm oking","Ġrig ging","Ġfix ation","Ġcat apult","ins ide","ĠC ascade","ĠBolshe vik","G aza","Dep th","Ġloud spe","Ġalmond s","me yer","l eness","j en","f resh","Ġunbeat en","ĠSqu id","ĠPres umably","Tim er","B W","Ġro sters","Ġell ipt","ĠHar riet","dat abase","ĠMut ual","ĠComm odore","uk ed","kn ife","ĠCOMM UN","h ya","Ġmel ts","arch ives","Ġrat ification","Ġmultip lying","Ġinter oper","Ġasc ert","w ings","ver ting","ĠScorp ion","ay e","ĠPorts mouth","ĠM TA","n it","iaz ep","Ġqu arantine","Ġslides how","Ġcent imeters","Ġsyn opsis","Ġsp ate","th irst","Ġnom inating","ĠMel vin","Pre view","Ġthro b","Ġgener ational","ĠRad ius","rest ling","put able","aw ar","N ECT","Ġunlaw fully","ĠRevel ations","Wik ipedia","sur v","Ġeye ing","ij n","ĠF W","Ġbr unt","Ġinter stellar","Ġcl itor","ĠCroat ian","ĠCh ic","ev a","ĠDis app","ĠA kin","iner ies","d ust","Interest ed","Ġgen esis","ĠE ucl","ö n","p icking","Ġmut ated","Ġdisappro ve","ĠHD L","Ġ6 25","Ì ¶","c ancer","Ġsqu ats","Ġle vers","Disc uss","= ]","D ex","ĠVIDE OS","A UD","Ġtrans act","ĠKin ect","ĠK uala","ĠC yp","7 47","Ġsh attering","Ġarsen ic","ĠInt ake","ĠAngel o","ĠQu it","ĠK he","Ġ18 93","M aker","0 29","ĠPain ting","Dis able","9 16","Ġanal ges","Ġtact ile","Ġprop hes","Ġd iced","ĠTravel s","ĠHe ader","ĠClub s","Ass istant","Ġinc rim","Ġd ips","Ġcruc ifix","ĠShan ahan","ĠInter pret","Ġ40 90","al ogy","abb a","Ġsimul ac","hus band","S IM","Ġrecy cle","uc er","ed ged","Ġre naissance","ĠBomb ay","Cath olic","ĠL INE","ĠCl othing","re ports","Ġpl aus","Ġd ag","ĠM ace","Z I","Ġintr uder","ĠVeter inary","g ru","Ġsne aky","ĠS ie","ĠC innamon","P OSE","Ġcou rier","ĠC NS","Ġemanc ipation","s it","Ġplay through","ĠFac ilities","v irt","ĠG auntlet","Thom pson","Ġunbeliev ably","Param eters","Ġst itching","ign e","ĠTH ESE","Priv acy","Ġshenan igans","Ġvit ri","ĠVal id","59 1","Ń ·","ĠProt otype","ink a","SC P","ĠT id","è Ī","old ed","Ġindividual ity","Ġbark ing","Ġm ars","ĠW D","Ġ8 20","Ġt ir","Ġsl apping","Ġdisgr untled","ĠAng ola","ri us","ĠTorn ado","ĠTh urs","Ġcapt cha","Ġang st","ĠP og","ĠAssass ins","ĠAd idas","Ġjoy ful","Ġwh ining","Emer gency","Ġphosph orus","Ġatt rition","oph on","ĠTimber wolves","ĠJ ah","ĠBr inging","ĠW ad","ĠEn sure","oh l","ĠX ie","omm el","c mp","Ġz ipper","Ġrel at","ĠCor ridor","m ilo","T ING","Av g","Ġcro pped","] }","Ġr aged","ĠLump ur","ĠGuer rero","our ke","N ut","Ġoff sets","og lu","dr m","Ġmort als","lat able","Ġdismiss ive","ä¸ ī","Ġthro ats","Ġchips et","ĠSpot light","Catal og","art ist","G b","Ġch illy","Ġst oked","Ġ3 74","W ard","L atin","Ġf iasco","Ġble ach","Ġb rav","Enh anced","Ġin oc","ĠFior ina","_ >","Ġle ukemia","Ġel uc","Ġannoun cer","ĠLith uan","ĠArm ageddon","å ĩ","Len in","ĠR uk","Ġpe pp","ĠRom antic","ĠP IT","ĠInter stellar","ĠAt kinson","R aid","J s","Go al","C ourse","Ġvan ishing","es ley","ĠR ounds","Els a","59 3","Ġredund ancy","ĠST AND","Ġprop hetic","Ġhabit able","ry u","Ġfaint ly","M ODE","Ġfl anked","IR C","Aw esome","Ġsp urious","ĠZ ah","ĠMS G","Ġsh ading","Ġmotiv ational","ĠSant ana","ĠS PR","Ġexc ruciating","om ial","ĠM iko","ĠLe opard","A byss","Ġ[ |","d irty","Ġbath s","Ġdem oral","and re","P B","Ġun ification","Ġsac rament","Ġ[ &","Ġpric eless","Ġgel atin","Ġeman ating","ĠAll aah","98 6","Ġout burst","Ġer as","ĠX VI","ĠSP I","O tt","ĠLaz arus","PL IED","F lying","blog s","W isconsin","R aven","Ġreb ate","Ġcreep s","ĠSp an","ĠPain ter","ĠKir a","ĠAm os","ĠCor vette","Cons umer","ĠRec over","ck i","Ġpes ky","ĠIn vention","Compan ies","Ġchalleng ers","ad emic","ĠUkrain ians","ĠNeuro log","ĠFors aken","Ġent rants","Ġemb attled","Ġdef unct","ĠGlac ier","Ġpo isons","ĠH orses","m akes","ĠD irt","Ġ4 23","hh h","ĠTrans formation","QUI RE","................ ..","Ġtrave ller","ĠSe xy","ĠK ern","ip olar","Ġransom ware","oooooooo oooooooo","E c","rub y","Prof essional","ĠOut break","arg ument","G rey","ĠFif a","ĠCH O","ĠFOR M","ĠAm trak","- [","Ġcr adle","Ġantioxid ants","ãģ®å ®","7 36","ĠNAS L","ĠContribut ions","Ind iana","ĠST EP","C SS","Ġsal ient","Ġall ocations","yr ights","Ġm ashed","ĠCut ter","Sex ual","Ġp ounded","Ġfan base","Ġc asc","ĠTrans parency","Ġanaly tic","ĠSummon er","× ŀ","ĠAD C","det ail","Ġvan quished","Ġcr abs","ar ie","Dest roy","ĠS ack","Ġtrans istor","Al abama","ĠK oen","ĠFisher ies","c one","Ġannex ed","ĠM GM","es a","Ġf aked","ĠCong ratulations","Ġhind ered","Ġcorrection al","ĠI TV","lee ve","Ġin appropriately","lic ks","Ġtresp ass","Ġp aws","Ġnegoti ator","ĠChrist ensen","lim its","ĠDian ne","Ġeleg ance","ĠContract s","an ke","Ob j","Ġvigil ance","Ġcast les","ĠN AD","ĠHol o","Ġemph atically","ĠTit us","ĠServ ing","ĠRich ie","ĠP igs","5 68","Ġanim osity","ĠAtt ributes","ĠU riel","M Q","my ra","ĠApplic ant","Ġpsychiat rists","ĠV ij","ĠAb by","ag ree","P ush","Ġk Wh","hib a","Ġinc ite","ĠWe asley","ĠTax i","minist ic","hy per","ĠF arn","Ġ6 01","ĠNation wide","F ake","95 2","Ġma ize","Ġinteract ed","Ġtransition ed","Ġparas itic","Ġharm onic","Ġdec aying","Ġbas eless","ns ics","Ġtrans pired","Ġabund antly","ĠFore nsic","Ġtread mill","ĠJ av","ab and","Ġssh d","Ġfront man","ĠJak arta","oll er","dro ps","ĠSERV ICES","rompt u","oph ical","h ospital","bled on","6 45","Ġmid range","ĠEV ENT","cul ated","raw led","Ġper ched","Ġover board","ĠPe el","ĠP wr","ĠCar th","ĠCOM PLE","co e","sh all","Ġdeter rence","M ETHOD","ĠAbs ent","M EN","Ġs ill","ĠLE VEL","Y ork","Ġsin ners","ĠOP EC","ĠN ur","ĠDesign s","se lection","Ġunw orthy","CH A","Ġstreng thens","88 3","ed ly","Ġslic ing","Ġmal nutrition","Ġfilm making","ĠPol k","ur ated","Ġ4 21","bre akers","!' \"","Ġwet lands","ĠDisc rimination","Ġallow able","Ġste ered","ĠSic ily","S AM","Ġmust ache","Ġm ids","Ġcl ipped","Ġcirc ulate","Ġbr ittle","ĠBuild ings","ra ised","ĠRound up","Ġwealth ier","Ġoverw rite","Ġover powered","ĠGerr ard","s ites","PD ATED","Ġacute ly","ĠGam ble","Ġp im","ĠK us","Typ ically","De ploy","ĠMoroc can","p otion","com be","Ġvigil ante","Ġ36 3","St ew","ĠB agg","Ġres ided","ĠSp o","Ġrem nant","Ġempt iness","br ainer","Ġout patient","pri ority","Ġle ptin","ĠPay ton","ĠGle aming","ĠS hed","ĠPol o","ĠMormon ism","rest ricted","arl ane","w x","Ġcreat ine","ĠAn on","ĠST UD","ĠJ UL","ĠT ee","5 28","08 9","Ġhat ched","Dis patch","ĠCompos ite","Ġ45 1","p uff","ĠX COM","ĠOr n","ĠTH ANK","END ED","ĠAshe ville","Ġà ľ","Ġman go","ĠS lightly","world ly","ĠW ander","ĠExp and","ĠCh r","M ist","Ġorthodox y","ĠUN ESCO","reg ate","Else where","k ie","ir led","Ġtopp le","Ġadopt ive","ĠLeg s","d ress","ĠS agan","b are","ĠGl ou","Cr unch","Ġhelp ers","Ġchron ically","ĠH uma","1 0000","Ġaccommod ating","äº Ķ","Ġwrink les","Ġdod ged","four th","Ġpre con","Ġcompress or","ĠK are","Ġev ict","ĠWar wick","im ar","Ġmodern ization","Ġband wagon","Ġref uted","Ġnet ted","ĠNa ples","ĠGen ie","per ors","Ġfield ed","Ġde re","ĠPar ables","le es","Ġtr out","asp ers","Ġn ihil","Ġhapp iest","Ġflo ppy","ĠLo ft","ĠHe ard","Ġun ison","Ġl ug","ĠRed mond","class ic","Supp orters","SH IP","G MT","Ġfue lled","ç IJ","Ġd d","ĠEmin em","Ġ18 97","NY SE","Ġsecret aries","ĠF IA","ĠCanaver al","F avorite","Ġp omp","Ġdetain ee","ers hip","aim on","i our","ĠA pex","Ġplant ations","am ia","ac ion","R ust","Ġtow ed","ĠTru ly","5 77","Ġshel tered","r ider","W o","Ġl air","ĠInt elligent","impro ve","m atically","Ġet iquette","ad ra","all o","ĠJun o","any thing","ĠStru ggle","ĠPred ict","ĠGr imes","ĠAMER ICA","ct x","ĠSit uation","W OOD","Ġsol uble","me ier","Ġintoler able","ang ering","Ġun interrupted","Ġtool tip","Ġinterrog ated","Ġgun ned","ĠSne ak","æŃ ¦","Ġt ether","Ġcr umble","L ens","Ġclust ered","ĠSy l","ĠHas an","Ġdystop ian","w ana","Ġjoy stick","ĠTh ib","amm u","Tom orrow","5 46","Ġoverc ame","Ġminim ized","cept or","Run ner","ENG TH","ĠBrend a","ĠAchieve ments","Ġtor ches","Ġrapp ort","ĠInvestig ator","ĠHand ling","rel ation","g rey","8 15","Ġk cal","ĠComm ands","d q","Ġcur ls","Ġbe arer","Ġcyn icism","it ri","ĠUse ful","B ee","D CS","Ġab ras","P ract","BIL ITIES","7 12","Ġdebug ger","Ġdebt or","ĠL ia","ĠK ers","Ġexacerb ate","ĠSt acy","ĠB land","ĠSc enes","Ġbranch ing","âĸĪâĸĪâĸĪâĸĪ âĸĪâĸĪâĸĪâĸĪ","ape ake","Ġs alsa","Ġmish and","ĠKon ami","ĠN ib","Ġanecd ote","Ġagree able","Ï ī","ĠNath aniel","ĠHe isman","ĠB eware","Ġ18 86","spect ive","69 1","5 22","Ġinhib its","Ġhas hing","Ġ18 89","å° Ĩ","v ich","P ure","Ġsolid ly","Ġaspir in","im aru","Ġstreet car","ĠU CS","ĠJ udd","Ġflash backs","p ins","Ġ14 40","ĠUN HCR","ĠSym ptoms","T IT","5 38","F ra","% );","Ġo oz","Ġcur few","Ġcal med","Ġparticip ates","Te X","Ġnons ensical","Ġfull back","ĠDe L","mon key","h ari","Ġmetabol ites","Ġloot ed","ĠAL WAYS","ĠB CC","L t","oc het","B one","Ġveto ed","Ġg cc","ĠCL ICK","Ġ18 88","s af","Ġstiff ness","Ġlow ly","ĠGe h","vers on","ors et","Ġun foreseen","Ġan esthesia","ĠOpt ical","Ġrecon structed","ĠT up","sh ows","NEW S","ĠNewsp aper","ĠA SA","ter a","N umbers","Ġinexpl icable","× ij","Ġhard ness","unt arily","ĠA cer","grad ient","ARD IS","Ġwood land","Ġmetaph ors","ĠWem bley","ĠPa vel","phil is","Ġre writing","Ġpercept ual","Ġ10 70","worm s","ĠDown s","Ġunsur prisingly","Ġtag ging","fl ame","Ġlit res","Ġboun ces","ĠB abe","sh ut","Ġoverd oses","ĠShe ila","ĠCh au","ĠBl ess","Capt ure","ĠSign ificant","ĠSc ion","Ġ38 9","ĠMc H","ĠTitan ium","ĠMe al","amed a","ag ents","agg ressive","B illy","76 3","ĠS aying","DER R","it one","Coll ins","B ound","Ġbol ted","ĠDM CA","95 3","Ġun iqueness","Ġep igen","un ci","ant am","Ġreck oning","ch airs","OG R","ĠSen egal","Ġ18 62","re levant","Ġ ¯","Ġpharm acies","ĠG eral","v ier","Y an","OR PG","Ġrab id","b ending","ĠUN ITED","Ġ4 65","As sembly","Ġwe ep","Ġbe hest","ĠMother s","ĠJ ace","h id","Ġwh irlwind","ĠUN IVERS","Ġut opian","Ġkidn ap","Ph ilipp","K in","89 3","Ġlivest ream","ĠM ISS","Ġsub versive","ĠTechn iques","ĠJUST ICE","ĠB ASE","Ġ38 7","Ġassail ants","ĠHard core","Ġsprink led","ĠP se","é ļ","print ed","ĠH au","OR GE","ĠT OUR","Ġl aced","Ġit ch","G iving","Ġport ed","78 1","//////////////// ////////////////","bre eding","Ġlog ger","ĠH OL","inn ie","First ly","Ġembry onic","Ġdeleg ated","p ai","O IL","Ġcentr ally","ĠR x","ĠSc outing","D utch","Ġhe reditary","ĠCru iser","s at","5 29","ĠMar riott","other mal","Ġprohib itions","E arn","ĠSt ab","ĠColleg es","ĠBel ief","st retched","ĠL H","ĠEntity Item","C IA","Ġun rem","Ġlaure ate","Ġdenomin ations","sum mary","h ler","S pect","ĠK laus","ĠBe ans","Ġins ur","ĠPA X","Ġfield er","ĠV et","ĠSp arrow","z ie","ĠS Q","ĠMond ays","ĠOff line","ĠLer ner","ĠExt ensions","Ire land","Ġpatron age","Ġcontrast ed","ĠMan ia","h irt","Mos cow","Ġcondem ns","ĠAn ge","Ġcomp osing","ĠPe pe","ĠP addock","Ġheter ogeneity","Ġide ologically","Ġf ishes","Ġcur sing","ĠR utherford","ĠFlo ating","ĠAm elia","Te a","Syn opsis","Ġstun ts","Ġbe ad","Ġstock ing","ĠM ILL","ob ook","mass ive","\\ <","Ġh ump","ĠPref erences","Engine Debug","ge ist","ĠNiet o","ome ver","ish y","eval uate","col onial","Altern ative","ĠGo Pro","ĠV ortex","ĠNET WORK","ans ky","Sec ure","ĠTh rust","Sn ake","Ġparcel s","Ġsam urai","Ġactress es","N ap","M F","ifer ation","Be er","5 23","ĠI ly","oint ment","P ing","Ġstri ped","ĠMell on","oss ession","Ġneut ron","end ium","Ġa ph","ĠFlav oring","Ġ38 3","Ġrespons iveness","ĠJ indal","ĠHitch cock","Den ver","ĠDRAG ON","sm anship","ĠDu pl","Ġs ly","Ġweb cam","ĠTw ain","ĠDar ling","ili ate","cons umer","D IT","Ġnames ake","Ġun orthodox","Ġfun er","ĠPL oS","ĠCONTR OL","ozy g","ogl obin","F ACE","ER G","ĠD ia","ĠF iesta","ce le","0 34","Ġencl ave","âĸ¬ âĸ¬","on ement","al ist","M and","Ġhome grown","ĠF ancy","Ġconcept ions","ĠCont ains","ure en","Ġreiter ate","Ġme ager","Ġinstall ments","Sp awn","6 27","Ġphot oc","ĠCab rera","ĠRos enthal","ĠLans ing","is ner","Ġinvest s","ĠUFO s","EX P","Hard ware","Ġtr agically","Ġconced es","ie ft","ch am","bor gh","ĠSch r","ĠMel anie","ĠH oy","Ġvisit ation","Ġid iosyncr","Ġfract ions","Ġfore skin","ob os","Ġpo aching","ĠVI EW","Ġstimul ates","ĠG ork","can on","M IC","ĠNem esis","ĠInd ra","ĠDM V","Ġ5 29","Ġinspect ing","Ġgrand ma","ĠW hedon","ĠSh ant","ĠP urg","ik an","ĠT eg","ĠCL R","z ac","Vict oria","ĠVer ify","ion ics","Ġpart ying","ĠM ou","col our","Ġtestim onies","l ations","Ġpress uring","hi ro","ac ers","Ġf id","ang ler","ĠCS I","Ġhere after","Ġdiss idents","report ing","iph any","che v","Ġsol itude","Ġl obe","Ġind is","Ġcred ential","re cent","ad ult","ĠNir vana","ĠFranch ise","L ayer","H yp","ĠBerks hire","Ġwill s","t if","Ġtot em","ĠJud ah","rep air","Inst ant","5 48","Ġemb assies","Ġbott leneck","Ġb ount","Ġtyp ew","ĠAl vin","j ing","im ilar","R ush","Ġbr im","ĠHEL P","A im","] '","Ġpass ively","Ġbound ed","ĠR ated","Ġcriminal ity","Ġbiom ark","Ġdisp atcher","ĠTow ards","Ġ+ ++","right eous","f rog","ĠP anc","C arter","0 32","æ© Ł","Ġult raviolet","ĠLic ensed","ĠT ata","ĠBl essing","ĠG AM","Ġchem ically","ĠSe af","ĠRE LE","ĠMerc enary","capital ist","Ġform ulations","Ġann ihilation","ĠVer b","ĠAr gon","Ġun loaded","Ġmorp hed","Ġconqu ering","back er","I ELD","Ġtheft s","Ġfront runner","ĠRoy ale","ĠFund amental","el ight","C hip","necess ary","ay n","ĠSl ip","Ġ4 48","cern ed","P ause","Ġshock ingly","ĠAB V","Ġcomp osure","7 33","ĠMotors port","ah ime","Mur ray","M ach","Ġgr ids","Ġdeb ian","Ġfurther more","Ġdexter ity","ĠCollect ions","os lov","il age","b j","ĠMont eneg","Ġstrut Connector","Ġmassac res","Ġbrief s","fet ched","uv ian","ol ition","Fail ure","emon ic","Ġfl ared","Ġclaim ant","Ġc ures","Ġgive aways","ĠSubst ance","al ions","Ġcr inge","ĠK ul","Ġarist ocracy","ĠUl ster","ol ated","h ousing","ĠM IS","Ġgl ared","ĠWil helm","ne eds","lam bda","build ers","ĠV IS","Ġradi ator","ĠGhost busters","Ġ4 36","act ual","Ġher ds","ç a","watch ing","Ġcounter ing","Ch arge","Ġchar red","Ġwar heads","Ġiod ine","ĠM acy","04 1","Ġdepart ures","ĠS ins","Ġdy ed","ĠConcept s","g ado","7 13","Ġquot ations","Ġg ist","ĠChrist y","Ġant igen","ĠHem p","ĠD rawn","ĠB arg","ez vous","Ġp aternity","Ġar du","ĠAnch orage","ĠR ik","Ġover loaded","ĠUs ername","ĠTam my","ĠN au","ĠCell ular","Ġw aning","Ġrod ent","ĠWor cester","il ts","ĠT ad","Ġdwell ings","Ġbull ish","4 31","Ġretali ate","Ġmig raine","ĠChev ron","CH ECK","Ġdon key","c rim","SP A","ĠAn alog","Ġmarqu ee","ĠHa as","B ir","ĠGD DR","ĠDownload s","Ġwill power","ĠFor th","ĠRecord ed","Ġimp ossibility","ĠLog ged","ĠFr anks","ĠR att","in itions","Ġclean ers","Ġsore ly","Ġflick ering","ĠEx amination","c atching","allow een","Ms g","Ġdun no","F a","Ġdys ph","c razy",".' '.","Ġmain line","Ġc s","Ġp tr","ĠW ally","ig un","95 1","ĠBig foot","f ights","Ġretrie ving","J r","Ġdupl ication","ĠExpl an","Ġrel ational","Ġqu aint","Ġbisc uits","Ġad o","Ġsh udder","Ġantid ote","blood ed","ks h","Ġsa uces","Ġrein vest","Ġdispens ary","ĠD iver","Ġ9 000","stud ent","Ġin separ","esc ap","Ġtodd lers","ĠGP IO","ĠAss ignment","head ers","Ġlack luster","Ġab ack","95 6","Ġtool bar","7 45","Ġo ust","Ġcontempl ation","ĠPRES IDENT","Ġ4 58","==== ==","Ġguarantee ing","ĠHe ist","ĠCann es","Ļ ½","Ġcollabor ator","ĠAm p","Ġg ou","ĠSH ALL","st ories","78 3","Ġmobil ized","Ġbro od","ĠL U","ĠðŁ ij","Ġref in","ĠAnthrop ology","v ind","ill i","Ġwarrant ies","ĠB abel","Ġsw ath","Ġc aches","Ġantagon ists","art ifacts","Ġhot ly","ĠSt arts","ĠG ö","z ag","!! !!!","Ġsc ourge","Ġcons piring","ru its","re verse","ĠShe en","ĠJes uit","ĠGiov anni","ad ies","Ġbutt ocks","ear cher","ac an","Ġvolley ball","Ġshroud ed","Ġscore board","b ats","ĠI PM","Ġass es","Ġde regulation","ĠTe legram","ĠReb oot","Ġ7 000","ĠCan ary","Ġk ernels","ĠFranç ois","ĠD uff","ĠP on","ĠLe ica","ĠGar min","Ġor phans","ĠClaud ia","Ġcal endars","ĠLe ilan","ent o","R ocket","Ġbr unch","ĠHaw king","ain ers","Ġsens ibilities","Ġk W","ĠK and","Ġre claimed","Ġinteresting ly","× ©","rom y","J M","ĠEnhance ment","b ush","Sk ip","Ġrapp ers","Ġg azing","p edia","ath lon","Rev olution","Ġsn ipers","Ġre verted","Ġconglomer ate","T erry","79 4","Ġhars her","Ġdes olate","ĠHit man","Comm ission","Ġ( /","â̦ .\"","Com par","Ġampl ification","om inated","Ġreg ress","ĠColl ider","Ġinform ants","Ġg azed"]}} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/long.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/long.json new file mode 100644 index 000000000000..34e5865c1dfd --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/long.json @@ -0,0 +1,849 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3", + "shape": "long", + "prompt_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ", + "expected_token_ids": [ + 83240, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 101339, + 55848, + 39208, + 10434, + 57037, + 14, + 67956, + 109387, + 51320, + 14, + 10012, + 696, + 312, + 4667, + 5158, + 14408, + 121876, + 329, + 3992, + 3404, + 7314, + 492, + 1231, + 95691, + 127631, + 86798, + 67, + 16, + 223 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/multi_turn_with_tools.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/multi_turn_with_tools.json new file mode 100644 index 000000000000..435aa2c784a3 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/multi_turn_with_tools.json @@ -0,0 +1,83 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3", + "shape": "multi_turn_with_tools", + "prompt_text": "<|im_start|>system\nYou have tools.<|im_end|>\n<|im_start|>user\nWeather in Paris?<|im_end|>\n<|im_start|>assistant\n\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}\n<|im_end|>\n", + "expected_token_ids": [ + 30, + 94, + 328, + 37864, + 94, + 32, + 27824, + 201, + 3476, + 611, + 6704, + 32334, + 94, + 328, + 42616, + 94, + 1018, + 30, + 94, + 328, + 37864, + 94, + 32, + 5265, + 201, + 58565, + 295, + 11111, + 33, + 30, + 94, + 328, + 42616, + 94, + 1018, + 30, + 94, + 328, + 37864, + 94, + 32, + 624, + 15059, + 201, + 30, + 72461, + 112042, + 1018, + 24313, + 2852, + 3362, + 582, + 1133, + 65, + 50219, + 1760, + 582, + 83772, + 3362, + 28612, + 37399, + 3362, + 582, + 51119, + 4, + 30316, + 1718, + 72461, + 112042, + 5451, + 94, + 328, + 42616, + 94, + 1018 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/short.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/short.json new file mode 100644 index 000000000000..0f0381b4f797 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/short.json @@ -0,0 +1,12 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3", + "shape": "short", + "prompt_text": "Hello, world!", + "expected_token_ids": [ + 19923, + 14, + 2058, + 3 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/special_token_heavy.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/special_token_heavy.json new file mode 100644 index 000000000000..b5008ef97924 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/deepseek-v3p2/special_token_heavy.json @@ -0,0 +1,63 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3", + "shape": "special_token_heavy", + "prompt_text": "<|im_start|>system\nYou are helpful.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\nHello<|im_end|>\n<|endoftext|>", + "expected_token_ids": [ + 30, + 94, + 328, + 37864, + 94, + 32, + 27824, + 201, + 3476, + 477, + 11502, + 32334, + 94, + 328, + 42616, + 94, + 1018, + 30, + 94, + 328, + 37864, + 94, + 32, + 5265, + 201, + 23166, + 30, + 94, + 328, + 42616, + 94, + 1018, + 30, + 94, + 328, + 37864, + 94, + 32, + 624, + 15059, + 201, + 19923, + 30, + 94, + 328, + 42616, + 94, + 1018, + 30, + 94, + 523, + 2154, + 2067, + 94, + 32 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/long.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/long.json new file mode 100644 index 000000000000..2341e0f95c34 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/long.json @@ -0,0 +1,669 @@ +{ + "model_id": "openai/gpt-oss-20b", + "shape": "long", + "prompt_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ", + "expected_token_ids": [ + 61495, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 86529, + 38714, + 25840, + 2353, + 36204, + 11, + 54472, + 91785, + 45688, + 11, + 10412, + 621, + 160226, + 14725, + 173578, + 4518, + 110546, + 859, + 79682, + 78404, + 151394, + 13, + 220 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/multi_turn_with_tools.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/multi_turn_with_tools.json new file mode 100644 index 000000000000..8d0a76b1c27d --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/multi_turn_with_tools.json @@ -0,0 +1,80 @@ +{ + "model_id": "openai/gpt-oss-20b", + "shape": "multi_turn_with_tools", + "prompt_text": "<|im_start|>system\nYou have tools.<|im_end|>\n<|im_start|>user\nWeather in Paris?<|im_end|>\n<|im_start|>assistant\n\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}\n<|im_end|>\n", + "expected_token_ids": [ + 27, + 91, + 321, + 10949, + 91, + 29, + 17360, + 198, + 3575, + 679, + 8437, + 30502, + 91, + 321, + 13707, + 91, + 523, + 27, + 91, + 321, + 10949, + 91, + 29, + 1428, + 198, + 29602, + 306, + 12650, + 190440, + 91, + 321, + 13707, + 91, + 523, + 27, + 91, + 321, + 10949, + 91, + 29, + 173781, + 198, + 27, + 17952, + 25158, + 523, + 10848, + 897, + 1243, + 392, + 522, + 170154, + 672, + 392, + 34317, + 1243, + 10494, + 17500, + 1243, + 392, + 72782, + 18583, + 739, + 808, + 17952, + 25158, + 3784, + 91, + 321, + 13707, + 91, + 523 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/short.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/short.json new file mode 100644 index 000000000000..ce307d428dfa --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/short.json @@ -0,0 +1,12 @@ +{ + "model_id": "openai/gpt-oss-20b", + "shape": "short", + "prompt_text": "Hello, world!", + "expected_token_ids": [ + 13225, + 11, + 2375, + 0 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/special_token_heavy.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/special_token_heavy.json new file mode 100644 index 000000000000..32e97dd49e01 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/gpt-oss-20b/special_token_heavy.json @@ -0,0 +1,56 @@ +{ + "model_id": "openai/gpt-oss-20b", + "shape": "special_token_heavy", + "prompt_text": "<|im_start|>system\nYou are helpful.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\nHello<|im_end|>\n<|endoftext|>", + "expected_token_ids": [ + 27, + 91, + 321, + 10949, + 91, + 29, + 17360, + 198, + 3575, + 553, + 10297, + 30502, + 91, + 321, + 13707, + 91, + 523, + 27, + 91, + 321, + 10949, + 91, + 29, + 1428, + 198, + 12194, + 27, + 91, + 321, + 13707, + 91, + 523, + 27, + 91, + 321, + 10949, + 91, + 29, + 173781, + 198, + 13225, + 27, + 91, + 321, + 13707, + 91, + 523, + 199999 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/long.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/long.json new file mode 100644 index 000000000000..7cdda5c45541 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/long.json @@ -0,0 +1,669 @@ +{ + "model_id": "Qwen/Qwen3-30B-A3B", + "shape": "long", + "prompt_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ", + "expected_token_ids": [ + 32783, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 46931, + 26342, + 23655, + 2444, + 27212, + 11, + 35140, + 57924, + 30060, + 11, + 10923, + 653, + 79122, + 18965, + 86404, + 8621, + 72204, + 1842, + 57296, + 58917, + 85927, + 13, + 220 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/multi_turn_with_tools.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/multi_turn_with_tools.json new file mode 100644 index 000000000000..ddf00fbff40c --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/multi_turn_with_tools.json @@ -0,0 +1,50 @@ +{ + "model_id": "Qwen/Qwen3-30B-A3B", + "shape": "multi_turn_with_tools", + "prompt_text": "<|im_start|>system\nYou have tools.<|im_end|>\n<|im_start|>user\nWeather in Paris?<|im_end|>\n<|im_start|>assistant\n\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}\n<|im_end|>\n", + "expected_token_ids": [ + 151644, + 8948, + 198, + 2610, + 614, + 7375, + 13, + 151645, + 198, + 151644, + 872, + 198, + 28981, + 304, + 12095, + 30, + 151645, + 198, + 151644, + 77091, + 198, + 151657, + 198, + 4913, + 606, + 788, + 330, + 455, + 69364, + 497, + 330, + 16370, + 788, + 5212, + 8926, + 788, + 330, + 59604, + 95642, + 151658, + 151645, + 198 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/short.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/short.json new file mode 100644 index 000000000000..231a474b34d7 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/short.json @@ -0,0 +1,12 @@ +{ + "model_id": "Qwen/Qwen3-30B-A3B", + "shape": "short", + "prompt_text": "Hello, world!", + "expected_token_ids": [ + 9707, + 11, + 1879, + 0 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/special_token_heavy.json b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/special_token_heavy.json new file mode 100644 index 000000000000..27968dff7057 --- /dev/null +++ b/experimental/sgl-router/tests/fixtures/tokenizer_parity/qwen3-30b/special_token_heavy.json @@ -0,0 +1,30 @@ +{ + "model_id": "Qwen/Qwen3-30B-A3B", + "shape": "special_token_heavy", + "prompt_text": "<|im_start|>system\nYou are helpful.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\nHello<|im_end|>\n<|endoftext|>", + "expected_token_ids": [ + 151644, + 8948, + 198, + 2610, + 525, + 10950, + 13, + 151645, + 198, + 151644, + 872, + 198, + 13048, + 151645, + 198, + 151644, + 77091, + 198, + 9707, + 151645, + 198, + 151643 + ], + "skip_special_tokens": false +} diff --git a/experimental/sgl-router/tests/proxy/chat_routing.rs b/experimental/sgl-router/tests/proxy/chat_routing.rs new file mode 100644 index 000000000000..c55fdc48d0d9 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/chat_routing.rs @@ -0,0 +1,1333 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ModelConfig, ObservabilityConfig, + PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, +}; +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry; +use sgl_router::proxy::Proxy; +use sgl_router::server::app::build_router; +use sgl_router::server::app_context::AppContext; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::{Worker, WorkerRegistry}; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use http_body_util::BodyExt; +use std::sync::Arc; +use std::time::Duration; +use tower::ServiceExt; + +const TEST_TIMEOUT: Duration = Duration::from_secs(5); + +fn config_for(_worker_url: &str) -> Config { + Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: ObservabilityConfig::default(), + models: vec![ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + } +} + +fn build_ctx_with_worker(url: &str) -> Arc { + let cfg = config_for(url); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + let _ = registry.add(WorkerSpec { + id: WorkerId("w1".into()), + url: url.to_string(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + // Per-request worker URLs flow from the registry through + // `forward_*_to(&worker.url, ...)`; the proxy itself is URL-less. + let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap()); + Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)) +} + +#[tokio::test] +async fn non_streaming_returns_200() { + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": false + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(v["choices"][0]["message"]["content"], "ok"); +} + +#[tokio::test] +async fn non_streaming_upstream_unreachable_returns_502_unreachable() { + // Bind a port, drop it — guarantees a closed/refused TCP destination. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dead_url = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + + let ctx = build_ctx_with_worker(&dead_url); + let app = build_router(ctx); + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "upstream_unreachable" + ); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let body_str = String::from_utf8_lossy(&bytes); + assert!( + body_str.contains("\"code\":\"upstream_unreachable\""), + "body: {body_str}" + ); + // Generic message — must not leak reqwest source or worker URL. + assert!( + !body_str.contains(&dead_url), + "worker URL must not leak in client-visible body: {body_str}" + ); +} + +#[tokio::test] +async fn streaming_chunks_pass_through() { + let chunks: Vec<&'static str> = vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\n", + "data: [DONE]\n\n", + ]; + let worker = crate::common::mock_worker::MockWorker::start(chunks.clone()).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": true + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + assert_eq!( + res.headers().get("content-type").unwrap().to_str().unwrap(), + "text/event-stream" + ); + + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let data = crate::common::streaming::parse_sse_data(&bytes); + assert_eq!(data.len(), 3); + assert!(data[0].contains("\"Hel\"")); + assert!(data[1].contains("\"lo\"")); + assert_eq!(data[2], "[DONE]"); +} + +#[tokio::test] +async fn streaming_first_chunk_before_completion() { + let chunks: Vec<&'static str> = vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"first\"}}]}\n\n", + "data: [DONE]\n\n", + ]; + let worker = crate::common::mock_worker::MockWorker::start(chunks).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": true + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + + // Asserting first-byte timing under axum::Body::from_stream requires + // poll-by-poll instrumentation; here we only sanity-check that the body + // collects at all so that a regression that buffers the entire stream + // before yielding will at minimum still pass through bytes. + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + assert!(bytes.windows(5).any(|w| w == b"first")); +} + +#[tokio::test] +async fn concurrent_streams_are_isolated() { + let chunks_a: Vec<&'static str> = vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"AAA\"}}]}\n\n", + "data: [DONE]\n\n", + ]; + let chunks_b: Vec<&'static str> = vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"BBB\"}}]}\n\n", + "data: [DONE]\n\n", + ]; + let worker_a = crate::common::mock_worker::MockWorker::start(chunks_a).await; + let worker_b = crate::common::mock_worker::MockWorker::start(chunks_b).await; + + let ctx_a = build_ctx_with_worker(&worker_a.url); + let ctx_b = build_ctx_with_worker(&worker_b.url); + let app_a = build_router(ctx_a); + let app_b = build_router(ctx_b); + + let req = |stream| { + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": stream + })) + .unwrap(), + )) + .unwrap() + }; + + let (ra, rb) = tokio::join!(app_a.oneshot(req(true)), app_b.oneshot(req(true)),); + let body_a = ra.unwrap().into_body().collect().await.unwrap().to_bytes(); + let body_b = rb.unwrap().into_body().collect().await.unwrap().to_bytes(); + assert!(body_a.windows(3).any(|w| w == b"AAA")); + assert!(body_b.windows(3).any(|w| w == b"BBB")); + assert!(!body_a.windows(3).any(|w| w == b"BBB")); + assert!(!body_b.windows(3).any(|w| w == b"AAA")); +} + +#[tokio::test] +async fn streaming_upstream_5xx_preserves_content_type() { + let worker = crate::common::mock_worker::MockWorker::start_returning_error( + StatusCode::INTERNAL_SERVER_ERROR, + serde_json::json!({"error": {"type": "upstream", "message": "boom"}}), + ) + .await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": true, + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + res.headers().get("content-type").unwrap().to_str().unwrap(), + "application/json", + "router must preserve upstream content-type on error, not force text/event-stream" + ); +} + +#[tokio::test] +async fn non_streaming_upstream_429_preserved() { + // Regression: a legitimate worker 4xx (rate limit, invalid model, etc.) + // must be proxied verbatim. The router is only a 502-wrapper for + // transport failures (connect/dns/timeout); upstream-application errors + // are OpenAI-compatible passthrough. + let upstream_body = serde_json::json!({ + "error": { + "type": "rate_limit_error", + "message": "Too many requests", + "code": "rate_limit_exceeded" + } + }); + let worker = crate::common::mock_worker::MockWorker::start_returning_error( + StatusCode::TOO_MANY_REQUESTS, + upstream_body.clone(), + ) + .await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!( + res.status(), + StatusCode::TOO_MANY_REQUESTS, + "non-streaming upstream 4xx must be proxied verbatim", + ); + assert_eq!( + res.headers().get("content-type").unwrap().to_str().unwrap(), + "application/json", + ); + // Router envelope code header must NOT be set — this is upstream's response. + assert!( + res.headers().get("x-router-error-code").is_none(), + "router envelope header must NOT be set on upstream-passthrough responses", + ); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let got: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(got, upstream_body, "body bytes must round-trip unchanged"); +} + +#[tokio::test] +async fn non_streaming_upstream_500_preserved() { + // Regression: worker-side 5xx (model crashed, OOM, etc.) is proxied + // verbatim on non-streaming requests. Mirrors streaming behaviour. Only + // transport failures get 502-wrapped. + let upstream_body = serde_json::json!({ + "error": {"type": "server_error", "message": "internal worker failure"} + }); + let worker = crate::common::mock_worker::MockWorker::start_returning_error( + StatusCode::INTERNAL_SERVER_ERROR, + upstream_body.clone(), + ) + .await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!( + res.headers().get("x-router-error-code").is_none(), + "router envelope must NOT wrap upstream 5xx — passthrough", + ); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let got: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(got, upstream_body); +} + +#[tokio::test] +async fn non_streaming_upstream_4xx_body_passthrough() { + // Regression: the worker's response bytes must reach the client + // unmodified — no router envelope wrap, no field rewriting. + // + // We register `tiny` as the model so the handler resolves it against + // the registry, then have the worker simulate a 4xx — this test is + // about *upstream-returned* errors passing through, not about a + // router-side model-not-found error. + let upstream_body = serde_json::json!({ + "error": {"type": "invalid_request_error", "message": "bad input"} + }); + let worker = crate::common::mock_worker::MockWorker::start_returning_error( + StatusCode::BAD_REQUEST, + upstream_body.clone(), + ) + .await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + // Byte-exact passthrough — compare via Value to be insensitive to + // whitespace, which is the only legal axis of variation for JSON. + let got: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(got, upstream_body); +} + +#[tokio::test] +async fn oversized_request_body_returns_413() { + // Regression: the router must enforce a body-size cap on + // `/v1/chat/completions`. A multi-MiB body from a hostile client must be + // rejected at the layer BEFORE the handler reads it into memory, and + // must NOT be forwarded to the upstream worker. + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + // 2 MiB body — the configured limit is 1 MiB. + let big = vec![b'x'; 2 * 1024 * 1024]; + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from(big)) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!( + res.status(), + StatusCode::PAYLOAD_TOO_LARGE, + "oversized body must be rejected with 413; got: {}", + res.status(), + ); + // The worker must NOT have received the oversized payload. + let captured = worker.captured.lock().unwrap(); + assert!( + captured.last_body.is_none(), + "router must not forward oversized body to upstream; got body of {} bytes", + captured.last_body.as_ref().map(|b| b.len()).unwrap_or(0), + ); +} + +#[tokio::test] +async fn chat_rejects_null_body_400() { + // Regression: a JSON `null` body is syntactically valid JSON but is NOT + // a chat-completions request shape. The router must reject it with 400 + // BadRequest and NOT forward it to the worker. + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from("null")) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "bad_request" + ); + let captured = worker.captured.lock().unwrap(); + assert!( + captured.last_body.is_none(), + "router must NOT forward `null` body to worker; got: {:?}", + captured.last_body, + ); +} + +#[tokio::test] +async fn chat_rejects_array_body_400() { + // Regression: a JSON array `[]` body is not a chat-completions request + // shape (object expected). Router must 400 and not forward. + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from("[]")) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "bad_request" + ); + let captured = worker.captured.lock().unwrap(); + assert!(captured.last_body.is_none()); +} + +#[tokio::test] +async fn chat_rejects_string_body_400() { + // Regression: a JSON string `"hi"` is not a chat-completions request + // shape. Router must 400 and not forward. + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from("\"hi\"")) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "bad_request" + ); + let captured = worker.captured.lock().unwrap(); + assert!(captured.last_body.is_none()); +} + +#[tokio::test] +async fn non_streaming_mid_body_drop_classified_as_upstream_status() { + // Regression: when the upstream replies with a status line and headers + // but drops the connection mid-body, the failure is NOT + // "upstream_unreachable" (the upstream demonstrably DID reply). It must + // be classified as `upstream_status` so the operator-visible envelope + // reflects that the worker partially served the request. + let worker = crate::common::mock_worker::MockWorker::start_returning_partial_body( + StatusCode::OK, + b"{\"partial\": ", + ) + .await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": false, + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!( + res.status(), + StatusCode::BAD_GATEWAY, + "mid-body drop must surface as 502", + ); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "upstream_status", + "mid-body drop must be upstream_status (worker DID reply), not upstream_unreachable", + ); +} + +#[tokio::test] +async fn malformed_json_returns_400_bad_request() { + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from("{not json}")) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "bad_request" + ); + // Worker must NOT have received a body for this request. + let captured = worker.captured.lock().unwrap(); + assert!( + captured.last_body.is_none(), + "router must not forward malformed JSON to upstream worker; got body: {:?}", + captured.last_body + ); +} + +#[tokio::test] +async fn no_healthy_workers_returns_503() { + // Build a context with an empty registry for model "tiny" — no workers. + let cfg = config_for("http://unused"); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); // empty — no workers added + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap()); + let ctx = Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "no_healthy_workers" + ); +} + +/// A worker is registered for a model that is NOT in `cfg.models` (so the +/// policy registry has no entry for it). The handler returns 404 +/// `model_not_found` rather than 500 — clients can recover by sending a +/// different model name; an internal_error would mask the misconfiguration. +#[tokio::test] +async fn unknown_model_with_no_policy_returns_404_model_not_found() { + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let cfg = config_for(&worker.url); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + // Register a worker that claims to serve "ghost-7b" — a model the + // policy registry knows nothing about. + let _ = registry.add(WorkerSpec { + id: WorkerId("w-ghost".into()), + url: worker.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("ghost-7b".into())], + bootstrap_port: None, + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap()); + let ctx = Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "ghost-7b", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::NOT_FOUND); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "model_not_found", + ); +} + +#[tokio::test] +async fn forward_json_to_records_failure_on_body_drop() { + // Regression: previously `forward_json_to` recorded breaker + // success/failure right after headers — so a worker that returned + // 200 OK and then dropped the body got credited as healthy. A worker + // that does this repeatedly stays eligible. The fix moves the + // breaker record to after the body completes, treating a body-drop + // as failure. + use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use sgl_router::server::error::ApiError; + use std::sync::Arc; + use std::time::Duration; + + let worker = crate::common::mock_worker::MockWorker::start_returning_partial_body( + StatusCode::OK, + b"{\"par", + ) + .await; + + let proxy = Proxy::new(Duration::from_secs(5)).unwrap(); + let breaker = Arc::new(CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(1).unwrap(), + cool_down: Duration::from_secs(30), + })); + + let headers = axum::http::HeaderMap::new(); + let body = bytes::Bytes::from(b"{}".to_vec()); + let res: Result<_, ApiError> = proxy + .forward_json_to( + &worker.url, + &breaker, + "/v1/chat/completions", + &headers, + body, + ) + .await; + assert!(res.is_err(), "body drop should surface as ApiError"); + assert!( + !breaker.would_allow(), + "body drop must trip the breaker (threshold=1)" + ); +} + +#[tokio::test] +async fn forward_json_to_records_success_only_after_body_completes() { + // Counterpart of the body-drop regression: clean 2xx + clean body + // MUST call `record_success` on the breaker, even if there were + // prior failures. Without this, the breaker can never recover from + // a transient failure spike — it would open on the threshold-th + // failure and stay open until cool_down, ignoring any successful + // traffic in between. + // + // An earlier version of this test only asserted `breaker.would_allow()` + // after a single clean call against a fresh breaker, which is true + // by default — the test never actually observed the success path + // affecting breaker state. We instead seed one prior failure (one + // short of threshold), make a clean call, then induce one more + // failure. If `record_success` fired on the clean call, the failure + // count is back to 1 and the breaker stays closed. If it didn't, + // the count is now 2 and the breaker opens. + use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use sgl_router::server::error::ApiError; + use std::sync::Arc; + use std::time::Duration; + + let ok_worker = crate::common::mock_worker::MockWorker::start_returning_error( + StatusCode::OK, + serde_json::json!({}), + ) + .await; + let proxy = Proxy::new(Duration::from_secs(5)).unwrap(); + let breaker = Arc::new(CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(2).unwrap(), + cool_down: Duration::from_secs(30), + })); + // Seed one prior failure (threshold-1) — breaker still admits. + breaker.record_failure(); + assert!( + breaker.would_allow(), + "one failure under threshold=2 keeps the breaker closed (sanity)", + ); + + let headers = axum::http::HeaderMap::new(); + let res: Result<_, ApiError> = proxy + .forward_json_to( + &ok_worker.url, + &breaker, + "/v1/chat/completions", + &headers, + bytes::Bytes::from_static(b"{}"), + ) + .await; + assert!(res.is_ok(), "clean OK call must succeed: {res:?}"); + + // The observable side-effect of `record_success` on the OK body + // path: failure count is reset to 0. One more failure now must + // leave us at 1 (not 2), so the breaker stays closed. + breaker.record_failure(); + assert!( + breaker.would_allow(), + "clean success on the OK body path must reset the failure count — \ + if `record_success` was never called, the seed failure would still \ + be live and this single new failure would trip threshold=2", + ); +} + +#[tokio::test] +async fn forward_streaming_to_records_failure_on_mid_stream_drop() { + // Streaming counterpart of the body-drop regression. Headers say 200 + // OK, then the worker drops mid-body. The breaker must observe this + // as a failure — `bytes_stream_to_body` reads the rest of the + // stream on a spawned pump, so the recording has to flow through + // that pump's completion path. + use http_body_util::BodyExt; + use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use sgl_router::server::error::ApiError; + use std::sync::Arc; + use std::time::Duration; + + let worker = crate::common::mock_worker::MockWorker::start_returning_partial_body( + StatusCode::OK, + b"data: hi\n\n", + ) + .await; + + let proxy = Proxy::new(Duration::from_secs(5)).unwrap(); + let breaker = Arc::new(CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(1).unwrap(), + cool_down: Duration::from_secs(30), + })); + + let headers = axum::http::HeaderMap::new(); + let body = bytes::Bytes::from(b"{}".to_vec()); + let res: Result<_, ApiError> = proxy + .forward_streaming_to( + &worker.url, + &breaker, + "/v1/chat/completions", + &headers, + body, + None, + ) + .await; + + let resp = res.expect("headers are 200 OK; transport-level Ok"); + // Drain the body — the pump will see the mid-flight drop and + // surface an error chunk, then close. + let _ = resp.into_body().collect().await; + // After the stream drains, the breaker MUST have recorded failure. + // Poll briefly because the pump runs on a spawned task. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while breaker.would_allow() && std::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !breaker.would_allow(), + "stream drop must trip the breaker (threshold=1)" + ); +} + +#[tokio::test] +async fn forward_json_to_records_failure_on_5xx() { + use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use sgl_router::server::error::ApiError; + use std::sync::Arc; + use std::time::Duration; + + let worker = crate::common::mock_worker::MockWorker::start_returning_error( + StatusCode::INTERNAL_SERVER_ERROR, + serde_json::json!({"error": {"type": "x"}}), + ) + .await; + + let proxy = Proxy::new(Duration::from_secs(5)).unwrap(); + let breaker = Arc::new(CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(1).unwrap(), + cool_down: Duration::from_secs(30), + })); + + let headers = axum::http::HeaderMap::new(); + let body = bytes::Bytes::from(b"{}".to_vec()); + let _: Result<_, ApiError> = proxy + .forward_json_to( + &worker.url, + &breaker, + "/v1/chat/completions", + &headers, + body, + ) + .await; + + assert!( + !breaker.allow(), + "one 5xx with threshold=1 should open the breaker" + ); +} + +#[tokio::test] +async fn forward_json_to_rejects_when_breaker_open() { + use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use sgl_router::server::error::ApiError; + use std::sync::Arc; + use std::time::Duration; + + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let proxy = Proxy::new(Duration::from_secs(5)).unwrap(); + let breaker = Arc::new(CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(1).unwrap(), + cool_down: Duration::from_secs(30), + })); + breaker.record_failure(); // open immediately + + let headers = axum::http::HeaderMap::new(); + let body = bytes::Bytes::from(b"{}".to_vec()); + let res = proxy + .forward_json_to( + &worker.url, + &breaker, + "/v1/chat/completions", + &headers, + body, + ) + .await; + + let err = res.expect_err("breaker open → ApiError"); + match err { + ApiError::BreakerOpen { .. } => {} + other => panic!("expected BreakerOpen, got {other:?}"), + } +} + +/// A malformed worker URL (operator typo in `discovery.static_urls`, broken k8s +/// annotation) must surface as 503 `worker_misconfigured` (not 500 +/// `internal_error`) AND trip the worker's circuit breaker so the malformed +/// worker drops out of `healthy_workers_for` and subsequent requests skip +/// it. +#[tokio::test] +async fn forward_json_to_malformed_url_returns_worker_misconfigured_and_trips_breaker() { + use sgl_router::health::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; + use sgl_router::server::error::ApiError; + use std::sync::Arc; + use std::time::Duration; + + let proxy = Proxy::new(Duration::from_secs(5)).unwrap(); + let breaker = Arc::new(CircuitBreaker::with_config(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(1).unwrap(), + cool_down: Duration::from_secs(30), + })); + + let headers = axum::http::HeaderMap::new(); + let body = bytes::Bytes::from(b"{}".to_vec()); + let res = proxy + .forward_json_to( + "not-a-url", + &breaker, + "/v1/chat/completions", + &headers, + body, + ) + .await; + + let err = res.expect_err("malformed URL → ApiError"); + match &err { + ApiError::WorkerMisconfigured { worker, .. } => { + assert_eq!(worker, "not-a-url", "{err:?}"); + } + other => panic!("expected WorkerMisconfigured, got {other:?}"), + } + assert!( + !breaker.allow(), + "WorkerMisconfigured must trip the breaker so the worker drops out of selection", + ); +} + +/// Regression test: LoadGuard must be held for the *body* lifetime of a +/// streaming response, not just for the handler lifetime. +/// +/// Before the fix, the handler dropped `_guard` as soon as it returned +/// (which happens when headers arrive), so `active_load()` was 0 while +/// the SSE pump was still relaying bytes. This test catches that bug. +#[tokio::test] +async fn streaming_load_guard_persists_for_body_lifetime() { + let chunks: Vec<&'static str> = vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"c\"}}]}\n\n", + "data: [DONE]\n\n", + ]; + // Each chunk is delayed by 50ms, total ~200ms of streaming. + let worker = crate::common::mock_worker::MockWorker::start_slow_stream( + chunks, + Duration::from_millis(50), + ) + .await; + + let cfg = config_for(&worker.url); + let registry = Arc::new(WorkerRegistry::default()); + let _ = registry.add(WorkerSpec { + id: WorkerId("w1".into()), + url: worker.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap()); + let ctx = Arc::new(AppContext::new( + cfg, + tokenizers, + proxy, + registry.clone(), + policies, + )); + let app = build_router(ctx); + + // Grab the Worker handle so we can assert active_load(). + let w_handle: Arc = registry + .workers_for(&ModelId("tiny".into())) + .into_iter() + .next() + .expect("worker registered"); + + let body = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": true, + })) + .unwrap(); + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(); + + let res = app.oneshot(req).await.unwrap(); + + // The handler has returned (headers arrived). Wait a moment for the + // first chunk's delay to pass, then assert load is still held. + tokio::time::sleep(Duration::from_millis(20)).await; + assert!( + w_handle.active_load() >= 1, + "load should be >= 1 mid-stream, got {}", + w_handle.active_load() + ); + + // Drain the entire body — this drives the SSE pump to completion. + let _bytes = BodyExt::collect(res.into_body()).await.unwrap().to_bytes(); + + // After the body is fully consumed and dropped, the guard must be + // released. Give the spawned task a brief moment to clean up. + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + w_handle.active_load(), + 0, + "load should be 0 after stream completes" + ); +} + +/// Task A: the chat handler mints an `ActiveLoadGuard` from the shared +/// `ActiveLoadRegistry` and drops it when the request completes. The +/// non-streaming path drops the guard on handler exit; this test +/// asserts the round-trip increment → 0 across a single request. +#[tokio::test] +async fn non_streaming_active_load_increments_then_returns_to_zero() { + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let active_load = Arc::clone(&ctx.active_load); + let app = build_router(ctx); + + assert_eq!( + active_load.inflight_count(), + 0, + "registry must start with no in-flight requests", + ); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": false, + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + // Drain the body so any pending background work runs to completion. + let _ = res.into_body().collect().await.unwrap().to_bytes(); + + // The handler has returned, so the active-load guard must have + // dropped — counters are back to zero. + assert_eq!( + active_load.inflight_count(), + 0, + "active-load registry must be empty after non-streaming handler returns", + ); + let w_id = WorkerId("w1".into()); + assert_eq!( + active_load.prefill_load(&w_id), + 0, + "prefill_load must decrement on response end", + ); +} + +/// Task A: the streaming path holds the `ActiveLoadGuard` until the +/// SSE pump finishes. Mid-stream the registry shows `inflight_count >= 1`; +/// after the body drains it returns to 0. Counterpart to +/// `streaming_load_guard_persists_for_body_lifetime` — both guards must +/// live for the FULL response lifetime. +#[tokio::test] +async fn streaming_active_load_persists_for_body_lifetime() { + let chunks: Vec<&'static str> = vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"c\"}}]}\n\n", + "data: [DONE]\n\n", + ]; + let worker = crate::common::mock_worker::MockWorker::start_slow_stream( + chunks, + Duration::from_millis(50), + ) + .await; + + let cfg = config_for(&worker.url); + let registry = Arc::new(WorkerRegistry::default()); + let _ = registry.add(WorkerSpec { + id: WorkerId("w1".into()), + url: worker.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap()); + let ctx = Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)); + let active_load = Arc::clone(&ctx.active_load); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": true, + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + + // The handler has returned (headers arrived). The streaming pump is + // still running, so the registry's per-request entry must remain. + tokio::time::sleep(Duration::from_millis(20)).await; + assert!( + active_load.inflight_count() >= 1, + "registry inflight must be >= 1 mid-stream, got {}", + active_load.inflight_count(), + ); + let w_id = WorkerId("w1".into()); + assert!( + active_load.prefill_load(&w_id) >= 1, + "prefill_load must be > 0 mid-stream, got {}", + active_load.prefill_load(&w_id), + ); + + // Drain the body — drives the SSE pump to completion. + let _ = res.into_body().collect().await.unwrap().to_bytes(); + tokio::time::sleep(Duration::from_millis(20)).await; + + assert_eq!( + active_load.inflight_count(), + 0, + "registry must be empty after stream drains", + ); + assert_eq!( + active_load.prefill_load(&w_id), + 0, + "prefill_load must be 0 after stream drains", + ); +} + +/// Task A: a streaming client that disconnects mid-stream still drops +/// both guards. The SSE pump's `tx.send().await.is_err()` branch is what +/// triggers the drop — when the axum Body is dropped on the client side, +/// the channel receiver closes and the pump exits. +#[tokio::test] +async fn streaming_active_load_drops_on_client_disconnect() { + // Slow stream: 4 chunks × 100 ms each. The test only reads the + // first chunk then drops the body, simulating a client disconnect. + let chunks: Vec<&'static str> = vec![ + "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"c\"}}]}\n\n", + "data: [DONE]\n\n", + ]; + let worker = crate::common::mock_worker::MockWorker::start_slow_stream( + chunks, + Duration::from_millis(100), + ) + .await; + let ctx = build_ctx_with_worker(&worker.url); + let active_load = Arc::clone(&ctx.active_load); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": true, + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + + // Read one chunk to confirm the stream is live, then drop the body. + use futures::StreamExt; + let mut data_stream = res.into_body().into_data_stream(); + let _first = data_stream.next().await; + drop(data_stream); + + // Wait long enough for the SSE pump to notice the receiver-drop and + // exit (per `bytes_stream_to_body_breaks_on_client_disconnect` test + // in sse.rs, that takes well under 200 ms). + tokio::time::sleep(Duration::from_millis(300)).await; + + assert_eq!( + active_load.inflight_count(), + 0, + "client disconnect must drop the streaming pump's guards within one tick", + ); +} + +/// Task D: stale-request janitor expiry surfaces as HTTP 504 with +/// `x-router-error-code: stale_request_expired`. The chat handler +/// races the upstream fetch against the janitor's per-request +/// cancellation token; when the token wins, the handler returns +/// `ApiError::StaleRequestExpired`. +/// +/// Wiring: build an `AppContext` with a short +/// `stale_request_timeout` `ActiveLoadRegistry` + spawn a janitor +/// with sub-second cadence + dispatch to a slow upstream that takes +/// longer than the timeout. The janitor sweeps before the upstream +/// returns; cancellation fires; handler returns 504. +#[tokio::test] +async fn janitor_expiry_returns_504_stale_request_expired() { + use sgl_router::policies::active_load::{spawn_janitor, ActiveLoadRegistry}; + // Upstream that takes 2s to respond — longer than our 50ms + // stale_request_timeout. + let worker = + crate::common::mock_worker::MockWorker::start_hanging(Duration::from_secs(2)).await; + + let cfg = config_for(&worker.url); + let registry = Arc::new(WorkerRegistry::default()); + let _ = registry.add(WorkerSpec { + id: WorkerId("w1".into()), + url: worker.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap()); + // Aggressive 50ms timeout: the janitor will sweep on the next + // tick (every 20ms) and fire the cancellation token before the + // upstream returns. + let active_load = ActiveLoadRegistry::new( + Arc::new(sgl_router::policies::active_load::SystemTimeClock), + Duration::from_millis(50), + ); + let _janitor = spawn_janitor(Arc::clone(&active_load), Duration::from_millis(20)); + let ctx = Arc::new(AppContext::with_active_load( + cfg, + tokenizers, + proxy, + registry, + policies, + active_load, + )); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": false, + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!( + res.status(), + StatusCode::GATEWAY_TIMEOUT, + "stale-request expiry must surface as 504", + ); + assert_eq!( + res.headers() + .get("x-router-error-code") + .and_then(|v| v.to_str().ok()), + Some("stale_request_expired"), + "504 response must carry x-router-error-code: stale_request_expired", + ); + let body = res.into_body().collect().await.unwrap().to_bytes(); + let body_str = String::from_utf8_lossy(&body); + assert!( + body_str.contains("\"code\":\"stale_request_expired\""), + "504 body must encode the same code in the JSON envelope: {body_str}", + ); +} + +/// Task A: a non-streaming request that errors out (upstream +/// unreachable) still drops the active-load guard. The handler's normal +/// return path is the only drop point — confirming the guard is on the +/// stack (not inside a long-lived future) is what this test pins. +#[tokio::test] +async fn non_streaming_error_path_drops_active_load_guard() { + // Dead upstream — first connect attempt fails fast. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dead_url = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + + let ctx = build_ctx_with_worker(&dead_url); + let active_load = Arc::clone(&ctx.active_load); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap(); + let res = app.oneshot(req).await.unwrap(); + assert_eq!(res.status(), StatusCode::BAD_GATEWAY); + + // Drain so any drop-on-body-end work runs. + let _ = res.into_body().collect().await.unwrap().to_bytes(); + assert_eq!( + active_load.inflight_count(), + 0, + "error path must drop the active-load guard", + ); +} diff --git a/experimental/sgl-router/tests/proxy/common/mock_worker.rs b/experimental/sgl-router/tests/proxy/common/mock_worker.rs new file mode 100644 index 000000000000..181937c29eb5 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/common/mock_worker.rs @@ -0,0 +1,437 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal axum mock of an SGLang HTTP worker for routing tests. + +use axum::body::Body; +use axum::extract::State; +use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::Json; +use bytes::Bytes; +use serde_json::Value; +use std::collections::{HashMap, HashSet}; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::oneshot; + +/// Headers captured from the most recent inbound request. +#[derive(Default)] +pub struct CapturedHeaders { + pub seen: HashSet, // names (kept for backwards compat) + pub headers: HashMap, // name -> value (last write wins) + pub last_body: Option, +} + +#[derive(Clone)] +#[allow(dead_code)] // Only used by some test files; mock_worker is shared. +pub struct MockWorkerState { + pub captured: Arc>, + pub stream_chunks: Arc>, +} + +/// A running mock SGLang worker. Shuts down on Drop via the oneshot sender. +pub struct MockWorker { + pub url: String, + // Used in header_forwarding_test; not every test file reads captured headers. + #[allow(dead_code)] + pub captured: Arc>, + _shutdown: oneshot::Sender<()>, +} + +impl MockWorker { + /// Bind to a random port on 127.0.0.1 and start serving. + /// + /// `stream_chunks` are the raw SSE bytes returned when a streaming + /// chat-completion request arrives. + #[allow(dead_code)] // Only used by some test files. + pub async fn start(stream_chunks: Vec<&'static str>) -> Self { + let captured = Arc::new(Mutex::new(CapturedHeaders::default())); + let state = MockWorkerState { + captured: captured.clone(), + stream_chunks: Arc::new(stream_chunks), + }; + // /server_info advertises served_model_name="tiny" so the + // worker-manager introspect step resolves model_ids for the + // "tiny" model the tests register a tokenizer + policy under. + let app = axum::Router::new() + .route("/v1/chat/completions", post(chat)) + .route("/server_info", get(serve_tiny_server_info)) + .with_state(state); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = rx.await; + }) + .await + .unwrap(); + }); + Self { + url, + captured, + _shutdown: tx, + } + } + + /// Bind to a random port and start a worker that accepts the request, + /// sleeps for `delay`, then returns `200 OK` with an empty JSON object. + /// Used to test router behaviour when the upstream wedges after accepting + /// the TCP connection but before sending response headers. + #[allow(dead_code)] + pub async fn start_hanging(delay: Duration) -> Self { + let captured = Arc::new(Mutex::new(CapturedHeaders::default())); + + #[derive(Clone)] + struct HangState { + captured: Arc>, + delay: Duration, + } + + async fn hang_handler( + State(s): State, + headers: HeaderMap, + body: Bytes, + ) -> Response { + { + let mut g = s.captured.lock().unwrap(); + g.last_body = Some(body.clone()); + for (k, v) in headers.iter() { + g.seen.insert(k.as_str().to_string()); + if let Ok(val) = v.to_str() { + g.headers.insert(k.as_str().to_string(), val.to_string()); + } + } + } + tokio::time::sleep(s.delay).await; + let mut r = Response::new(Body::from("{}")); + *r.status_mut() = StatusCode::OK; + r.headers_mut().insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/json"), + ); + r + } + + let state = HangState { + captured: captured.clone(), + delay, + }; + let app = axum::Router::new() + .route("/v1/chat/completions", post(hang_handler)) + .route("/server_info", get(serve_tiny_server_info)) + .with_state(state); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = rx.await; + }) + .await + .unwrap(); + }); + Self { + url, + captured, + _shutdown: tx, + } + } + + /// Bind to a random port and start a worker that streams `chunks` with a + /// fixed `delay` between each chunk. Used to test that load guards survive + /// the full body lifetime for streaming responses. + #[allow(dead_code)] + pub async fn start_slow_stream(chunks: Vec<&'static str>, delay: Duration) -> Self { + let captured = Arc::new(Mutex::new(CapturedHeaders::default())); + + #[derive(Clone)] + struct SlowState { + captured: Arc>, + chunks: Arc>, + delay: Duration, + } + + async fn slow_chat( + State(s): State, + headers: HeaderMap, + body: Bytes, + ) -> Response { + { + let mut g = s.captured.lock().unwrap(); + g.last_body = Some(body.clone()); + for (k, v) in headers.iter() { + g.seen.insert(k.as_str().to_string()); + if let Ok(val) = v.to_str() { + g.headers.insert(k.as_str().to_string(), val.to_string()); + } + } + } + let chunks = s.chunks.clone(); + let delay = s.delay; + // Stream chunks via a channel, sleeping between each send. + let (tx, rx) = tokio::sync::mpsc::channel::>(4); + tokio::spawn(async move { + for chunk in chunks.iter() { + tokio::time::sleep(delay).await; + if tx.send(Ok(Bytes::from(*chunk))).await.is_err() { + break; + } + } + }); + let body = Body::from_stream(tokio_stream::wrappers::ReceiverStream::new(rx)); + let mut r = Response::new(body); + *r.status_mut() = StatusCode::OK; + r.headers_mut().insert( + HeaderName::from_static("content-type"), + "text/event-stream".parse().unwrap(), + ); + r + } + + let state = SlowState { + captured: captured.clone(), + chunks: Arc::new(chunks), + delay, + }; + let app = axum::Router::new() + .route("/v1/chat/completions", post(slow_chat)) + .route("/server_info", get(serve_tiny_server_info)) + .with_state(state); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = rx.await; + }) + .await + .unwrap(); + }); + Self { + url, + captured, + _shutdown: tx, + } + } + + /// Bind to a raw TCP listener and start a worker that writes a status + /// line + headers with a large declared `Content-Length`, then writes + /// only `partial_body_bytes` of body before closing the connection. + /// + /// Used to test router behaviour when the upstream replies with a status + /// but drops the connection mid-body. We can't build this with axum + /// directly (it owns the response lifecycle); raw TCP gives us frame-level + /// control to short-write the body and close. + /// + /// NOTE: unlike the axum-based variants, this helper does NOT serve + /// `/server_info` (one-shot raw-TCP accept, no path routing). Callers + /// that wire this through `spawn_discovery` will see introspect fail + /// with empty `model_ids`. All current callers inject the worker via + /// `registry.add()` directly, which bypasses introspect. + #[allow(dead_code)] + pub async fn start_returning_partial_body( + status: StatusCode, + partial_body_bytes: &'static [u8], + ) -> Self { + let captured = Arc::new(Mutex::new(CapturedHeaders::default())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + let (tx, mut rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + // Accept one connection (or exit on shutdown). + tokio::select! { + _ = &mut rx => (), + accept = listener.accept() => { + let (mut sock, _) = match accept { + Ok(v) => v, + Err(_) => return, + }; + // Drain the request bytes until we see end-of-headers + // (`\r\n\r\n`). We deliberately do NOT fully consume the + // request body — the router has already sent it before + // awaiting our response, and we want to write the + // truncated response promptly. + let mut buf = [0u8; 4096]; + let mut acc: Vec = Vec::new(); + while !acc.windows(4).any(|w| w == b"\r\n\r\n") { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => return, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if acc.len() > 64 * 1024 { + // Defensive: don't loop forever if the request + // never produces a header terminator. + break; + } + } + // Write a response with a Content-Length larger than the + // bytes we will actually write, then drop the socket + // before the body completes. + let declared_len = partial_body_bytes.len() + 1024; + let head = format!( + "HTTP/1.1 {status_u16} {phrase}\r\n\ + content-type: application/json\r\n\ + content-length: {declared_len}\r\n\ + connection: close\r\n\ + \r\n", + status_u16 = status.as_u16(), + phrase = status.canonical_reason().unwrap_or("OK"), + ); + if sock.write_all(head.as_bytes()).await.is_err() { + return; + } + if sock.write_all(partial_body_bytes).await.is_err() { + return; + } + // Flush, then drop — the client should see content-length + // mismatch as a transport-level body read failure. + let _ = sock.flush().await; + drop(sock); + } + } + }); + Self { + url, + captured, + _shutdown: tx, + } + } + + /// Bind to a random port and start a worker that ALWAYS returns the given + /// HTTP status code and JSON body with `Content-Type: application/json`. + /// Used to test router behaviour when the upstream returns an error. + #[allow(dead_code)] + pub async fn start_returning_error(status: StatusCode, body: Value) -> Self { + let captured = Arc::new(Mutex::new(CapturedHeaders::default())); + let body_arc = Arc::new(body.to_string()); + + #[derive(Clone)] + struct ErrorState { + captured: Arc>, + body_str: Arc, + status: StatusCode, + } + + async fn error_handler( + State(s): State, + headers: HeaderMap, + body: Bytes, + ) -> Response { + { + let mut g = s.captured.lock().unwrap(); + g.last_body = Some(body); + for (k, v) in headers.iter() { + g.seen.insert(k.as_str().to_string()); + if let Ok(val) = v.to_str() { + g.headers.insert(k.as_str().to_string(), val.to_string()); + } + } + } + let mut r = Response::new(Body::from(s.body_str.as_ref().clone())); + *r.status_mut() = s.status; + r.headers_mut().insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/json"), + ); + r + } + + let state = ErrorState { + captured: captured.clone(), + body_str: body_arc, + status, + }; + let app = axum::Router::new() + .route("/v1/chat/completions", post(error_handler)) + .route("/server_info", get(serve_tiny_server_info)) + .with_state(state); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr: SocketAddr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + let (tx, rx) = oneshot::channel::<()>(); + tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = rx.await; + }) + .await + .unwrap(); + }); + Self { + url, + captured, + _shutdown: tx, + } + } +} + +/// Stateless `/server_info` handler shared by every axum-based +/// `MockWorker::start_*` variant. Advertising `served_model_name="tiny"` +/// lets the worker manager's introspect step resolve `model_ids` for any +/// variant that flows through `spawn_discovery`, instead of burning 3 × +/// `SERVER_INFO_TIMEOUT` of retries before registering with empty +/// `model_ids`. Adding it unconditionally is cheaper than tracking which +/// variants do or don't get introspected. +#[allow(dead_code)] // shared across all axum variants +async fn serve_tiny_server_info() -> Json { + Json(serde_json::json!({"served_model_name": "tiny"})) +} + +#[allow(dead_code)] // Used by `MockWorker::start`, only some test files need it. +async fn chat(State(s): State, headers: HeaderMap, body: Bytes) -> Response { + { + let mut g = s.captured.lock().unwrap(); + g.last_body = Some(body.clone()); + for (k, v) in headers.iter() { + g.seen.insert(k.as_str().to_string()); + if let Ok(val) = v.to_str() { + g.headers.insert(k.as_str().to_string(), val.to_string()); + } + } + } + let v: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + let streaming = v.get("stream").and_then(|x| x.as_bool()).unwrap_or(false); + if streaming { + let chunks: Vec<_> = s + .stream_chunks + .iter() + .map(|c| Ok::<_, std::io::Error>(Bytes::from(*c))) + .collect(); + let body = Body::from_stream(futures::stream::iter(chunks)); + let mut r = Response::new(body); + *r.status_mut() = StatusCode::OK; + r.headers_mut().insert( + HeaderName::from_static("content-type"), + "text/event-stream".parse().unwrap(), + ); + return r; + } + let resp = serde_json::json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": v["model"].as_str().unwrap_or("unknown"), + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }] + }); + Json(resp).into_response() +} diff --git a/experimental/sgl-router/tests/proxy/common/mod.rs b/experimental/sgl-router/tests/proxy/common/mod.rs new file mode 100644 index 000000000000..9ad730c43e97 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/common/mod.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared test harness re-exports. + +pub mod mock_worker; +pub mod streaming; diff --git a/experimental/sgl-router/tests/proxy/common/streaming.rs b/experimental/sgl-router/tests/proxy/common/streaming.rs new file mode 100644 index 000000000000..2ac1e3bcf132 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/common/streaming.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! SSE parsing and body-collection helpers for integration tests. + +use bytes::Bytes; + +/// Parse an SSE stream's `data: …` payloads (one per event). +#[allow(dead_code)] +pub fn parse_sse_data(raw: &[u8]) -> Vec { + let s = std::str::from_utf8(raw).unwrap_or(""); + s.lines() + .filter_map(|l| l.strip_prefix("data: ")) + .map(|l| l.to_string()) + .collect() +} + +/// Collect an axum Body to bytes in tests. +#[allow(dead_code)] +pub async fn collect_body(body: axum::body::Body) -> Bytes { + use http_body_util::BodyExt; + body.collect().await.unwrap().to_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Ported from SMG tests/api/streaming_tests.rs::test_sse_format_parsing. + /// Verifies that parse_sse_data: + /// 1. Extracts standard `data: …` lines. + /// 2. Silently ignores SSE `event: …` type fields (not data lines). + /// 3. Silently ignores SSE `: …` comment lines. + /// 4. Correctly parses `[DONE]` sentinel. + /// + /// These edge-cases matter because SGLang workers may emit `event: message` + /// fields in their SSE frames. A parser that accidentally leaks those into + /// the payload list would cause clients to fail on JSON-parse. + #[test] + fn parse_sse_data_extracts_data_lines_only() { + // Basic: three data lines including the [DONE] sentinel. + let basic = + b"data: {\"text\":\"Hello\"}\n\ndata: {\"text\":\" world\"}\n\ndata: [DONE]\n\n"; + let events = parse_sse_data(basic); + assert_eq!(events.len(), 3, "expected 3 data events, got: {events:?}"); + assert_eq!(events[0], "{\"text\":\"Hello\"}"); + assert_eq!(events[1], "{\"text\":\" world\"}"); + assert_eq!(events[2], "[DONE]"); + + // Mixed: event: type field + comment line — neither must appear in output. + let mixed = b"event: message\ndata: {\"test\":true}\n\n: comment line\ndata: [DONE]\n\n"; + let events = parse_sse_data(mixed); + assert_eq!( + events.len(), + 2, + "event: and : comment lines must be ignored; got: {events:?}" + ); + assert_eq!(events[0], "{\"test\":true}"); + assert_eq!(events[1], "[DONE]"); + } +} diff --git a/experimental/sgl-router/tests/proxy/failover.rs b/experimental/sgl-router/tests/proxy/failover.rs new file mode 100644 index 000000000000..4e048497f7f1 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/failover.rs @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use axum::body::Body; +use axum::http::Request; +use sgl_router::config::*; +use sgl_router::discovery::{spawn_discovery, ModelId}; +use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry; +use sgl_router::proxy::Proxy; +use sgl_router::server::app::build_router; +use sgl_router::server::app_context::AppContext; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::manager; +use sgl_router::workers::WorkerRegistry; +use std::sync::Arc; +use std::time::Duration; +use tower::ServiceExt; + +#[tokio::test] +async fn failover_when_one_worker_dies() { + // Three mock workers. Each advertises served_model_name = "tiny" on + // /server_info, so the worker manager's introspect step resolves the + // registry's model_ids without us having to hand-declare them here. + let w1 = crate::common::mock_worker::MockWorker::start(vec![]).await; + let w2 = crate::common::mock_worker::MockWorker::start(vec![]).await; + let w3 = crate::common::mock_worker::MockWorker::start(vec![]).await; + + let cfg = Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: Default::default(), + models: vec![ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: Some(CircuitBreakerConfig { + threshold: std::num::NonZeroU32::new(1).unwrap(), // open after first failure + cool_down_secs: 30, + }), + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec![w1.url.clone(), w2.url.clone(), w3.url.clone()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + }; + + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + + let (event_rx, _disc) = spawn_discovery(&cfg).await.unwrap(); + let _mgr = tokio::spawn(manager::run_with_config( + event_rx, + registry.clone(), + Some(Arc::new(cfg.clone())), + None, + None, + )); + + // Poll for the registry to converge — `register_one` introspect is + // a per-task spawn (manager.rs:127), so order of registration is + // non-deterministic under load. Cap the wait so a real hang surfaces + // instead of becoming a flake. + let converged = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if registry.workers_for(&ModelId("tiny".into())).len() == 3 { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + converged.is_ok(), + "registry should contain all 3 workers after discovery; have {}", + registry.workers_for(&ModelId("tiny".into())).len() + ); + + let proxy = Arc::new(Proxy::new(Duration::from_secs(5)).unwrap()); + let ctx = Arc::new(AppContext::new( + cfg, + tokenizers, + proxy, + registry.clone(), + policies, + )); + ctx.mark_ready(); + let app = build_router(ctx); + + // Kill w2 by dropping its handle, then poll until its socket + // actually refuses connections. Without this, the first request + // routed to w2 can race against the listener's graceful shutdown + // and succeed, masking the failover assertion below. + let w2_url = w2.url.clone(); + drop(w2); + let host_port = w2_url.trim_start_matches("http://"); + let down = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if tokio::net::TcpStream::connect(host_port).await.is_err() { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!(down.is_ok(), "w2 socket never went down"); + + // Send 6 requests; round-robin would route 2 to w2 → connection refused → + // breaker opens (threshold=1); subsequent round-robin picks rotate among + // the 2 healthy workers (#1 and #3) because healthy_workers_for filters out w2. + let mut errs = 0usize; + let mut oks = 0usize; + for i in 0..6 { + let body = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": format!("hi {i}")}], + })) + .unwrap(); + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(); + let res = app.clone().oneshot(req).await.unwrap(); + if res.status().is_success() { + oks += 1; + } else { + errs += 1; + } + } + // We expect exactly 1 error — the first call routed to w2 fails and opens + // its breaker; subsequent round-robin picks rotate among the 2 healthy + // workers since registry.healthy_workers_for filters out the open breaker. + assert_eq!(errs, 1, "exactly the first w2 pick should error"); + assert_eq!(oks, 5, "remaining 5 picks should succeed via filtered RR"); +} diff --git a/experimental/sgl-router/tests/proxy/graceful_shutdown.rs b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs new file mode 100644 index 000000000000..6d87029c9f80 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs @@ -0,0 +1,230 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Pins the contract that `axum::serve(...).with_graceful_shutdown(...)` — +//! exactly as wired in `src/main.rs` — drains every in-flight streaming +//! request through the **real** `build_router(ctx)` stack before the +//! server future resolves. A k8s SIGTERM must not truncate streaming +//! completions. +//! +//! Why route the test through the real router (chat handler + proxy + +//! SSE pump) rather than a synthetic `Router::new().route(...)`: a +//! truncation regression could live in `forward_streaming_to`'s +//! `bytes_stream_to_body` completion hook, in `chat::chat_completions`' +//! guards, or in the SSE pump's `tx.send().await` race — all of which +//! would be silently skipped by a synthetic-handler test. + +use bytes::Bytes; +use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ModelConfig, ObservabilityConfig, + PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, +}; +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::factory::build_registry_with_defaults; +use sgl_router::proxy::Proxy; +use sgl_router::server::app::build_router; +use sgl_router::server::app_context::AppContext; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::WorkerRegistry; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +const TEST_TIMEOUT: Duration = Duration::from_secs(15); + +fn build_ctx_with_worker(worker_url: &str) -> Arc { + let cfg = Config { + server: ServerConfig { + host: "127.0.0.1".into(), + port: 0, + }, + observability: ObservabilityConfig::default(), + models: vec![ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + }; + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + registry + .add(WorkerSpec { + id: WorkerId("w1".into()), + url: worker_url.to_string(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }) + .expect("test worker accepted"); + let policies = Arc::new(build_registry_with_defaults(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(TEST_TIMEOUT).unwrap()); + let ctx = AppContext::new(cfg, tokenizers, proxy, registry, policies); + ctx.mark_ready(); + Arc::new(ctx) +} + +/// Streaming chat-completions body the worker hands back chunk-by-chunk. +/// One ~60 ms delay per chunk × 8 chunks ≈ ~480 ms per request, long +/// enough that we can race in ~100 concurrent clients and trigger +/// shutdown while every stream is still mid-flight. +const SLOW_CHUNKS: &[&str] = &[ + "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"c\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"d\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"e\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"f\"}}]}\n\n", + "data: {\"choices\":[{\"delta\":{\"content\":\"g\"}}]}\n\n", + "data: [DONE]\n\n", +]; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn shutdown_drains_100_inflight_streaming_chat_completions() { + // 1. Spin up a slow streaming worker. + let worker = crate::common::mock_worker::MockWorker::start_slow_stream( + SLOW_CHUNKS.to_vec(), + Duration::from_millis(60), + ) + .await; + let ctx = build_ctx_with_worker(&worker.url); + + // 2. Serve the REAL `build_router(ctx)` on a random port with the + // `with_graceful_shutdown` wiring main.rs uses. + let app = build_router(ctx); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let url = format!("http://{addr}/v1/chat/completions"); + + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }) + .await + .expect("axum::serve cleanly resolves on shutdown"); + }); + + // 3. Fire 100 concurrent streaming clients. + const N: usize = 100; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(); + let body = serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": true, + })) + .unwrap(); + + let mut handles = Vec::with_capacity(N); + for i in 0..N { + let c = client.clone(); + let u = url.clone(); + let b = body.clone(); + handles.push(tokio::spawn(async move { + let resp = c + .post(&u) + .header("content-type", "application/json") + .body(b) + .send() + .await + .map_err(|e| format!("client {i} send: {e}"))?; + if !resp.status().is_success() { + return Err(format!("client {i} non-2xx: {}", resp.status())); + } + let bytes: Bytes = resp + .bytes() + .await + .map_err(|e| format!("client {i} body: {e}"))?; + Ok::(bytes) + })); + } + + // 4. Let every request grab a connection and start receiving data. + // 100 ms is past the first chunk delay (60 ms) for every stream + // but well before the last chunk fires. + tokio::time::sleep(Duration::from_millis(100)).await; + + // 5. Trigger shutdown. axum stops accepting new connections but + // MUST drain the 100 already-attached streams. + let started = Instant::now(); + shutdown_tx.send(()).unwrap(); + + // 6. Every in-flight request must complete with a `[DONE]` terminator + // — proving the stream was NOT truncated by shutdown. + let mut bytes_total: usize = 0; + let mut done_count: usize = 0; + for h in handles { + let result = h + .await + .expect("client task panicked") + .expect("client completed"); + bytes_total += result.len(); + let body_str = String::from_utf8_lossy(&result); + if body_str.contains("data: [DONE]") { + done_count += 1; + } + } + // Server task must exit cleanly once all 100 in-flight requests drained. + server.await.expect("server task joins after shutdown"); + + let elapsed = started.elapsed(); + assert_eq!( + done_count, N, + "all {N} streams must terminate with `data: [DONE]` during graceful shutdown (got {done_count})" + ); + assert!( + bytes_total > 0, + "expected non-zero body bytes across {N} clients" + ); + // Drain MUST have taken at least ~400 ms (7 remaining chunks * 60ms). + // A shorter wait implies the streams were truncated. + assert!( + elapsed >= Duration::from_millis(300), + "graceful shutdown returned too fast ({elapsed:?}) — likely truncated streams" + ); +} + +#[tokio::test] +async fn shutdown_with_no_inflight_returns_promptly() { + // Complement of the load test: when nothing is in flight, the + // shutdown future resolves quickly. Catches a regression where the + // server might hang waiting on an idle connection pool. + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx_with_worker(&worker.url); + let app = build_router(ctx); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }) + .await + .unwrap(); + }); + + let started = Instant::now(); + shutdown_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .expect("server resolves within 2s when idle") + .expect("server task joined cleanly"); + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(1), + "idle shutdown took too long: {elapsed:?}" + ); +} diff --git a/experimental/sgl-router/tests/proxy/header_forwarding.rs b/experimental/sgl-router/tests/proxy/header_forwarding.rs new file mode 100644 index 000000000000..410ec6e898d8 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/header_forwarding.rs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +use axum::body::Body; +use axum::http::Request; +use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ModelConfig, ObservabilityConfig, + PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, +}; +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry; +use sgl_router::proxy::Proxy; +use sgl_router::server::app::build_router; +use sgl_router::server::app_context::AppContext; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::WorkerRegistry; +use std::sync::Arc; +use std::time::Duration; +use tower::ServiceExt; + +#[tokio::test] +async fn forwards_whitelisted_headers_strips_others() { + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let cfg = Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: ObservabilityConfig::default(), + models: vec![ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + }; + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + let _ = registry.add(WorkerSpec { + id: WorkerId("w1".into()), + url: worker.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(Duration::from_secs(5)).unwrap()); + let app = build_router(Arc::new(AppContext::new( + cfg, tokenizers, proxy, registry, policies, + ))); + + let body = serde_json::to_vec(&serde_json::json!({ + "model":"tiny","messages":[{"role":"user","content":"hi"}] + })) + .unwrap(); + + // Use a spoofed content-length that differs from the real body length so we + // can distinguish "inbound value forwarded" from "reqwest auto-computed it". + let spoofed_content_length = "99999"; + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .header("authorization", "Bearer test") + .header("x-request-id", "abc-123") + .header("x-sgl-route-key", "k1") + .header("cookie", "should-not-forward=true") + .header("host", "example.com") + .header("content-length", spoofed_content_length) + .header("transfer-encoding", "chunked") + .body(Body::from(body)) + .unwrap(); + app.oneshot(req).await.unwrap(); + + let seen = worker.captured.lock().unwrap(); + // Whitelisted headers are forwarded with their inbound VALUES intact — + // a regression that mangles, uppercases, or drops the value (e.g., + // forwarding the name but not the value) must fail this assertion. + assert_eq!( + seen.headers.get("authorization").map(String::as_str), + Some("Bearer test"), + "authorization must be forwarded with its inbound value verbatim", + ); + assert_eq!( + seen.headers.get("x-request-id").map(String::as_str), + Some("abc-123"), + "x-request-id must be forwarded with its inbound value verbatim", + ); + assert_eq!( + seen.headers.get("x-sgl-route-key").map(String::as_str), + Some("k1"), + "x-sgl-route-key must be forwarded with its inbound value verbatim", + ); + // Cookie must be stripped. + assert!(!seen.seen.contains("cookie")); + // transfer-encoding is hop-by-hop and must not be forwarded (reqwest does not + // re-add it for a regular body, so absence check is reliable here). + assert!( + !seen.seen.contains("transfer-encoding"), + "transfer-encoding is hop-by-hop and must be stripped" + ); + // content-length: the inbound spoofed value must not reach the upstream. + // reqwest may auto-compute its own content-length for the outbound body, + // so we assert value-inequality rather than absence. + assert_ne!( + seen.headers.get("content-length").map(|s| s.as_str()), + Some(spoofed_content_length), + "router must not forward the inbound content-length value to upstream" + ); + // Host: the inbound value must not reach the upstream. + let captured_host: Option<&String> = seen.headers.get("host"); + assert_ne!( + captured_host, + Some(&"example.com".to_string()), + "router must not forward the inbound Host header to upstream" + ); +} diff --git a/experimental/sgl-router/tests/proxy/main.rs b/experimental/sgl-router/tests/proxy/main.rs new file mode 100644 index 000000000000..9507099bb36d --- /dev/null +++ b/experimental/sgl-router/tests/proxy/main.rs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Full HTTP proxy integration tests. +//! +//! Each submodule spins up the router via `build_router(AppContext)` and +//! drives real requests through a `common::mock_worker::MockWorker` +//! backend. For component-scope tests that don't need the router, see +//! `tests/component/`. + +mod common; + +mod chat_routing; +mod failover; +mod graceful_shutdown; +mod header_forwarding; +mod pd_bootstrap_injection; +mod pd_pool_isolation; +mod timeout; diff --git a/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs new file mode 100644 index 000000000000..b48bf5fe4ede --- /dev/null +++ b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! PD-disagg bootstrap-room injection + dual-dispatch — end-to-end +//! at the HTTP layer using MockWorkers. +//! +//! Asserts the router-side contract for SGLang disagg-prefill HTTP mode: +//! +//! * Every PD-mode `/v1/chat/completions` request fans out to BOTH a +//! prefill and a decode worker (the prefill is `tokio::spawn`'d in +//! the background; the decode is awaited for the client response). +//! * Both bodies carry the SAME flat top-level fields: +//! - `bootstrap_host` = the chosen prefill worker's host +//! - `bootstrap_port` = the chosen prefill worker's bootstrap port +//! - `bootstrap_room` = a random u64 in `[0, i64::MAX]` (63-bit) +//! * Plain-mode requests do NOT carry any `bootstrap_*` field — the +//! injection step is gated on `worker.mode() == Prefill`. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use bytes::Bytes; +use serde_json::{json, Value}; +use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ModelConfig, ObservabilityConfig, + PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, +}; +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::factory::build_registry_with_defaults; +use sgl_router::proxy::Proxy; +use sgl_router::server::app::build_router; +use sgl_router::server::app_context::AppContext; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::WorkerRegistry; +use std::sync::Arc; +use std::time::Duration; +use tower::ServiceExt; + +fn config() -> Config { + Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: ObservabilityConfig::default(), + models: vec![ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + } +} + +fn build_ctx(specs: Vec) -> Arc { + let cfg = config(); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + for s in specs { + let _ = registry.add(s); + } + let policies = Arc::new(build_registry_with_defaults(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(Duration::from_secs(5)).unwrap()); + Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)) +} + +fn chat_request() -> Request { + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap() +} + +/// Pattern-B dispatch: prefill is `tokio::spawn`'d as a detached task +/// so the client response can return as soon as decode is reachable — +/// the prefill body is captured *eventually* but may not be present +/// when the handler returns. Poll with a short bound rather than +/// sleeping a fixed duration. +async fn await_captured_body( + mock: &crate::common::mock_worker::MockWorker, + timeout: Duration, + label: &str, +) -> Bytes { + let start = std::time::Instant::now(); + loop { + // Release the `std::sync::Mutex` guard before the sleep.await + // (clippy: await_holding_lock). + let captured = mock.captured.lock().unwrap().last_body.clone(); + if let Some(b) = captured { + return b; + } + if start.elapsed() > timeout { + panic!("{label}: no request body captured within {timeout:?}"); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +fn parse_body(b: &Bytes) -> Value { + serde_json::from_slice(b).expect("body must be valid JSON") +} + +/// Helper: extract bootstrap_host as &str. +fn bootstrap_host(v: &Value) -> Option<&str> { + v.get("bootstrap_host").and_then(|x| x.as_str()) +} +/// Helper: extract bootstrap_port as u16. +fn bootstrap_port(v: &Value) -> Option { + v.get("bootstrap_port") + .and_then(|x| x.as_u64()) + .map(|p| p as u16) +} +/// Helper: extract bootstrap_room as u64. +fn bootstrap_room(v: &Value) -> Option { + v.get("bootstrap_room").and_then(|x| x.as_u64()) +} + +/// PD-mode chat fans out to BOTH prefill and decode with identical +/// bootstrap fields injected into both bodies. +#[tokio::test] +async fn pd_mode_chat_injects_bootstrap_fields_into_both_bodies() { + let prefill = crate::common::mock_worker::MockWorker::start(vec![]).await; + let decode = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![ + WorkerSpec { + id: WorkerId("p1".into()), + url: prefill.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: Some(8997), + }, + WorkerSpec { + id: WorkerId("d1".into()), + url: decode.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + ]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK, "decode side should 200"); + + let prefill_body = await_captured_body(&prefill, Duration::from_secs(2), "prefill").await; + let decode_body = await_captured_body(&decode, Duration::from_secs(2), "decode").await; + let pj = parse_body(&prefill_body); + let dj = parse_body(&decode_body); + + // Same bootstrap_room on both sides (one room minted per request). + let p_room = bootstrap_room(&pj).expect("prefill body missing bootstrap_room"); + let d_room = bootstrap_room(&dj).expect("decode body missing bootstrap_room"); + assert_eq!( + p_room, d_room, + "prefill and decode must share the same bootstrap_room" + ); + + // Room must be in [0, i64::MAX]: the SGLang prefill stores it as + // i64 internally, so values with the top bit set wrap negative. + assert!( + p_room <= i64::MAX as u64, + "bootstrap_room {p_room} exceeds 63-bit range; SGLang would mis-store as negative i64", + ); + + // bootstrap_host on both sides == prefill worker's hostname + // (MockWorker binds to 127.0.0.1). + assert_eq!(bootstrap_host(&pj), Some("127.0.0.1")); + assert_eq!(bootstrap_host(&dj), Some("127.0.0.1")); + + // bootstrap_port on both sides == prefill's configured bootstrap_port. + assert_eq!(bootstrap_port(&pj), Some(8997)); + assert_eq!(bootstrap_port(&dj), Some(8997)); +} + +/// Plain-mode (non-PD) requests do NOT carry any `bootstrap_*` field. +/// The injection step is gated on `worker.mode() == Prefill`; plain +/// workers serve the chat route directly without disagg bootstrapping. +#[tokio::test] +async fn plain_mode_chat_does_not_inject_bootstrap_fields() { + let plain = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![WorkerSpec { + id: WorkerId("w1".into()), + url: plain.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + + let body = await_captured_body(&plain, Duration::from_secs(2), "plain").await; + let v = parse_body(&body); + assert!( + v.get("bootstrap_room").is_none(), + "plain-mode request must not carry bootstrap_room; got {v}" + ); + assert!( + v.get("bootstrap_host").is_none(), + "plain-mode request must not carry bootstrap_host; got {v}" + ); + assert!( + v.get("bootstrap_port").is_none(), + "plain-mode request must not carry bootstrap_port; got {v}" + ); +} + +/// PD-mode with multiple prefill workers + different `bootstrap_port` +/// values: the bootstrap_port injected MUST match the actually-chosen +/// prefill (not e.g. the first registered or a global config value). +#[tokio::test] +async fn pd_mode_bootstrap_port_matches_chosen_prefill_worker() { + let prefill_a = crate::common::mock_worker::MockWorker::start(vec![]).await; + let prefill_b = crate::common::mock_worker::MockWorker::start(vec![]).await; + let decode = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![ + WorkerSpec { + id: WorkerId("pA".into()), + url: prefill_a.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: Some(11111), + }, + WorkerSpec { + id: WorkerId("pB".into()), + url: prefill_b.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: Some(22222), + }, + WorkerSpec { + id: WorkerId("d1".into()), + url: decode.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + ]); + let app = build_router(ctx); + + // Fire enough requests to ensure round-robin hits both prefill workers. + for _ in 0..6 { + let res = app.clone().oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + } + + // Wait until both prefill workers have captured at least one body. + let body_a = await_captured_body(&prefill_a, Duration::from_secs(2), "prefill_a").await; + let body_b = await_captured_body(&prefill_b, Duration::from_secs(2), "prefill_b").await; + let va = parse_body(&body_a); + let vb = parse_body(&body_b); + // Each prefill must see its OWN bootstrap_port — never the other's. + assert_eq!( + bootstrap_port(&va), + Some(11111), + "prefill_a body should carry its own bootstrap_port" + ); + assert_eq!( + bootstrap_port(&vb), + Some(22222), + "prefill_b body should carry its own bootstrap_port" + ); +} + +/// Pin Pattern B's "prefill failure is invisible to the client" +/// contract: when the spawned prefill task gets a 5xx (or any other +/// upstream error), the decode response still reaches the client +/// unmodified. The router intentionally does not wire fail-fast here — +/// the decode side will eventually hang on `bootstrap_room` and time +/// out, but the chat handler itself doesn't propagate the prefill +/// error. Matches llm-d / aibrix behaviour. +#[tokio::test] +async fn pd_mode_prefill_5xx_does_not_poison_decode_response() { + let prefill = crate::common::mock_worker::MockWorker::start_returning_error( + StatusCode::INTERNAL_SERVER_ERROR, + json!({"error": "simulated prefill failure"}), + ) + .await; + let decode = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![ + WorkerSpec { + id: WorkerId("p1".into()), + url: prefill.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: Some(8997), + }, + WorkerSpec { + id: WorkerId("d1".into()), + url: decode.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + ]); + let app = build_router(ctx); + + // Client must see decode's 200 — the failing prefill is invisible. + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!( + res.status(), + StatusCode::OK, + "decode response should reach the client even when prefill returned 5xx", + ); + + // Decode received its body (proves dual dispatch fired despite + // the prefill failure). + let decode_body = await_captured_body(&decode, Duration::from_secs(2), "decode").await; + let v = parse_body(&decode_body); + assert_eq!(bootstrap_port(&v), Some(8997)); + + // Prefill also received its body — it just returned 5xx. The + // bootstrap fields are present so the engine WOULD have honoured + // the bootstrap_room if the mock had succeeded. + let prefill_body = await_captured_body(&prefill, Duration::from_secs(2), "prefill").await; + let pv = parse_body(&prefill_body); + assert_eq!(bootstrap_port(&pv), Some(8997)); +} diff --git a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs new file mode 100644 index 000000000000..ba484ddd35a8 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs @@ -0,0 +1,425 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! PD pool isolation — end-to-end at the HTTP layer using MockWorker. +//! +//! Drives the chat handler with: +//! +//! * A model whose registered workers are all `WorkerMode::Decode`. The +//! handler dispatches **prefill** traffic (chat-completions is the +//! prefill phase of a PD request), so it must return 503 with +//! `no_prefill_workers_available`. +//! * A model with no workers at all → 503 `no_healthy_workers` +//! (existing code path; pinned here so a future PD wiring change +//! doesn't silently swap codes). +//! * A PD-disagg model with both pools healthy → request flows to the +//! prefill worker (smoke; the decode worker MUST NOT be selected for +//! the chat route). + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use http_body_util::BodyExt; +use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ModelConfig, ObservabilityConfig, + PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, +}; +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::factory::build_registry_with_defaults; +use sgl_router::proxy::Proxy; +use sgl_router::server::app::build_router; +use sgl_router::server::app_context::AppContext; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::WorkerRegistry; +use std::sync::Arc; +use std::time::Duration; +use tower::ServiceExt; + +fn config() -> Config { + Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: ObservabilityConfig::default(), + models: vec![ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + } +} + +fn build_ctx(specs: Vec) -> Arc { + let cfg = config(); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + for s in specs { + let _ = registry.add(s); + } + let policies = Arc::new(build_registry_with_defaults(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(Duration::from_secs(5)).unwrap()); + Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)) +} + +fn chat_request() -> Request { + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + })) + .unwrap(), + )) + .unwrap() +} + +/// Gap closer #1: PD mode with only decode workers → 503 with +/// `no_prefill_workers_available`. The chat route is a prefill +/// dispatch, so a decode-only pool means partial failure. +#[tokio::test] +async fn pd_mode_decode_only_returns_no_prefill_workers_available() { + let worker = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![WorkerSpec { + id: WorkerId("d1".into()), + url: worker.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "no_prefill_workers_available", + ); + let body = res.into_body().collect().await.unwrap().to_bytes(); + let body_str = String::from_utf8_lossy(&body); + assert!( + body_str.contains("\"code\":\"no_prefill_workers_available\""), + "body: {body_str}" + ); +} + +/// Pin the existing-code-path branch: no workers at all → 503 with +/// `no_healthy_workers`. Ensures the new PD code path didn't swap the +/// code for the "model has zero workers" case. +#[tokio::test] +async fn no_workers_returns_no_healthy_workers() { + let ctx = build_ctx(vec![]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "no_healthy_workers", + ); +} + +/// PD-disagg deployment with both pools healthy → chat dispatch fans +/// out to BOTH the prefill and the decode worker (Pattern B: prefill +/// in a detached task, decode awaited for the client response). Both +/// receive the same bootstrap-injected body so the SGLang engine can +/// match KV transfers via `bootstrap_room`. Pool *isolation* — the +/// guarantee that the policy's prefill candidate set excludes decode +/// workers — is exercised at the resolver layer +/// (`policies::registry::tests::pd_resolution_returns_distinct_pools`). +/// Here we only assert the HTTP-layer wiring of the dual dispatch. +#[tokio::test] +async fn pd_mode_chat_dispatch_fans_to_both_prefill_and_decode() { + let prefill = crate::common::mock_worker::MockWorker::start(vec![]).await; + let decode = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![ + WorkerSpec { + id: WorkerId("p1".into()), + url: prefill.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: Some(8997), + }, + WorkerSpec { + id: WorkerId("d1".into()), + url: decode.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + ]); + let app = build_router(ctx); + + // Fire a single request; both prefill (spawn-and-forget) and + // decode (awaited) must receive a body with the injected + // bootstrap fields. The decode body is what the client sees on + // the response. + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!( + res.status(), + StatusCode::OK, + "decode response status should reach the client", + ); + + // Decode receives its body synchronously (we awaited it), so it's + // guaranteed captured by the time the response returned. Scope + // the lock guard to this block so it doesn't span the `.await` + // below (clippy: await_holding_lock). + { + let decode_seen = decode.captured.lock().unwrap(); + assert!( + decode_seen.last_body.is_some(), + "decode worker must receive the bootstrap-injected request body in PD mode", + ); + } + + // Prefill is detached; poll briefly until its capture lands. The + // prefill task races the HTTP response back to the client. The + // local binding releases the `std::sync::Mutex` guard before the + // `.await` — holding a sync mutex across an await would let one + // task pin the lock while another tries to acquire it. + let prefill_body = tokio::time::timeout(Duration::from_secs(2), async { + loop { + let captured = prefill.captured.lock().unwrap().last_body.clone(); + if let Some(b) = captured { + return b; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("prefill MUST eventually receive its body via the detached task"); + assert!(!prefill_body.is_empty()); +} + +/// Task C: PD-mode chat request carries an `x-sgl-decode-url` header +/// pointing at the host-affinity decode peer. With two prefill workers +/// on different hosts and a decode worker on each, the affinity helper +/// MUST pick the decode peer co-located with the chosen prefill. +/// +/// Round-robin will select prefill workers deterministically (alphabetic +/// dashmap order is not guaranteed; the test fires several requests so +/// at least one lands on each prefill, and asserts the per-host pairing +/// holds across all of them). +#[tokio::test] +async fn pd_mode_chat_dispatch_sets_decode_affinity_header() { + use std::collections::HashSet; + let prefill_a = crate::common::mock_worker::MockWorker::start(vec![]).await; + let prefill_b = crate::common::mock_worker::MockWorker::start(vec![]).await; + let decode_a = crate::common::mock_worker::MockWorker::start(vec![]).await; + let decode_b = crate::common::mock_worker::MockWorker::start(vec![]).await; + // MockWorker URLs always bind to `127.0.0.1`, so every worker + // shares the same host string and the affinity helper's + // same-host branch is moot here — the helper still returns a + // decode peer via the load-tiebreak fallback. The unit tests in + // `policies::registry::tests::decoder_picks_same_host_when_available` + // carry the real burden of pinning the host-affinity rules; this + // integration test only asserts the wiring is in place (the + // `x-sgl-decode-url` header IS set on PD requests, and the + // value is one of the registered decode worker URLs). + let ctx = build_ctx(vec![ + WorkerSpec { + id: WorkerId("p1".into()), + url: prefill_a.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + WorkerSpec { + id: WorkerId("p2".into()), + url: prefill_b.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + WorkerSpec { + id: WorkerId("d1".into()), + url: decode_a.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + WorkerSpec { + id: WorkerId("d2".into()), + url: decode_b.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + ]); + let app = build_router(ctx); + + // Fire 4 requests; both prefill workers see traffic via round-robin. + for _ in 0..4 { + let res = app.clone().oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + } + + // Every request that hit a prefill mock MUST carry the decode-hint + // header. The header value MUST be one of the two registered + // decode worker URLs. + let decode_urls: HashSet = [decode_a.url.clone(), decode_b.url.clone()] + .into_iter() + .collect(); + for (label, p) in [("prefill_a", &prefill_a), ("prefill_b", &prefill_b)] { + let g = p.captured.lock().unwrap(); + if g.last_body.is_none() { + // This prefill didn't receive a request — round-robin's + // dashmap iteration is non-deterministic, so one side may + // skip in a 4-request fire. Continue. + continue; + } + let hdr = g.headers.get("x-sgl-decode-url").unwrap_or_else(|| { + panic!( + "{label} did not receive an x-sgl-decode-url header. headers: {:?}", + g.headers + ) + }); + assert!( + decode_urls.contains(hdr), + "{label} got decode hint {hdr}, expected one of {decode_urls:?}", + ); + } +} + +/// Task C: plain-mode (non-PD) request does NOT carry the +/// `x-sgl-decode-url` header. Pin: the affinity step is gated on +/// `worker.mode() == Prefill` so plain workers are not asked to +/// bootstrap nonexistent decode peers. +#[tokio::test] +async fn plain_mode_chat_dispatch_omits_decode_affinity_header() { + let plain = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![WorkerSpec { + id: WorkerId("w1".into()), + url: plain.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + + let g = plain.captured.lock().unwrap(); + assert!( + !g.headers.contains_key("x-sgl-decode-url"), + "plain-mode worker must not receive a decode-affinity header. headers: {:?}", + g.headers, + ); +} + +/// Task C: PD-mode prefill request with NO decode workers → 503 +/// `no_decode_workers_available`. Pin: failure mode is loud and +/// distinct from the existing `no_prefill_workers_available` path. +#[tokio::test] +async fn pd_mode_prefill_only_returns_no_decode_workers_available() { + let prefill = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![WorkerSpec { + id: WorkerId("p1".into()), + url: prefill.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "no_decode_workers_available", + ); +} + +/// PD-mode chat response carries `x-sgl-decode-url` so external tests +/// can observe decode affinity end-to-end (without sniffing the proxy +/// hop into the upstream prefill worker). Mirrors the request-side +/// behavior asserted by `pd_mode_chat_dispatch_sets_decode_affinity_header`. +#[tokio::test] +async fn pd_mode_chat_response_carries_decode_affinity_header() { + use std::collections::HashSet; + let prefill = crate::common::mock_worker::MockWorker::start(vec![]).await; + let decode_a = crate::common::mock_worker::MockWorker::start(vec![]).await; + let decode_b = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![ + WorkerSpec { + id: WorkerId("p1".into()), + url: prefill.url.clone(), + mode: WorkerMode::Prefill, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + WorkerSpec { + id: WorkerId("d1".into()), + url: decode_a.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + WorkerSpec { + id: WorkerId("d2".into()), + url: decode_b.url.clone(), + mode: WorkerMode::Decode, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }, + ]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + let decode_urls: HashSet = [decode_a.url.clone(), decode_b.url.clone()] + .into_iter() + .collect(); + let hdr = res + .headers() + .get("x-sgl-decode-url") + .unwrap_or_else(|| { + panic!( + "PD-mode chat response did not carry x-sgl-decode-url; headers: {:?}", + res.headers(), + ) + }) + .to_str() + .unwrap() + .to_owned(); + assert!( + decode_urls.contains(&hdr), + "response carried decode hint {hdr}, expected one of {decode_urls:?}", + ); +} + +/// Plain-mode chat response does NOT carry `x-sgl-decode-url`. Pin: the +/// response-side mirror is gated on PD-mode dispatch. +#[tokio::test] +async fn plain_mode_chat_response_omits_decode_affinity_header() { + let plain = crate::common::mock_worker::MockWorker::start(vec![]).await; + let ctx = build_ctx(vec![WorkerSpec { + id: WorkerId("w1".into()), + url: plain.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }]); + let app = build_router(ctx); + + let res = app.oneshot(chat_request()).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + assert!( + !res.headers().contains_key("x-sgl-decode-url"), + "plain-mode chat response must not carry x-sgl-decode-url; headers: {:?}", + res.headers(), + ); +} diff --git a/experimental/sgl-router/tests/proxy/timeout.rs b/experimental/sgl-router/tests/proxy/timeout.rs new file mode 100644 index 000000000000..24c71f160a98 --- /dev/null +++ b/experimental/sgl-router/tests/proxy/timeout.rs @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Tests that the router does not wedge indefinitely when an upstream +//! worker accepts the TCP connection but never sends response headers. +//! +//! Without a configured `.timeout(...)` on the reqwest client, a stalled +//! backend hangs the axum handler future forever and the test harness +//! would just timeout. We assert here that the router returns a fast, +//! clean 502 (`upstream_timeout`) instead. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use http_body_util::BodyExt; +use sgl_router::config::{ + ActiveLoadConfig, Config, DiscoveryBackend, DiscoveryConfig, ModelConfig, ObservabilityConfig, + PolicyKind, ProxyConfig, ServerConfig, StaticUrlsDiscoveryConfig, +}; +use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; +use sgl_router::policies::factory::build_registry_with_defaults as build_policy_registry; +use sgl_router::proxy::Proxy; +use sgl_router::server::app::build_router; +use sgl_router::server::app_context::AppContext; +use sgl_router::tokenizer::TokenizerRegistry; +use sgl_router::workers::WorkerRegistry; +use std::sync::Arc; +use std::time::Duration; +use tower::ServiceExt; + +fn config(_worker_url: &str) -> Config { + Config { + server: ServerConfig { + host: "0".into(), + port: 0, + }, + observability: ObservabilityConfig::default(), + models: vec![ModelConfig { + id: "tiny".into(), + tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + policy: PolicyKind::RoundRobin, + circuit_breaker: None, + cache_aware: None, + }], + discovery: DiscoveryConfig { + backend: DiscoveryBackend::StaticUrls(StaticUrlsDiscoveryConfig { + urls: vec!["http://placeholder:0".into()], + }), + }, + proxy: ProxyConfig::default(), + active_load: ActiveLoadConfig::default(), + } +} + +#[tokio::test] +async fn non_streaming_request_times_out_when_worker_hangs() { + // Worker accepts and then sleeps for 5s; router timeout is 200ms. + let worker = + crate::common::mock_worker::MockWorker::start_hanging(Duration::from_secs(5)).await; + let cfg = config(&worker.url); + let tokenizers = Arc::new(TokenizerRegistry::load_from_config(&cfg).unwrap()); + let registry = Arc::new(WorkerRegistry::default()); + let _ = registry.add(WorkerSpec { + id: WorkerId("w1".into()), + url: worker.url.clone(), + mode: WorkerMode::Plain, + model_ids: vec![ModelId("tiny".into())], + bootstrap_port: None, + }); + let policies = Arc::new(build_policy_registry(&cfg).unwrap()); + let proxy = Arc::new(Proxy::new(Duration::from_millis(200)).unwrap()); + let ctx = Arc::new(AppContext::new(cfg, tokenizers, proxy, registry, policies)); + let app = build_router(ctx); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({ + "model": "tiny", + "messages": [{"role": "user", "content": "hi"}], + "stream": false + })) + .unwrap(), + )) + .unwrap(); + + let started = std::time::Instant::now(); + // Outer guard so a regression doesn't wedge CI forever. + let res = tokio::time::timeout(Duration::from_secs(2), app.oneshot(req)) + .await + .expect("router must return within 2s when proxy timeout is 200ms") + .unwrap(); + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(1), + "router must short-circuit on upstream timeout; elapsed {elapsed:?}" + ); + assert_eq!(res.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + res.headers().get("x-router-error-code").unwrap(), + "upstream_timeout" + ); + let bytes = res.into_body().collect().await.unwrap().to_bytes(); + let body_str = String::from_utf8_lossy(&bytes); + assert!( + body_str.contains("\"code\":\"upstream_timeout\""), + "body: {body_str}" + ); + // No leak of worker URL or reqwest source chain to the client. + assert!( + !body_str.contains(&worker.url), + "worker URL must not leak in client-visible body: {body_str}" + ); +} diff --git a/experimental/sgl-router/tests/scripts/generate_kv_events_hash_parity.py b/experimental/sgl-router/tests/scripts/generate_kv_events_hash_parity.py new file mode 100644 index 000000000000..794187143d26 --- /dev/null +++ b/experimental/sgl-router/tests/scripts/generate_kv_events_hash_parity.py @@ -0,0 +1,237 @@ +""" +Generator + validator for KV-event block-hash parity fixtures. + +Two modes: + + python3 experimental/sgl-router/tests/scripts/generate_kv_events_hash_parity.py + Regenerate the committed JSON fixture from the locally-replicated + algorithm. Run this when changing block-hash logic or adding new + shape coverage. CI's drift-check step runs this in --check mode. + + python3 experimental/sgl-router/tests/scripts/generate_kv_events_hash_parity.py --validate-against-sglang + Import the real `sglang.srt.mem_cache.radix_cache.RadixKey.hash_page` + and assert it agrees with the locally-replicated algorithm on every + fixture case. This is the only place the replica and the real + SGLang implementation are checked against each other. Run it + nightly (or whenever sglang is available on the Python path). + +# Authority + +Source-of-truth implementation: + - `python/sglang/srt/mem_cache/radix_cache.py::RadixKey.hash_page` + - `python/sglang/srt/mem_cache/utils.py::hash_str_to_int64` + +`hash_page_chain` below replicates that algorithm verbatim (no `import +sglang`) so the script runs without the heavy SGLang dependency tree and +can be audited at a glance. The algorithm is intentionally tiny: + + sha256(prior_digest_bytes ++ token_LE_u32 ++ token_LE_u32 ++ ...) + truncate to i64 = signed(first 16 hex chars) + +If SGLang ever changes the algorithm, update both the SGLang side AND +this script in the same commit; the Rust port in +`src/policies/kv_events/hash.rs` will then need the corresponding +update. The nightly `--validate-against-sglang` job is the safety net +that catches an SGLang-side change the human forgot to mirror here. + +# Output format + +A JSON array of cases. Each case is: + { + "name": "", + "tokens": [, ...], + "block_size": , + "expected_i64_hashes": [, ...] + } +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import pathlib +import sys + + +def hash_page_chain(tokens: list[int], block_size: int) -> list[int]: + """Compute the i64-truncated block hashes for `tokens` using SGLang's + `RadixKey.hash_page` algorithm + `hash_str_to_int64`. + + Returns one i64 per full or partial block. A partial last block (when + `len(tokens) % block_size != 0`) chains against the previous block's + full 32-byte SHA256 digest, matching SGLang's behaviour. + """ + if block_size == 0: + raise ValueError("block_size must be positive") + + out: list[int] = [] + prior_digest: bytes | None = None + n = len(tokens) + if n == 0: + return out + # Walk every page boundary, including a trailing partial page. + start = 0 + while start < n: + end = min(start + block_size, n) + hasher = hashlib.sha256() + if prior_digest is not None: + hasher.update(prior_digest) + for t in tokens[start:end]: + hasher.update(t.to_bytes(4, byteorder="little", signed=False)) + digest = hasher.digest() + prior_digest = digest + # hash_str_to_int64: first 16 hex chars (top 64 bits) -> signed i64. + hex_digest = digest.hex() + uint64_val = int(hex_digest[:16], 16) + if uint64_val >= 2**63: + i64 = uint64_val - 2**64 + else: + i64 = uint64_val + out.append(i64) + start = end + return out + + +# Cases mirror the three existing `cross_language_golden_*` tests plus +# additional shape coverage that exercises (a) zero-token edge, (b) +# block_size = 1, (c) very long sequences, (d) odd boundaries. +CASES: list[dict] = [ + { + "name": "single_full_block", + "tokens": [1, 2, 3, 4], + "block_size": 4, + }, + { + "name": "partial_last_block", + "tokens": [1, 2, 3, 4, 5], + "block_size": 4, + }, + { + "name": "multi_block", + "tokens": [10, 20, 30, 40, 50, 60, 70, 80], + "block_size": 2, + }, + { + "name": "empty_tokens", + "tokens": [], + "block_size": 4, + }, + { + "name": "block_size_one", + "tokens": [7, 8, 9], + "block_size": 1, + }, + { + "name": "odd_boundary", + "tokens": [100, 200, 300, 400, 500, 600, 700], + "block_size": 3, + }, + { + "name": "long_sequence", + # 128 tokens at block_size 16 → 8 blocks exactly. + "tokens": list(range(1, 129)), + "block_size": 16, + }, +] + + +def _materialize_cases() -> list[dict]: + return [ + { + "name": c["name"], + "tokens": c["tokens"], + "block_size": c["block_size"], + "expected_i64_hashes": hash_page_chain(c["tokens"], c["block_size"]), + } + for c in CASES + ] + + +def _validate_against_sglang() -> int: + """Import the real SGLang `RadixKey.hash_page` and compare its output + case-by-case against the locally-replicated `hash_page_chain`. Exits + non-zero (and prints a diff-friendly summary) on any mismatch. + + Returns 0 on success. This is the parity safety net for nightly CI. + """ + try: + from sglang.srt.mem_cache.radix_cache import RadixKey + except ImportError as e: + print( + f"--validate-against-sglang: cannot import sglang ({e}). " + "Install sglang into the Python path before running this mode.", + file=sys.stderr, + ) + return 2 + + failures: list[str] = [] + for c in CASES: + local = hash_page_chain(c["tokens"], c["block_size"]) + if c["block_size"] == 0 or not c["tokens"]: + # `RadixKey.hash_page` requires a non-empty page; the local + # replica handles edge cases (empty input → empty list) + # which the SGLang oracle would refuse. Skip these cases + # under validation — the replica owns the boundary semantics. + continue + sglang_hashes: list[int] = [] + prior_hex: str | None = None + for start in range(0, len(c["tokens"]), c["block_size"]): + page = c["tokens"][start : start + c["block_size"]] + key = RadixKey(token_ids=page, extra_key=None) + hex_digest = key.hash_page(prior_hex) + # SGLang's hash_page returns the hex digest; truncate to i64 + # the same way `hash_str_to_int64` does. + uint64_val = int(hex_digest[:16], 16) + i64 = uint64_val - (1 << 64) if uint64_val >= (1 << 63) else uint64_val + sglang_hashes.append(i64) + prior_hex = hex_digest + if sglang_hashes != local: + failures.append(f"case {c['name']}: local={local} sglang={sglang_hashes}") + + if failures: + print( + "--validate-against-sglang: replica/SGLang DRIFT detected:", + file=sys.stderr, + ) + for f in failures: + print(f" {f}", file=sys.stderr) + return 1 + print(f"--validate-against-sglang: OK ({len(CASES)} cases agreed)") + return 0 + + +def _write_fixture(cases_out: list[dict]) -> pathlib.Path: + out_path = ( + pathlib.Path(__file__).resolve().parent.parent + / "fixtures" + / "kv_events_hash_parity.json" + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w") as f: + json.dump(cases_out, f, indent=2, sort_keys=False) + f.write("\n") + return out_path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--validate-against-sglang", + action="store_true", + help="Compare the local replica to the imported SGLang implementation " + "and exit non-zero on drift. Requires sglang on the Python path.", + ) + args = parser.parse_args() + + if args.validate_against_sglang: + return _validate_against_sglang() + + cases_out = _materialize_cases() + out_path = _write_fixture(cases_out) + print(f"wrote {len(cases_out)} cases to {out_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experimental/sgl-router/tests/scripts/generate_parity_fixtures.py b/experimental/sgl-router/tests/scripts/generate_parity_fixtures.py new file mode 100644 index 000000000000..5d33813f29f0 --- /dev/null +++ b/experimental/sgl-router/tests/scripts/generate_parity_fixtures.py @@ -0,0 +1,115 @@ +""" +One-shot generator for tokenizer parity fixtures. + +Run manually when adding a model or changing a prompt shape: + python3 -m venv /tmp/parity-fixture-venv + /tmp/parity-fixture-venv/bin/pip install transformers + /tmp/parity-fixture-venv/bin/python experimental/sgl-router/tests/scripts/generate_parity_fixtures.py + +CI does NOT run this — it consumes the committed JSON. + +Model substitutions (gated models → public siblings of same family): + - Qwen/Qwen3-30B-A3B (gated) → Qwen/Qwen3-0.6B (same Qwen3 family, public) + - deepseek-ai/DeepSeek-V3.2-Exp (gated) → deepseek-ai/DeepSeek-V3 (older public sibling) + - openai/gpt-oss-20b → openai/gpt-oss-20b (public, used as-is) + +The acceptance criterion is "3 production model families × 4 shapes". +Using a smaller model from the same family satisfies the tokenizer parity +requirement because they share the same tokenizer.json vocabulary and merges. +""" + +import json +import pathlib +import sys + +try: + from transformers import AutoTokenizer +except ImportError: + sys.exit("pip install transformers first") + +ROOT = pathlib.Path(__file__).resolve().parents[1] / "fixtures" / "tokenizer_parity" + +# Primary model ids (may be gated). Fallbacks used automatically if 401/403. +MODELS = [ + # (primary_hf_id, fallback_hf_id, slug) + ("Qwen/Qwen3-30B-A3B", "Qwen/Qwen3-0.6B", "qwen3-30b"), + ("deepseek-ai/DeepSeek-V3.2-Exp", "deepseek-ai/DeepSeek-V3", "deepseek-v3p2"), + ("openai/gpt-oss-20b", None, "gpt-oss-20b"), +] + +LOREM = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod " + "tempor incididunt ut labore et dolore magna aliqua. " * 30 +) + +SHAPES = { + "short": "Hello, world!", + "long": LOREM, + "special_token_heavy": ( + "<|im_start|>system\nYou are helpful.<|im_end|>\n" + "<|im_start|>user\nHi<|im_end|>\n" + "<|im_start|>assistant\nHello<|im_end|>\n<|endoftext|>" + ), + "multi_turn_with_tools": ( + "<|im_start|>system\nYou have tools.<|im_end|>\n" + "<|im_start|>user\nWeather in Paris?<|im_end|>\n" + "<|im_start|>assistant\n\n" + '{"name": "get_weather", "arguments": {"city": "Paris"}}\n' + "<|im_end|>\n" + ), +} + + +def load_tokenizer_with_fallback(primary, fallback, slug): + """Try primary model id; fall back to sibling on any load failure. + + Failure modes handled: + - 401/403/gated: access denied on HuggingFace + - ValueError/KeyError: model type too new for installed transformers + - AttributeError: broken config chain in transformers compatibility layer + - OSError/requests errors: network / hub issues + """ + for hf_id in filter(None, [primary, fallback]): + try: + print(f" Trying {hf_id}...", flush=True) + tok = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True) + print(f" Loaded {hf_id}", flush=True) + return hf_id, tok + except (ValueError, KeyError, AttributeError, OSError) as e: + msg = str(e) + print( + f" {hf_id}: load failed ({type(e).__name__}: {msg[:120]}), trying fallback...", + flush=True, + ) + if fallback is None: + raise + continue + raise RuntimeError( + f"No accessible tokenizer for slug={slug} " f"(tried: {primary}, {fallback})" + ) + + +def main(): + total = 0 + for primary, fallback, slug in MODELS: + out = ROOT / slug + out.mkdir(parents=True, exist_ok=True) + print(f"\nLoading tokenizer for slug={slug}:", flush=True) + actual_hf_id, tok = load_tokenizer_with_fallback(primary, fallback, slug) + for shape, text in SHAPES.items(): + ids = tok.encode(text, add_special_tokens=False) + fixture = { + "model_id": actual_hf_id, + "shape": shape, + "prompt_text": text, + "expected_token_ids": ids, + "skip_special_tokens": False, + } + (out / f"{shape}.json").write_text(json.dumps(fixture, indent=2)) + print(f" {slug}/{shape}: {len(ids)} tokens", flush=True) + total += 1 + print(f"\nDone: {total} fixtures written to {ROOT}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index 6fb3b95d69db..4757563450de 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -57,6 +57,7 @@ dependencies = [ "tiktoken", "timm==1.0.16", "torchao==0.9.0", + "torchaudio==2.8.0", "tqdm", "mistral_common>=1.11.0", "transformers==5.8.1", diff --git a/python/sglang/jit_kernel/benchmark/bench_cast.py b/python/sglang/jit_kernel/benchmark/bench_cast.py deleted file mode 100644 index 4be874ce51a7..000000000000 --- a/python/sglang/jit_kernel/benchmark/bench_cast.py +++ /dev/null @@ -1,106 +0,0 @@ -import torch -import triton -import triton.testing - -from sglang.jit_kernel.benchmark.utils import ( - DEFAULT_DEVICE, - get_benchmark_range, - run_benchmark, -) -from sglang.jit_kernel.cast import downcast_fp8 as downcast_fp8_jit -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=10, suite="base-b-kernel-benchmark-1-gpu-large") - -DEVICE = DEFAULT_DEVICE -DTYPE = torch.bfloat16 - - -# ── Config ranges ────────────────────────────────────────────────────────────── - -SL_LIST = get_benchmark_range( - full_range=[4, 16, 64, 256, 512, 1024, 2048], - ci_range=[4, 64], -) - -HEAD_DIM_LIST = get_benchmark_range( - full_range=[(8, 128), (32, 128), (8, 256), (32, 256)], - ci_range=[(8, 128)], -) - -CONFIGS = [(sl, h, d, sl * 2) for sl in SL_LIST for h, d in HEAD_DIM_LIST] - -LINE_VALS = ["jit"] -LINE_NAMES = ["JIT (cast.cuh, 256 threads, 2D grid)"] -STYLES = [("orange", "-")] - - -# ── Perf report ──────────────────────────────────────────────────────────────── - - -@triton.testing.perf_report( - triton.testing.Benchmark( - x_names=["input_sl", "head", "dim", "out_sl"], - x_vals=CONFIGS, - line_arg="provider", - line_vals=LINE_VALS, - line_names=LINE_NAMES, - styles=STYLES, - ylabel="us", - plot_name="downcast-fp8-jit", - args={}, - ) -) -def benchmark(input_sl, head, dim, out_sl, provider): - k = torch.randn(input_sl, head, dim, dtype=DTYPE, device=DEVICE) - v = torch.randn(input_sl, head, dim, dtype=DTYPE, device=DEVICE) - k_out = torch.zeros(out_sl, head, dim, dtype=torch.uint8, device=DEVICE) - v_out = torch.zeros(out_sl, head, dim, dtype=torch.uint8, device=DEVICE) - k_scale = torch.tensor([1.0], dtype=torch.float32, device=DEVICE) - v_scale = torch.tensor([1.0], dtype=torch.float32, device=DEVICE) - loc = torch.arange(input_sl, dtype=torch.int64, device=DEVICE) - - fn = lambda: downcast_fp8_jit(k, v, k_out, v_out, k_scale, v_scale, loc) - - return run_benchmark(fn) - - -# ── Bandwidth analysis ───────────────────────────────────────────────────────── - - -def _report_bandwidth(input_sl, head, dim, dtype): - elem_bytes = torch.finfo(dtype).bits // 8 - total_bytes = input_sl * head * dim * (2 * elem_bytes + 2) - - k = torch.randn(input_sl, head, dim, dtype=dtype, device=DEVICE) - v = torch.randn(input_sl, head, dim, dtype=dtype, device=DEVICE) - k_out = torch.zeros(input_sl * 2, head, dim, dtype=torch.uint8, device=DEVICE) - v_out = torch.zeros(input_sl * 2, head, dim, dtype=torch.uint8, device=DEVICE) - k_scale = torch.tensor([1.0], dtype=torch.float32, device=DEVICE) - v_scale = torch.tensor([1.0], dtype=torch.float32, device=DEVICE) - loc = torch.arange(input_sl, dtype=torch.int64, device=DEVICE) - - jit_fn = lambda: downcast_fp8_jit(k, v, k_out, v_out, k_scale, v_scale, loc) - - jit_ms, _, _ = triton.testing.do_bench(jit_fn, quantiles=[0.5, 0.2, 0.8]) - - def fmt(ms): - return f"{ms*1000:6.2f}us {total_bytes/(ms*1e-3)/1e9:6.0f}GB/s" - - print(f" sl={input_sl:5d} h={head:2d} d={dim:4d}" f" | jit {fmt(jit_ms)}") - - -def report_bandwidth(): - print(f"\n{'='*95}") - print(" JIT (cast.cuh, 256 threads, 2D grid)") - print(f" dtype={DTYPE}, device={DEVICE}") - print(f"{'='*95}") - for sl in [64, 256, 1024, 2048]: - for h, d in [(8, 128), (32, 128), (8, 256), (32, 256)]: - _report_bandwidth(sl, h, d, DTYPE) - print() - - -if __name__ == "__main__": - benchmark.run(print_data=True) - report_bandwidth() diff --git a/python/sglang/jit_kernel/cast.py b/python/sglang/jit_kernel/cast.py deleted file mode 100644 index f0201c4aba20..000000000000 --- a/python/sglang/jit_kernel/cast.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import torch - -from sglang.jit_kernel.utils import cache_once, load_jit, make_cpp_args - -if TYPE_CHECKING: - from tvm_ffi.module import Module - - -@cache_once -def _jit_cast_module(dtype: torch.dtype) -> Module: - args = make_cpp_args(dtype) - return load_jit( - "cast", - *args, - cuda_files=["elementwise/cast.cuh"], - cuda_wrappers=[("downcast_fp8", f"downcast_fp8<{args}>")], - ) - - -def downcast_fp8( - k: torch.Tensor, - v: torch.Tensor, - k_out: torch.Tensor, - v_out: torch.Tensor, - k_scale: torch.Tensor, - v_scale: torch.Tensor, - loc: torch.Tensor, - mult: int = 1, - offset: int = 0, -) -> None: - """Fused downcast of KV cache tensors from bf16/fp16 to fp8 (E4M3). - - Scales each value by the inverse of its per-tensor scale, clamps to the - fp8 representable range [-448, 448], then converts to fp8 storage. - - Args: - k: [input_sl, head, dim] bf16/fp16 CUDA tensor - v: [input_sl, head, dim] bf16/fp16 CUDA tensor - k_out: [out_sl, head, dim] uint8 CUDA tensor (fp8 storage) - v_out: [out_sl, head, dim] uint8 CUDA tensor (fp8 storage) - k_scale: [1] float32 CUDA tensor, scale for k - v_scale: [1] float32 CUDA tensor, scale for v - loc: [input_sl] int64 CUDA tensor, destination sequence indices - mult: stride multiplier for output index (default 1) - offset: offset added to output index (default 0) - """ - module = _jit_cast_module(k.dtype) - module.downcast_fp8(k, v, k_out, v_out, k_scale, v_scale, loc, mult, offset) diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_v2.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_v2.cuh index e625919e1e82..31353e6a1531 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c128_v2.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c128_v2.cuh @@ -348,7 +348,7 @@ struct FlashCompress128Kernel { auto N = SymbolicSize{"batch_size"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({-1, 128, Trait::kElementSize}) // kv score .with_dtype() @@ -395,7 +395,7 @@ struct FlashCompress128Kernel { auto C = SymbolicSize{"num_c_plans"}; auto W = SymbolicSize{"num_w_plans"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({-1, 128, Trait::kElementSize}) // kv score .with_dtype() diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c4_v2.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c4_v2.cuh index b970e38473c1..efa9f05100a7 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c4_v2.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c4_v2.cuh @@ -309,7 +309,7 @@ struct FlashCompress4Kernel { auto N = SymbolicSize{"batch_size"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({-1, 4, Trait::kElementSize}) // kv score .with_dtype() @@ -356,7 +356,7 @@ struct FlashCompress4Kernel { auto C = SymbolicSize{"num_c_plans"}; auto W = SymbolicSize{"num_w_plans"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({-1, 4, Trait::kElementSize}) // kv score .with_dtype() diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh index 4518dcffa729..3e4aaaf5f0db 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/c_plan.cuh @@ -104,7 +104,11 @@ SGL_DEVICE uint32_t warp_inclusive_sum(uint32_t lane_id, uint32_t val) { static_assert(device::kWarpThreads == 32); #pragma unroll for (uint32_t offset = 1; offset < 32; offset *= 2) { - uint32_t n = __shfl_up_sync(0xFFFFFFFF, val, offset); +#ifndef USE_ROCM + uint32_t n = __shfl_up_sync(device::kFullMask, val, offset); +#else + uint32_t n = __shfl_up(val, offset, 32); +#endif if (lane_id >= offset) val += n; } return val; @@ -115,7 +119,11 @@ SGL_DEVICE uint32_t warp_inclusive_sum(uint32_t lane_id, uint32_t val) { SGL_DEVICE uint32_t warp_reduce_max_u32(uint32_t val) { #pragma unroll for (uint32_t mask = 16; mask > 0; mask >>= 1) { - val = max(val, __shfl_xor_sync(0xFFFFFFFF, val, mask, 32)); +#ifndef USE_ROCM + val = max(val, __shfl_xor_sync(device::kFullMask, val, mask, 32)); +#else + val = max(val, __shfl_xor(val, mask, 32)); +#endif } return val; } @@ -123,7 +131,11 @@ SGL_DEVICE uint32_t warp_reduce_max_u32(uint32_t val) { SGL_DEVICE uint32_t warp_reduce_min_u32(uint32_t val) { #pragma unroll for (uint32_t mask = 16; mask > 0; mask >>= 1) { - val = min(val, __shfl_xor_sync(0xFFFFFFFF, val, mask, 32)); +#ifndef USE_ROCM + val = min(val, __shfl_xor_sync(device::kFullMask, val, mask, 32)); +#else + val = min(val, __shfl_xor(val, mask, 32)); +#endif } return val; } @@ -452,8 +464,8 @@ inline PrefillPlan plan_compress_prefill( auto N = SymbolicSize{"num_q_tokens"}; auto cpu_or_gpu = SymbolicDevice{}; auto device_ = SymbolicDevice{}; - cpu_or_gpu.set_options(); - device_.set_options(); + cpu_or_gpu.set_options(); + device_.set_options(); TensorMatcher({B}) // .with_dtype() @@ -499,7 +511,7 @@ inline PrefillPlan plan_compress_prefill( constexpr int32_t kMaxMTPDraftTokens = 4; const auto mtp_pad = std::min(ring_size - compress_ratio, kMaxMTPDraftTokens); - if (cpu_or_gpu.unwrap().device_type == kDLCUDA) { + if (cpu_or_gpu.unwrap().device_type == kDLGPU) { // GPU input path: kernel0 builds the (CPU-loop-equivalent) plan metadata directly // on device, padding to num_q_tokens with invalid; kernel_1 then finalizes the // SWA-translated read/write locations. Used for MTP / cuda-graph capture where @@ -628,7 +640,7 @@ inline tvm::ffi::Tensor plan_compress_decode( const int32_t ring_size) { auto B = SymbolicSize{"batch_size"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({B}) // .with_dtype() @@ -691,7 +703,7 @@ inline PrefillPlan plan_compress_prefill_legacy( const bool use_cuda_graph) { auto B = SymbolicSize{"batch_size"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({B}) // .with_dtype() @@ -794,7 +806,7 @@ inline tvm::ffi::Tensor plan_compress_decode_legacy( const int32_t compress_ratio) { auto B = SymbolicSize{"batch_size"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({B}) // .with_dtype() diff --git a/python/sglang/jit_kernel/csrc/deepseek_v4/fused_norm_rope_v2.cuh b/python/sglang/jit_kernel/csrc/deepseek_v4/fused_norm_rope_v2.cuh index 653ff8750dbf..811dc41f1868 100644 --- a/python/sglang/jit_kernel/csrc/deepseek_v4/fused_norm_rope_v2.cuh +++ b/python/sglang/jit_kernel/csrc/deepseek_v4/fused_norm_rope_v2.cuh @@ -163,7 +163,11 @@ INDEXER_KERNEL void fused_norm_rope_indexer(const __grid_constant__ FusedNormRop for (uint32_t mask = 1; mask < kWarpThreads; mask <<= 1) { #pragma unroll for (int i = 0; i < kVecSize; ++i) { - const float other = __shfl_xor_sync(0xFFFFFFFFu, data[i], mask, kWarpThreads); +#ifndef USE_ROCM + const float other = __shfl_xor_sync(kFullMask, data[i], mask, kWarpThreads); +#else + const float other = __shfl_xor(data[i], mask, kWarpThreads); +#endif data[i] = (lane_id & mask) ? (other - data[i]) : (data[i] + other); } } @@ -307,8 +311,10 @@ FLASHMLA_KERNEL void fused_norm_rope_flashmla(const __grid_constant__ FusedNormR reinterpret_cast(rope_ptr)[lane_id] = result; } else { // Non-rope warp: per-warp UE8M0 group (64 elems -> 64 fp8 + 1 scale byte). - const auto x = data[0]; - const auto y = data[1]; + // BF16 round-trip to match the precision of the non-fused path + // (which goes through quant_to_nope_fp8_rope_bf16_pack_triton with bf16 input). + const auto x = cast(cast(data[0])); + const auto y = cast(cast(data[1])); const auto abs_max = warp::reduce_max(fmaxf(fabs(x), fabs(y))); const auto scale_raw = fmaxf(1e-4f, abs_max) / math::FP8_E4M3_MAX; const auto scale_ue8m0 = cast_to_ue8m0(scale_raw); @@ -359,7 +365,7 @@ struct FusedNormRopeKernel { auto N = SymbolicSize{"num_tokens"}; auto device_ = SymbolicDevice{}; - device_.set_options(); + device_.set_options(); TensorMatcher({N, kHeadDim}) // input .with_dtype() diff --git a/python/sglang/jit_kernel/csrc/elementwise/cast.cuh b/python/sglang/jit_kernel/csrc/elementwise/cast.cuh deleted file mode 100644 index f537ddc58819..000000000000 --- a/python/sglang/jit_kernel/csrc/elementwise/cast.cuh +++ /dev/null @@ -1,137 +0,0 @@ -#pragma once - -// Optimized cast kernel: fixed 256 threads, scaled out via 2D grid. -// Each thread handles exactly one float4 (kVecSize fp16/bf16 elements). -// No per-thread loop — pure grid scaling for any head*dim. - -#include -#include - -#include // For dtype_trait fp8 specialization -#include // For LaunchKernel -#include // For AlignedVector - -#include -#include - -#include - -namespace { - -constexpr int kBlockSize = 256; - -template -__global__ void fused_downcast_kernel( - const T* __restrict__ cache_k, - const T* __restrict__ cache_v, - const float* __restrict__ k_scale, - const float* __restrict__ v_scale, - fp8_e4m3_t* __restrict__ output_k, - fp8_e4m3_t* __restrict__ output_v, - const int input_num_tokens, - const int head, - const int dim, - const T max_fp8, - const T min_fp8, - const int64_t mult, - const int64_t offset, - const int64_t* __restrict__ loc) { - using namespace device; - - constexpr int kVecSize = 16 / sizeof(T); - using vec_t = AlignedVector; - using out_vec_t = AlignedVector; - - const int token_idx = blockIdx.x; - const int vec_idx = blockIdx.y * kBlockSize + threadIdx.x; - const int num_vecs = head * dim / kVecSize; - - if (token_idx >= input_num_tokens || vec_idx >= num_vecs) return; - - T k_scale_inv = static_cast(1.f) / cast(k_scale[0]); - T v_scale_inv = static_cast(1.f) / cast(v_scale[0]); - - auto clamp = [&](T val) { return val > max_fp8 ? max_fp8 : (min_fp8 > val ? min_fp8 : val); }; - - const int out_seq_idx = loc[token_idx]; - const T* in_k_base = cache_k + token_idx * head * dim; - const T* in_v_base = cache_v + token_idx * head * dim; - fp8_e4m3_t* out_k_base = output_k + (out_seq_idx * mult + offset) * head * dim; - fp8_e4m3_t* out_v_base = output_v + (out_seq_idx * mult + offset) * head * dim; - - vec_t k_vec, v_vec; - k_vec.load(in_k_base, vec_idx); - v_vec.load(in_v_base, vec_idx); - - out_vec_t out_k, out_v; -#pragma unroll - for (int j = 0; j < kVecSize; j++) { - out_k[j] = cast(clamp(k_vec[j] * k_scale_inv)); - out_v[j] = cast(clamp(v_vec[j] * v_scale_inv)); - } - - out_k.store(out_k_base, vec_idx); - out_v.store(out_v_base, vec_idx); -} - -template -void downcast_fp8( - tvm::ffi::TensorView k, - tvm::ffi::TensorView v, - tvm::ffi::TensorView k_out, - tvm::ffi::TensorView v_out, - tvm::ffi::TensorView k_scale, - tvm::ffi::TensorView v_scale, - tvm::ffi::TensorView loc, - int64_t mult, - int64_t offset) { - using namespace host; - - auto input_num_tokens = SymbolicSize{"input_num_tokens"}; - auto head = SymbolicSize{"head"}; - auto dim = SymbolicSize{"dim"}; - auto output_num_tokens = SymbolicSize{"out_sl"}; - auto device = SymbolicDevice{}; - device.set_options(); - - TensorMatcher({input_num_tokens, head, dim}).with_dtype().with_device(device).verify(k); - TensorMatcher({input_num_tokens, head, dim}).with_dtype().with_device(device).verify(v); - TensorMatcher({output_num_tokens, head, dim}).with_dtype().with_device(device).verify(k_out); - TensorMatcher({output_num_tokens, head, dim}).with_dtype().with_device(device).verify(v_out); - TensorMatcher({1}).with_dtype().with_device(device).verify(k_scale); - TensorMatcher({1}).with_dtype().with_device(device).verify(v_scale); - TensorMatcher({input_num_tokens}).with_dtype().with_device(device).verify(loc); - - const int num_tokens = static_cast(input_num_tokens.unwrap()); - const int h = static_cast(head.unwrap()); - const int d = static_cast(dim.unwrap()); - - constexpr int kVecSize = 16 / sizeof(T); - const int num_vecs = h * d / kVecSize; - const int grid_y = (num_vecs + kBlockSize - 1) / kBlockSize; - - dim3 grid(num_tokens, grid_y); - dim3 block(kBlockSize); - - const T max_fp8 = static_cast(kFP8E4M3Max); - const T min_fp8 = static_cast(-kFP8E4M3Max); - - LaunchKernel(grid, block, device.unwrap())( - fused_downcast_kernel, - static_cast(k.data_ptr()), - static_cast(v.data_ptr()), - static_cast(k_scale.data_ptr()), - static_cast(v_scale.data_ptr()), - static_cast(k_out.data_ptr()), - static_cast(v_out.data_ptr()), - num_tokens, - h, - d, - max_fp8, - min_fp8, - mult, - offset, - static_cast(loc.data_ptr())); -} - -} // namespace diff --git a/python/sglang/jit_kernel/dsv4/attn.py b/python/sglang/jit_kernel/dsv4/attn.py index aa8f24b4ffd9..87a265ecf495 100644 --- a/python/sglang/jit_kernel/dsv4/attn.py +++ b/python/sglang/jit_kernel/dsv4/attn.py @@ -7,6 +7,7 @@ from sglang.jit_kernel.utils import ( cache_once, is_arch_support_pdl, + is_hip_runtime, load_jit, make_cpp_args, ) @@ -58,13 +59,18 @@ def fused_store_cache( page_size: int, type: Literal["flashmla", "indexer"], ) -> None: - module = _jit_fused_store_module( - name=type, - input_dtype=input.dtype, - index_dtype=indices.dtype, - page_size=page_size, - ) - module.run(input, cache, indices) + if is_hip_runtime(): + from sglang.jit_kernel.triton_store_cache import triton_fused_store_cache + + triton_fused_store_cache(input, cache, indices, page_size=page_size, type=type) + else: + module = _jit_fused_store_module( + name=type, + input_dtype=input.dtype, + index_dtype=indices.dtype, + page_size=page_size, + ) + module.run(input, cache, indices) @triton.jit diff --git a/python/sglang/jit_kernel/dsv4/compress.py b/python/sglang/jit_kernel/dsv4/compress.py index 7885a82f6b66..38d696cdb2f0 100644 --- a/python/sglang/jit_kernel/dsv4/compress.py +++ b/python/sglang/jit_kernel/dsv4/compress.py @@ -1,9 +1,8 @@ from __future__ import annotations -from typing import Literal, NamedTuple, Optional, Union +from typing import TYPE_CHECKING, Literal, NamedTuple, Optional, Union import torch -from tvm_ffi.module import Module from sglang.jit_kernel.utils import ( cache_once, @@ -14,6 +13,9 @@ from .utils import make_name +if TYPE_CHECKING: + from tvm_ffi.module import Module + @cache_once def _jit_compress_norm_rope_module( diff --git a/python/sglang/jit_kernel/dsv4/elementwise.py b/python/sglang/jit_kernel/dsv4/elementwise.py index d2bbb8f3cb67..b721c841d8dc 100644 --- a/python/sglang/jit_kernel/dsv4/elementwise.py +++ b/python/sglang/jit_kernel/dsv4/elementwise.py @@ -132,10 +132,27 @@ def fused_q_indexer_rope_hadamard_quant( weights_out = torch.empty( (*q_input.shape[:-1], 1), dtype=torch.float32, device=q_input.device ) - module = _jit_main_q_indexer_rope_hadamard_quant_module(q_input.dtype) - module.forward( - q_input, q_fp8, weight, weights_out, float(weight_scale), freqs_real, positions - ) + if _is_hip: + torch.ops.sgl_kernel.dsv4_fused_q_indexer_rope_hadamard_quant( + q_input, + q_fp8, + weight, + weights_out, + float(weight_scale), + freqs_real, + positions, + ) + else: + module = _jit_main_q_indexer_rope_hadamard_quant_module(q_input.dtype) + module.forward( + q_input, + q_fp8, + weight, + weights_out, + float(weight_scale), + freqs_real, + positions, + ) return q_fp8, weights_out diff --git a/python/sglang/jit_kernel/dsv4/gemm.py b/python/sglang/jit_kernel/dsv4/gemm.py index c89ca32a1caa..da60eccecadb 100644 --- a/python/sglang/jit_kernel/dsv4/gemm.py +++ b/python/sglang/jit_kernel/dsv4/gemm.py @@ -14,11 +14,11 @@ def linear_bf16_fp32(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - if _linear_bf16_fp32_algo == "deep_gemm": + if _use_aiter: + return tgemm.mm(x, y, otype=x.dtype).float() + elif _linear_bf16_fp32_algo == "deep_gemm": z = torch.empty(x.size(0), y.size(0), dtype=torch.float32, device=x.device) deep_gemm_wrapper.gemm_nt_bf16bf16f32(x, y, z) return z - elif _use_aiter: - return tgemm.mm(x, y, otype=torch.float32) else: return torch.mm(x, y.t(), out_dtype=torch.float32) diff --git a/python/sglang/jit_kernel/dsv4/moe.py b/python/sglang/jit_kernel/dsv4/moe.py index a5372c781119..931805e448af 100644 --- a/python/sglang/jit_kernel/dsv4/moe.py +++ b/python/sglang/jit_kernel/dsv4/moe.py @@ -5,6 +5,7 @@ from sglang.jit_kernel.utils import ( cache_once, is_arch_support_pdl, + is_hip_runtime, load_jit, make_cpp_args, ) @@ -114,25 +115,37 @@ def hash_topk( scoring_func: str = "sqrtsoftplus", ) -> Tuple[torch.Tensor, torch.Tensor]: assert scoring_func == "sqrtsoftplus" - num_tokens = router_logits.size(0) - topk_routed = tid2eid.size(1) - topk_fused = topk_routed + num_fused_shared_experts - topk_ids = torch.empty( - (num_tokens, topk_fused), dtype=torch.int32, device=router_logits.device - ) - topk_weights = torch.empty( - (num_tokens, topk_fused), dtype=torch.float32, device=router_logits.device - ) - module = _jit_hash_topk_module() - module.hash_topk( - router_logits, - input_ids, - tid2eid, - topk_weights, - topk_ids, - routed_scaling_factor, - ) - return topk_weights, topk_ids + if is_hip_runtime(): + from sglang.jit_kernel.triton.hash_topk import hash_topk_triton + + return hash_topk_triton( + router_logits, + input_ids, + tid2eid, + num_fused_shared_experts, + routed_scaling_factor, + scoring_func, + ) + else: + num_tokens = router_logits.size(0) + topk_routed = tid2eid.size(1) + topk_fused = topk_routed + num_fused_shared_experts + topk_ids = torch.empty( + (num_tokens, topk_fused), dtype=torch.int32, device=router_logits.device + ) + topk_weights = torch.empty( + (num_tokens, topk_fused), dtype=torch.float32, device=router_logits.device + ) + module = _jit_hash_topk_module() + module.hash_topk( + router_logits, + input_ids, + tid2eid, + topk_weights, + topk_ids, + routed_scaling_factor, + ) + return topk_weights, topk_ids def mega_moe_pre_dispatch( diff --git a/python/sglang/jit_kernel/dsv4/topk.py b/python/sglang/jit_kernel/dsv4/topk.py index 3c7d14769561..a27245186f18 100644 --- a/python/sglang/jit_kernel/dsv4/topk.py +++ b/python/sglang/jit_kernel/dsv4/topk.py @@ -7,6 +7,7 @@ from sglang.jit_kernel.utils import ( cache_once, is_arch_support_pdl, + is_hip_runtime, load_jit, make_cpp_args, ) @@ -48,10 +49,15 @@ def topk_transform_512( page_size: int, out_raw_indices: Optional[torch.Tensor] = None, ) -> None: - module = _jit_topk_v1_module(out_page_indices.shape[1]) - module.topk_transform( - scores, seq_lens, page_tables, out_page_indices, page_size, out_raw_indices - ) + if is_hip_runtime(): + torch.ops.sgl_kernel.deepseek_v4_topk_transform_512( + scores, seq_lens, page_tables, out_page_indices, page_size, out_raw_indices + ) + else: + module = _jit_topk_v1_module(out_page_indices.shape[1]) + module.topk_transform( + scores, seq_lens, page_tables, out_page_indices, page_size, out_raw_indices + ) _WORKSPACE_INTS_PER_BATCH = 2 + 1024 * 2 diff --git a/python/sglang/jit_kernel/hadamard.py b/python/sglang/jit_kernel/hadamard.py index 25930ce942d3..6e845474903e 100644 --- a/python/sglang/jit_kernel/hadamard.py +++ b/python/sglang/jit_kernel/hadamard.py @@ -5,6 +5,7 @@ import torch from sglang.jit_kernel.utils import KERNEL_PATH, cache_once, load_jit, make_cpp_args +from sglang.srt.utils.custom_op import register_custom_op if TYPE_CHECKING: from tvm_ffi.module import Module @@ -56,6 +57,14 @@ def _hadamard_transform_impl( return out.reshape(shapes_og) +def _hadamard_transform_fake_impl( + x: torch.Tensor, + scale: float = 1.0, +) -> torch.Tensor: + return torch.empty_like(x) + + +@register_custom_op(fake_impl=_hadamard_transform_fake_impl) def hadamard_transform(x: torch.Tensor, scale: float = 1.0) -> torch.Tensor: module = _jit_hadamard_module(x.dtype) return _hadamard_transform_impl(x, scale, 8, module.hadamard_transform) diff --git a/python/sglang/jit_kernel/include/sgl_kernel/deepseek_v4/fp8_utils.cuh b/python/sglang/jit_kernel/include/sgl_kernel/deepseek_v4/fp8_utils.cuh index 4fdbb062c3cd..53a62755b4c1 100644 --- a/python/sglang/jit_kernel/include/sgl_kernel/deepseek_v4/fp8_utils.cuh +++ b/python/sglang/jit_kernel/include/sgl_kernel/deepseek_v4/fp8_utils.cuh @@ -5,7 +5,9 @@ #include #include +#ifndef USE_ROCM #include +#endif // Small helpers shared by the DeepSeek-V4 FP8/UE8M0 quantization kernels // (silu_and_mul_masked_post_quant, store, mega_moe_pre_dispatch, ...). @@ -30,14 +32,81 @@ SGL_DEVICE float inv_scale_ue8m0(int32_t exp) { } // Clamp to [-FP8_E4M3_MAX, FP8_E4M3_MAX]. +// Uses platform-specific max from type.cuh (448 for E4M3FN, 224 for E4M3FNUZ). SGL_DEVICE float fp8_e4m3_clip(float val) { - namespace math = device::math; - return math::max(math::min(val, math::FP8_E4M3_MAX), -math::FP8_E4M3_MAX); + return fmaxf(fminf(val, kFP8E4M3Max), -kFP8E4M3Max); } +#ifndef USE_ROCM // Pack two fp32 values into a single fp8x2_e4m3 with clamping. SGL_DEVICE fp8x2_e4m3_t pack_fp8(float x, float y) { return fp8x2_e4m3_t{fp32x2_t{fp8_e4m3_clip(x), fp8_e4m3_clip(y)}}; } +#else +// Software float -> FP8 E4M3 conversion for ROCm/HIP. +// Supports both E4M3FN (MI350X, gfx950) and E4M3FNUZ (MI300X, gfx942). +SGL_DEVICE uint8_t cvt_float_to_fp8_e4m3(float val) { + val = fp8_e4m3_clip(val); + if (val == 0.0f) return 0; + + uint32_t f32 = __float_as_uint(val); + uint8_t sign = static_cast((f32 >> 31) << 7); + int32_t exp32 = static_cast((f32 >> 23) & 0xFF) - 127; + uint32_t mant23 = f32 & 0x7FFFFF; + +#if HIP_FP8_TYPE_FNUZ + // E4M3FNUZ: bias=8, max=240, no negative zero, NaN=0x80 + constexpr int32_t kBias = 8; + constexpr int32_t kMaxExp = 15; + constexpr int32_t kMinSubnormExp = -10; // min subnormal exponent + constexpr int32_t kMinNormExp = -7; // min normal exponent + constexpr uint8_t kSaturate = 0x7Fu; // max normal = 0_1111_111 = 240.0 +#else + // E4M3FN: bias=7, max=448, NaN=0x7F + constexpr int32_t kBias = 7; + constexpr int32_t kMaxExp = 15; + constexpr int32_t kMinSubnormExp = -9; + constexpr int32_t kMinNormExp = -6; + constexpr uint8_t kSaturate = 0x7Eu; // max normal = 0_1111_110 = 448.0 +#endif + + int32_t exp8; + uint8_t mant3; + + if (exp32 < kMinSubnormExp) { + return sign; + } else if (exp32 < kMinNormExp) { + // Subnormal range + int32_t shift = -(kBias - 1) - exp32; // 1..3 + uint32_t subnorm_mant = (0x800000 | mant23) >> (shift + 20); + uint32_t round_bit = ((0x800000 | mant23) >> (shift + 19)) & 1; + subnorm_mant += round_bit; + mant3 = static_cast(subnorm_mant & 0x07); + exp8 = 0; + if (subnorm_mant > 7) { + exp8 = 1; + mant3 = 0; + } + } else { + exp8 = exp32 + kBias; + mant3 = static_cast(mant23 >> 20); + uint32_t round_bit = (mant23 >> 19) & 1; + mant3 += round_bit; + if (mant3 > 7) { + mant3 = 0; + exp8++; + } + if (exp8 >= kMaxExp) return sign | kSaturate; + } + return sign | (static_cast(exp8) << 3) | mant3; +} + +// Pack two fp32 values into a single fp8x2_e4m3 (uint16_t on HIP). +SGL_DEVICE fp8x2_e4m3_t pack_fp8(float x, float y) { + uint8_t x8 = cvt_float_to_fp8_e4m3(x); + uint8_t y8 = cvt_float_to_fp8_e4m3(y); + return static_cast(x8) | (static_cast(y8) << 8); +} +#endif } // namespace deepseek_v4::fp8 diff --git a/python/sglang/jit_kernel/include/sgl_kernel/runtime.cuh b/python/sglang/jit_kernel/include/sgl_kernel/runtime.cuh index 2812a2f8e1ce..4ea722a3fe79 100644 --- a/python/sglang/jit_kernel/include/sgl_kernel/runtime.cuh +++ b/python/sglang/jit_kernel/include/sgl_kernel/runtime.cuh @@ -10,7 +10,38 @@ #include #include +#ifndef USE_ROCM #include +#else +#include +#ifndef cudaOccupancyMaxActiveBlocksPerMultiprocessor +#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor +#endif +#ifndef cudaDeviceGetAttribute +#define cudaDeviceGetAttribute hipDeviceGetAttribute +#endif +#ifndef cudaDevAttrMultiProcessorCount +#define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount +#endif +#ifndef cudaDevAttrComputeCapabilityMajor +#define cudaDevAttrComputeCapabilityMajor hipDeviceAttributeComputeCapabilityMajor +#endif +#ifndef cudaRuntimeGetVersion +#define cudaRuntimeGetVersion hipRuntimeGetVersion +#endif +#ifndef cudaOccupancyAvailableDynamicSMemPerBlock +inline hipError_t +cudaOccupancyAvailableDynamicSMemPerBlock(std::size_t* smem, const void* func, int num_blocks, int block_size) { + // HIP does not expose this directly; return max shared mem as conservative estimate + hipDeviceProp_t prop; + int device; + hipGetDevice(&device); + hipGetDeviceProperties(&prop, device); + *smem = prop.sharedMemPerBlock; + return hipSuccess; +} +#endif +#endif namespace host::runtime { diff --git a/python/sglang/jit_kernel/include/sgl_kernel/tensor.h b/python/sglang/jit_kernel/include/sgl_kernel/tensor.h index 484c969b5dbd..1ae9233a61d9 100644 --- a/python/sglang/jit_kernel/include/sgl_kernel/tensor.h +++ b/python/sglang/jit_kernel/include/sgl_kernel/tensor.h @@ -33,6 +33,8 @@ #ifdef __CUDACC__ #include +#elif defined(__HIPCC__) +#include #endif namespace host { @@ -79,6 +81,15 @@ template <> struct _dtype_trait { inline static constexpr DLDataType value = {.code = DLDataTypeCode::kDLFloat8_e4m3fn, .bits = 8, .lanes = 1}; }; +#elif defined(__HIPCC__) +template <> +struct _dtype_trait { + inline static constexpr DLDataType value = {.code = DLDataTypeCode::kDLFloat, .bits = 16, .lanes = 1}; +}; +template <> +struct _dtype_trait { + inline static constexpr DLDataType value = {.code = DLDataTypeCode::kDLBfloat, .bits = 16, .lanes = 1}; +}; #endif template diff --git a/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh b/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh index a5abcdd4fa55..2dd6f3dc93a4 100644 --- a/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh +++ b/python/sglang/jit_kernel/include/sgl_kernel/utils.cuh @@ -44,6 +44,9 @@ inline constexpr auto cudaSuccess = hipSuccess; #define cudaGetErrorString hipGetErrorString #define cudaGetLastError hipGetLastError #define cudaLaunchKernel hipLaunchKernel +#define cudaMemcpyAsync hipMemcpyAsync +#define cudaMemcpyHostToDevice hipMemcpyHostToDevice +#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost #endif #ifndef USE_ROCM @@ -83,6 +86,13 @@ using fp32x4_t = float4; #define SGLANG_LDG(arg) *(arg) #endif +// DLPack device type for the current platform +#ifndef USE_ROCM +inline constexpr auto kDLGPU = kDLCUDA; +#else +inline constexpr auto kDLGPU = kDLROCM; +#endif + namespace device { /// \brief Macro: forced-inline device function qualifier. @@ -114,7 +124,11 @@ inline constexpr std::size_t kMaxVecBytes = SGL_ARCH_BLACKWELL_OR_GREATER ? 32 : /// \brief Number of threads per warp (always 32 on NVIDIA/AMD GPUs). inline constexpr auto kWarpThreads = 32u; /// \brief Full warp active mask (all 32 lanes). +#ifndef USE_ROCM inline constexpr auto kFullMask = 0xffffffffu; +#else +inline constexpr auto kFullMask = 0xffffffffffffffffULL; +#endif /** * \brief PDL (Programmatic Dependent Launch): wait for the primary kernel. diff --git a/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh b/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh index 975065e035c9..9d82efae1e37 100644 --- a/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh +++ b/python/sglang/jit_kernel/include/sgl_kernel/warp.cuh @@ -1,5 +1,5 @@ /// \file warp.cuh -/// \brief Warp-level reduction primitives using `__shfl_xor_sync`. +/// \brief Warp-level reduction primitives. #pragma once #include @@ -7,52 +7,49 @@ namespace device::warp { -/// \brief Full 32-thread active mask. +/// \brief Full warp active mask. +#ifndef USE_ROCM static constexpr uint32_t kFullMask = 0xffffffffu; +using mask_t = uint32_t; +#else +static constexpr uint64_t kFullMask = 0xffffffffffffffffULL; +using mask_t = uint64_t; +#endif /** * \brief Warp-level sum reduction. * - * Computes the sum of `value` across all active lanes specified by - * `active_mask` using butterfly (XOR) shuffles. The result is - * broadcast to all participating lanes. - * - * \tparam kNumThreads Group size for the reduction (defaults to a full warp). - * \tparam T Numeric type (e.g. float). - * \param value Per-lane input value. - * \param active_mask Bitmask of participating lanes (default: all 32). - * \return The sum across all active lanes. + * On CUDA: uses __shfl_xor_sync with width=32. + * On HIP: uses __shfl_xor with explicit width parameter (supports wave64 sub-groups). */ template -SGL_DEVICE T reduce_sum(T value, uint32_t active_mask = kFullMask) { +SGL_DEVICE T reduce_sum(T value, mask_t active_mask = kFullMask) { static_assert(kNumThreads >= 1 && kNumThreads <= kWarpThreads); static_assert(std::has_single_bit(kNumThreads), "must be pow of 2"); #pragma unroll for (int mask = kNumThreads / 2; mask > 0; mask >>= 1) +#ifndef USE_ROCM value = value + __shfl_xor_sync(active_mask, value, mask, 32); +#else + value = value + __shfl_xor(value, mask, kNumThreads); +#endif return value; } /** * \brief Warp-level max reduction. - * - * Computes the maximum of `value` across all active lanes using - * butterfly shuffles. The result is broadcast to all participating - * lanes. - * - * \tparam kNumThreads Group size for the reduction (defaults to a full warp). - * \tparam T Numeric type (must be supported by `math::max`). - * \param value Per-lane input value. - * \param active_mask Bitmask of participating lanes (default: all 32). - * \return The maximum across all active lanes. */ template -SGL_DEVICE T reduce_max(T value, uint32_t active_mask = kFullMask) { +SGL_DEVICE T reduce_max(T value, mask_t active_mask = kFullMask) { static_assert(kNumThreads >= 1 && kNumThreads <= kWarpThreads); static_assert(std::has_single_bit(kNumThreads), "must be pow of 2"); #pragma unroll for (int mask = kNumThreads / 2; mask > 0; mask >>= 1) +#ifndef USE_ROCM value = math::max(value, __shfl_xor_sync(active_mask, value, mask, 32)); +#else + value = math::max(value, __shfl_xor(value, mask, kNumThreads)); +#endif return value; } diff --git a/python/sglang/jit_kernel/tests/diffusion/test_diffusion_nvfp4_scaled_mm.py b/python/sglang/jit_kernel/tests/diffusion/test_diffusion_nvfp4_scaled_mm.py index 1214497d7eac..9c016158b7d2 100644 --- a/python/sglang/jit_kernel/tests/diffusion/test_diffusion_nvfp4_scaled_mm.py +++ b/python/sglang/jit_kernel/tests/diffusion/test_diffusion_nvfp4_scaled_mm.py @@ -136,6 +136,7 @@ def _build_layer( weight_global_scale: torch.Tensor, *, weight_scale_device: torch.device | str | None = None, + checkpoint_weight_scale_layout: str = "linear", ) -> tuple[ModelOptFp4LinearMethod, torch.nn.Module]: output_size, input_size_half = weight_fp4.shape input_size = input_size_half * 2 @@ -144,6 +145,7 @@ def _build_layer( is_checkpoint_nvfp4_serialized=True, group_size=BLOCK_SIZE, swap_weight_nibbles=True, + checkpoint_weight_scale_layout=checkpoint_weight_scale_layout, ) ) layer = torch.nn.Module() @@ -179,7 +181,11 @@ def _build_layer( expected_weight, _ = pad_nvfp4_weight( weight_fp4, n_alignment=128, k_alignment=0 ) - expected_scale = weight_scale_linear + expected_scale = ( + _swizzled_to_linear(weight_scale_linear, output_size, input_size) + if checkpoint_weight_scale_layout == "swizzled" + else weight_scale_linear + ) if expected_scale.shape[0] != expected_weight.shape[0]: pad_n = expected_weight.shape[0] - expected_scale.shape[0] expected_scale = torch.nn.functional.pad(expected_scale, (0, 0, 0, pad_n)) @@ -370,6 +376,57 @@ def test_flux2_shape_correctness_flashinfer_trtllm( assert diff < DEEPGEMM_FP4_MAX_DIFF, f"{m=}, {n=}, {k=}, {diff=:.6f}" +@pytest.mark.skipif( + not _nvfp4_supported(), + reason="Diffusion NVFP4 scaled mm correctness requires Blackwell GPUs", +) +def test_flux2_swizzled_scale_checkpoint_flashinfer_trtllm_matches_cudnn( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_diffusion_fp4_backend(monkeypatch, "flashinfer_trtllm") + + m, n, k = FLUX2_PROJECTION_SHAPE + generator = torch.Generator(device=DEVICE) + generator.manual_seed(20260517 + m + n + k) + + x = torch.randn((m, k), device=DEVICE, dtype=DTYPE, generator=generator) + weight = torch.randn((n, k), device=DEVICE, dtype=DTYPE, generator=generator) + input_global_scale = _make_global_scale(x) + weight_global_scale = _make_global_scale(weight) + alpha = (1.0 / (input_global_scale * weight_global_scale)).to(torch.float32) + + x_fp4, x_scale_swizzled = flashinfer.fp4_quantize(x, input_global_scale) + weight_fp4, weight_scale_swizzled = flashinfer.fp4_quantize( + weight, weight_global_scale + ) + if x_scale_swizzled.dtype == torch.uint8: + x_scale_swizzled = x_scale_swizzled.view(torch.float8_e4m3fn) + if weight_scale_swizzled.dtype == torch.uint8: + weight_scale_swizzled = weight_scale_swizzled.view(torch.float8_e4m3fn) + + method, layer = _build_layer( + weight_fp4, + weight_scale_swizzled, + input_global_scale, + weight_global_scale, + checkpoint_weight_scale_layout="swizzled", + ) + actual = method.apply(layer, x) + + expected = flashinfer.mm_fp4( + x_fp4, + weight_fp4.t(), + x_scale_swizzled, + weight_scale_swizzled.t(), + alpha, + DTYPE, + backend="cudnn", + ) + + diff = _calc_diff(actual, expected) + assert diff < DEEPGEMM_FP4_MAX_DIFF, f"{m=}, {n=}, {k=}, {diff=:.6f}" + + @pytest.mark.skipif( not _nvfp4_supported(), reason="Diffusion NVFP4 scaled mm correctness requires Blackwell GPUs", diff --git a/python/sglang/jit_kernel/triton/hash_topk.py b/python/sglang/jit_kernel/triton/hash_topk.py new file mode 100644 index 000000000000..b4e67fe1fc1f --- /dev/null +++ b/python/sglang/jit_kernel/triton/hash_topk.py @@ -0,0 +1,99 @@ +"""HIP fallback for ``hash_topk``: ``csrc/deepseek_v4/hash_topk.cuh`` uses +CUDA-only primitives, so on ROCm we dispatch to this Triton implementation. +""" + +from __future__ import annotations + +from typing import Tuple + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _hash_topk_triton_kernel( + router_logits_ptr, + input_ids_ptr, + tid2eid_ptr, + topk_weights_ptr, + topk_ids_ptr, + num_routed_experts: tl.constexpr, + topk_routed: tl.constexpr, + topk_fused: tl.constexpr, + routed_scaling_factor, + BLOCK_K: tl.constexpr, +): + token_pos = tl.program_id(0) + token_id = tl.load(input_ids_ptr + token_pos).to(tl.int64) + + k_off = tl.arange(0, BLOCK_K) + routed_mask = k_off < topk_routed + fused_mask = k_off < topk_fused + is_shared = k_off >= topk_routed + + expert_id = tl.load( + tid2eid_ptr + token_id * topk_routed + k_off, + mask=routed_mask, + other=0, + ).to(tl.int32) + logit = tl.load( + router_logits_ptr + token_pos * num_routed_experts + expert_id, + mask=routed_mask, + other=0.0, + ).to(tl.float32) + + softplus = tl.maximum(logit, 0.0) + tl.log(1.0 + tl.exp(-tl.abs(logit))) + weight = tl.sqrt(softplus) + weight = tl.where(routed_mask, weight, 0.0) + routed_sum = tl.sum(weight, axis=0) + + shared_weight = 1.0 / routed_scaling_factor + final_weight = tl.where(is_shared, shared_weight, weight / routed_sum) + shared_id = num_routed_experts + (k_off - topk_routed) + final_id = tl.where(is_shared, shared_id, expert_id).to(tl.int32) + + out_off = token_pos * topk_fused + k_off + tl.store(topk_weights_ptr + out_off, final_weight, mask=fused_mask) + tl.store(topk_ids_ptr + out_off, final_id, mask=fused_mask) + + +def hash_topk_triton( + router_logits: torch.Tensor, + input_ids: torch.Tensor, + tid2eid: torch.Tensor, + num_fused_shared_experts: int, + routed_scaling_factor: float, + scoring_func: str, +) -> Tuple[torch.Tensor, torch.Tensor]: + assert scoring_func == "sqrtsoftplus" + + num_tokens = router_logits.size(0) + num_routed_experts = router_logits.size(1) + topk_routed = tid2eid.size(1) + topk_fused = topk_routed + num_fused_shared_experts + + topk_weights = torch.empty( + (num_tokens, topk_fused), dtype=torch.float32, device=router_logits.device + ) + topk_ids = torch.empty( + (num_tokens, topk_fused), dtype=torch.int32, device=router_logits.device + ) + if num_tokens == 0: + return topk_weights, topk_ids + + block_k = max(triton.next_power_of_2(topk_fused), 1) + _hash_topk_triton_kernel[(num_tokens,)]( + router_logits, + input_ids, + tid2eid, + topk_weights, + topk_ids, + num_routed_experts=num_routed_experts, + topk_routed=topk_routed, + topk_fused=topk_fused, + routed_scaling_factor=float(routed_scaling_factor), + BLOCK_K=block_k, + num_warps=1, + ) + return topk_weights, topk_ids diff --git a/python/sglang/jit_kernel/triton_store_cache.py b/python/sglang/jit_kernel/triton_store_cache.py new file mode 100644 index 000000000000..b42a272e79a6 --- /dev/null +++ b/python/sglang/jit_kernel/triton_store_cache.py @@ -0,0 +1,237 @@ +from typing import Literal + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz + +_FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn +_FP8_INFO = torch.finfo(_FP8_DTYPE) + +# DeepSeek-V4 MLA paged FP8 cache layout +_MLA_HEAD_DIM = 512 # full MLA token dim (elements per input row) +_MLA_NOPE_DIM = 448 # nope sub-dim (elements) +_MLA_TILE_SIZE = 64 # FP8 tile width (also rope copy stride) +_MLA_SLOT_BYTES = 576 # bytes per slot in the paged FP8 cache +_MLA_BF16_SLOT_ELEMS = _MLA_SLOT_BYTES // 2 # bf16-view slot stride (elements) +_MLA_BF16_ROPE_OFFSET = _MLA_NOPE_DIM // 2 # bf16-view rope offset (elements) +_MLA_SCALES_PER_TOKEN = 8 # UE8M0 scales per token (7 nope tiles + 1 padding) +_MLA_NUM_TILES = 8 # 7 nope quant tiles + 1 rope copy tile +_MLA_ROPE_TILE_ID = 7 # tile id reserved for the rope copy + +# C4 indexer paged FP8 cache layout +_INDEXER_HEAD_DIM = 128 + +_UE8M0_EXPONENT_BIAS = 127 + + +@triton.jit +def _triton_fused_store_flashmla_kernel( + input_ptr, + cache_fp8_ptr, + cache_bf16_ptr, + cache_u8_ptr, + indices_ptr, + N, + PAGE_SIZE: tl.constexpr, + BYTES_PER_PAGE: tl.constexpr, + BYTES_PER_PAGE_BF16: tl.constexpr, + S_OFFSET: tl.constexpr, + TILE_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, + NOPE_DIM: tl.constexpr, + SLOT_BYTES: tl.constexpr, + BF16_SLOT_ELEMS: tl.constexpr, + BF16_ROPE_OFFSET: tl.constexpr, + SCALES_PER_TOKEN: tl.constexpr, + ROPE_TILE_ID: tl.constexpr, + UE8M0_BIAS: tl.constexpr, + FP8_MIN: tl.constexpr, + FP8_MAX: tl.constexpr, + EPS: tl.constexpr, +): + token_id = tl.program_id(0) + tile_id = tl.program_id(1) + + if token_id >= N: + return + + loc = tl.load(indices_ptr + token_id).to(tl.int32) + page = loc // PAGE_SIZE + slot = loc % PAGE_SIZE + + if tile_id == ROPE_TILE_ID: + rope_lane = tl.arange(0, TILE_SIZE) + rope_vals = tl.load(input_ptr + token_id * HEAD_DIM + NOPE_DIM + rope_lane) + rope_bf16_offset = ( + page * BYTES_PER_PAGE_BF16 + + slot * BF16_SLOT_ELEMS + + BF16_ROPE_OFFSET + + rope_lane + ) + tl.store(cache_bf16_ptr + rope_bf16_offset, rope_vals) + else: + tile_lane = tl.arange(0, TILE_SIZE) + x_bf16 = tl.load( + input_ptr + token_id * HEAD_DIM + tile_id * TILE_SIZE + tile_lane + ) + x_fp32 = x_bf16.to(tl.float32) + + abs_max = tl.max(tl.abs(x_fp32)) + scale = tl.maximum(abs_max, EPS) / FP8_MAX + + # cast scale to ue8m0 format + log2_scale = tl.log2(scale) + ceil_log2 = tl.math.ceil(log2_scale) + inv_scale = tl.exp2(-ceil_log2) + + x_fp8 = tl.clamp(x_fp32 * inv_scale, FP8_MIN, FP8_MAX).to( + cache_fp8_ptr.dtype.element_ty + ) + + nope_offset = ( + page * BYTES_PER_PAGE + slot * SLOT_BYTES + tile_id * TILE_SIZE + tile_lane + ) + tl.store(cache_fp8_ptr + nope_offset, x_fp8) + + ue8m0 = (ceil_log2.to(tl.int32) + UE8M0_BIAS).to(tl.uint8) + scale_offset = ( + page * BYTES_PER_PAGE + S_OFFSET + slot * SCALES_PER_TOKEN + tile_id + ) + tl.store(cache_u8_ptr + scale_offset, ue8m0) + + +def triton_fused_store_flashmla( + input: torch.Tensor, + cache: torch.Tensor, + indices: torch.Tensor, + page_size: int, +) -> None: + """Fused FP8 quantise + paged scatter for the SWA (flashmla) KV cache.""" + N = input.shape[0] + if N == 0: + return + + bytes_per_page = cache.shape[1] + cache_fp8 = cache.view(_FP8_DTYPE) + cache_bf16 = cache.view(torch.bfloat16) + indices_i32 = indices.to(torch.int32) if indices.dtype != torch.int32 else indices + + _triton_fused_store_flashmla_kernel[(N, _MLA_NUM_TILES)]( + input, + cache_fp8, + cache_bf16, + cache, + indices_i32, + N, + PAGE_SIZE=page_size, + BYTES_PER_PAGE=bytes_per_page, + BYTES_PER_PAGE_BF16=bytes_per_page // 2, + S_OFFSET=page_size * _MLA_SLOT_BYTES, + TILE_SIZE=_MLA_TILE_SIZE, + HEAD_DIM=_MLA_HEAD_DIM, + NOPE_DIM=_MLA_NOPE_DIM, + SLOT_BYTES=_MLA_SLOT_BYTES, + BF16_SLOT_ELEMS=_MLA_BF16_SLOT_ELEMS, + BF16_ROPE_OFFSET=_MLA_BF16_ROPE_OFFSET, + SCALES_PER_TOKEN=_MLA_SCALES_PER_TOKEN, + ROPE_TILE_ID=_MLA_ROPE_TILE_ID, + UE8M0_BIAS=_UE8M0_EXPONENT_BIAS, + FP8_MIN=_FP8_INFO.min, + FP8_MAX=_FP8_INFO.max, + EPS=1e-8, + ) + + +@triton.jit +def _triton_fused_store_indexer_kernel( + input_ptr, + cache_fp8_ptr, + cache_f32_ptr, + indices_ptr, + N, + PAGE_SIZE: tl.constexpr, + BYTES_PER_PAGE: tl.constexpr, + BYTES_PER_PAGE_F32: tl.constexpr, + SCALE_PAGE_OFFSET_F32: tl.constexpr, + HEAD_DIM: tl.constexpr, + FP8_MIN: tl.constexpr, + FP8_MAX: tl.constexpr, + EPS: tl.constexpr, +): + token_id = tl.program_id(0) + if token_id >= N: + return + + loc = tl.load(indices_ptr + token_id).to(tl.int32) + page = loc // PAGE_SIZE + slot = loc % PAGE_SIZE + + lane = tl.arange(0, HEAD_DIM) + x_fp32 = tl.load(input_ptr + token_id * HEAD_DIM + lane).to(tl.float32) + + abs_max = tl.max(tl.abs(x_fp32)) + scale = tl.maximum(abs_max, EPS) / FP8_MAX + inv_scale = 1.0 / scale + + x_fp8 = tl.clamp(x_fp32 * inv_scale, FP8_MIN, FP8_MAX).to( + cache_fp8_ptr.dtype.element_ty + ) + + fp8_offset = page * BYTES_PER_PAGE + slot * HEAD_DIM + lane + tl.store(cache_fp8_ptr + fp8_offset, x_fp8) + + f32_offset = page * BYTES_PER_PAGE_F32 + SCALE_PAGE_OFFSET_F32 + slot + tl.store(cache_f32_ptr + f32_offset, scale) + + +def triton_fused_store_indexer( + input: torch.Tensor, + cache: torch.Tensor, + indices: torch.Tensor, + page_size: int, +) -> None: + """Fused FP8 quantise + paged scatter for the C4 indexer KV cache.""" + N = input.shape[0] + if N == 0: + return + + bytes_per_page = cache.shape[1] + bytes_per_page_f32 = bytes_per_page // 4 + scale_page_offset_f32 = (_INDEXER_HEAD_DIM * page_size) // 4 + + cache_fp8 = cache.view(_FP8_DTYPE) + cache_f32 = cache.view(torch.float32) + indices_i32 = indices.to(torch.int32) if indices.dtype != torch.int32 else indices + + _triton_fused_store_indexer_kernel[(N,)]( + input, + cache_fp8, + cache_f32, + indices_i32, + N, + PAGE_SIZE=page_size, + BYTES_PER_PAGE=bytes_per_page, + BYTES_PER_PAGE_F32=bytes_per_page_f32, + SCALE_PAGE_OFFSET_F32=scale_page_offset_f32, + HEAD_DIM=_INDEXER_HEAD_DIM, + FP8_MIN=_FP8_INFO.min, + FP8_MAX=_FP8_INFO.max, + EPS=1e-8, + ) + + +def triton_fused_store_cache( + input: torch.Tensor, + cache: torch.Tensor, + indices: torch.Tensor, + *, + page_size: int, + type: Literal["flashmla", "indexer"], +) -> None: + """ROCm dispatch for fused_store_cache().""" + if type == "flashmla": + triton_fused_store_flashmla(input, cache, indices, page_size) + else: + triton_fused_store_indexer(input, cache, indices, page_size) diff --git a/python/sglang/jit_kernel/utils.py b/python/sglang/jit_kernel/utils.py index bcd42e5ce349..2bd0390fb524 100644 --- a/python/sglang/jit_kernel/utils.py +++ b/python/sglang/jit_kernel/utils.py @@ -277,7 +277,18 @@ def _jit_compile_context(): # NOTE: this might also be used in __main__.py for compile flags export def _get_default_target_flags() -> List[str]: if is_hip_runtime(): - return ["-DUSE_ROCM", "-std=c++20", "-O3"] + flags = ["-DUSE_ROCM", "-std=c++20", "-O3"] + # Detect FP8 type based on GPU architecture + try: + device = torch.cuda.current_device() + gcn_arch = torch.cuda.get_device_properties(device).gcnArchName + if "gfx942" in gcn_arch: + flags.append("-DHIP_FP8_TYPE_FNUZ=1") + else: + flags.append("-DHIP_FP8_TYPE_E4M3=1") + except Exception: + flags.append("-DHIP_FP8_TYPE_E4M3=1") + return flags else: return [ get_jit_cuda_arch().jit_flag, diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py index 876c233ede66..bbb2d4023d03 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py @@ -798,3 +798,28 @@ def _apply_chat_template(prompt: str) -> str: return_tensors=return_tensors, **tok_kwargs, ) + + +@dataclass +class Flux2KleinBasePipelineConfig(Flux2KleinPipelineConfig): + # Undistilled Klein base model, with guidance embeddings + should_use_guidance: bool = True + + def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype): + txt_seq_lens = self.require_text_seq_lens( + batch, + 0, + negative=True, + expected_batch_size=batch.negative_prompt_embeds[0].shape[0], + ) + return { + "freqs_cis": self.get_freqs_cis( + batch.negative_prompt_embeds[0], + batch.width, + batch.height, + device, + rotary_emb, + batch, + txt_seq_lens, + ) + } diff --git a/python/sglang/multimodal_gen/configs/sample/flux.py b/python/sglang/multimodal_gen/configs/sample/flux.py index 0b094957b7ba..5b4d81632463 100644 --- a/python/sglang/multimodal_gen/configs/sample/flux.py +++ b/python/sglang/multimodal_gen/configs/sample/flux.py @@ -29,3 +29,11 @@ class Flux2KleinSamplingParams(Flux2SamplingParams): # Klein is step-distilled, so default to 4 steps guidance_scale: float = 1.0 num_inference_steps: int = 4 + + +@dataclass +class Flux2KleinBaseSamplingParams(FluxSamplingParams): + # Klein-base is undistilled + num_inference_steps: int = 50 + guidance_scale: float = 4.0 + negative_prompt: str = "" diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py index 76f8af71ceed..3dc73dcacb7c 100644 --- a/python/sglang/multimodal_gen/envs.py +++ b/python/sglang/multimodal_gen/envs.py @@ -32,6 +32,7 @@ VERBOSE: bool = False SGLANG_DIFFUSION_SERVER_DEV_MODE: bool = False SGLANG_DIFFUSION_STAGE_LOGGING: bool = False + SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0 # cache-dit env vars (primary transformer) SGLANG_CACHE_DIT_ENABLED: bool = False SGLANG_CACHE_DIT_FN: int = 1 @@ -56,7 +57,7 @@ # model loading SGLANG_USE_RUNAI_MODEL_STREAMER: bool = True SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND: str | None = None - SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D: bool = True + SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D: str = "auto" SGLANG_USE_CUDA_HUNYUANVIDEO_GROUP_NORM_SILU: bool = False SGLANG_USE_ROCM_VAE: bool = False SGLANG_USE_ROCM_CUDNN_BENCHMARK: bool = False @@ -250,8 +251,13 @@ def _getter(): # If set, sgl_diffusion will enable stage logging, which will print the time # taken for each stage "SGLANG_DIFFUSION_STAGE_LOGGING": _lazy_bool("SGLANG_DIFFUSION_STAGE_LOGGING"), - "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": _lazy_bool( - "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", "true" + # Fraction of denoising steps that run both CFG branches before reusing the + # last conditional-minus-unconditional residual. Keep 1.0 to disable. + "SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float( + "SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0 + ), + "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": _lazy_str( + "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", "auto" ), # ================== cache-dit Env Vars ================== # Enable cache-dit acceleration for DiT inference @@ -283,6 +289,7 @@ def _getter(): "SGLANG_USE_RUNAI_MODEL_STREAMER", "true" ), # FlashInfer FP4 GEMM backend override for diffusion NVFP4. + # When unset, diffusion ModelOpt NVFP4 defaults to flashinfer_trtllm. # Supported values: # - auto # - flashinfer_cudnn diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index 484219238be0..9e7612162e69 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -46,6 +46,7 @@ ErnieImagePipelineConfig, ) from sglang.multimodal_gen.configs.pipeline_configs.flux import ( + Flux2KleinBasePipelineConfig, Flux2KleinPipelineConfig, Flux2PipelineConfig, ) @@ -85,6 +86,7 @@ ) from sglang.multimodal_gen.configs.sample.ernie_image import ErnieImageSamplingParams from sglang.multimodal_gen.configs.sample.flux import ( + Flux2KleinBaseSamplingParams, Flux2KleinSamplingParams, Flux2SamplingParams, FluxSamplingParams, @@ -788,8 +790,24 @@ def _register_configs(): "black-forest-labs/FLUX.2-klein-9B", ], model_detectors=[ - lambda hf_id: "flux.2-klein" in hf_id.lower() - or "flux2-klein" in hf_id.lower() + lambda hf_id: ( + "flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower() + ) + and "base" not in hf_id.lower() + ], + ) + register_configs( + sampling_param_cls=Flux2KleinBaseSamplingParams, + pipeline_config_cls=Flux2KleinBasePipelineConfig, + hf_model_paths=[ + "black-forest-labs/FLUX.2-klein-base-4B", + "black-forest-labs/FLUX.2-klein-base-9B", + ], + model_detectors=[ + lambda hf_id: ( + "flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower() + ) + and "base" in hf_id.lower() ], ) register_configs( diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/roles.py b/python/sglang/multimodal_gen/runtime/disaggregation/roles.py index b85b9244be39..e4aa3e48c54e 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/roles.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/roles.py @@ -25,7 +25,7 @@ def from_string(cls, value: str) -> "RoleType": @classmethod def choices(cls) -> list[str]: - return [role.value for role in cls] + return [role.value for role in cls] + sorted(_ROLE_ALIASES) def get_module_role(module_name: str) -> "RoleType | None": @@ -37,32 +37,53 @@ def get_module_role(module_name: str) -> "RoleType | None": "image_processor", "processor", "connectors", + "vision_language_encoder", ) if any( module_name == p or module_name.startswith(p + "_") for p in encoder_prefixes ): return RoleType.ENCODER - denoising_prefixes = ("transformer",) + if module_name in {"hy3dshape_conditioner", "hy3dshape_image_processor"}: + return RoleType.ENCODER + + denoising_prefixes = ( + "transformer", + "video_dit", + "audio_dit", + "dual_tower_bridge", + ) if any( module_name == p or module_name.startswith(p + "_") for p in denoising_prefixes ): return RoleType.DENOISER + if module_name == "hy3dshape_model": + return RoleType.DENOISER + decoder_prefixes = ("vae", "audio_vae", "video_vae", "vocoder") if any( module_name == p or module_name.startswith(p + "_") for p in decoder_prefixes ): return RoleType.DECODER + if module_name == "hy3dshape_vae": + return RoleType.DECODER + return None -def filter_modules_for_role(module_names: list[str], role: "RoleType") -> list[str]: +def filter_modules_for_role( + module_names: list[str], + role: "RoleType", + *, + extra_allowed_modules: set[str] | None = None, +) -> list[str]: """Filter module names to only those needed by the given role.""" if role in (RoleType.MONOLITHIC, RoleType.SERVER): return module_names + extra_allowed_modules = extra_allowed_modules or set() filtered = [] for name in module_names: module_role = get_module_role(name) @@ -71,8 +92,7 @@ def filter_modules_for_role(module_names: list[str], role: "RoleType") -> list[s filtered.append(name) elif module_role == role: filtered.append(name) - elif role == RoleType.ENCODER and module_role == RoleType.DECODER: - # Encoder also needs VAE for ImageVAEEncoding stages + elif name in extra_allowed_modules: filtered.append(name) return filtered diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py b/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py index 0cb1b0aacd85..24dc6280604f 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/scheduler_mixin.py @@ -51,6 +51,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.diffusion_scheduler_utils import ( clone_scheduler_runtime, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.common import get_zmq_socket from sglang.multimodal_gen.runtime.utils.distributed import broadcast_pyobj from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -205,7 +206,7 @@ def _init_disagg_request_scheduler(self: Scheduler, req: Req) -> None: scheduler_template = self.worker.pipeline.get_module("scheduler") if scheduler_template is None: return - device = torch.device(f"cuda:{self.worker.local_rank}") + device = torch.device(f"{current_platform.device_type}:{self.worker.local_rank}") _init_request_scheduler_from_template(scheduler_template, req, device) @@ -384,8 +385,8 @@ def _init_disagg_state(self: Scheduler, server_args, local_rank: int) -> None: if self._disagg_role != RoleType.MONOLITHIC: self._disagg_metrics = DisaggMetrics(role=self._disagg_role.value) - device = torch.device(f"cuda:{local_rank}") - self._transfer_stream = torch.cuda.Stream(device=device) + device = torch.device(f"{current_platform.device_type}:{local_rank}") + self._transfer_stream = torch.get_device_module().Stream(device=device) self._init_disagg_sockets() self._init_disagg_transfer_manager() @@ -455,7 +456,11 @@ def _init_disagg_transfer_manager(self: Scheduler): ) # Use GPU buffer when engine supports GPUDirect RDMA, CPU pinned otherwise - device = f"cuda:{physical_gpu_id}" if engine.supports_gpu_direct else "cpu" + device = ( + f"{current_platform.device_type}:{physical_gpu_id}" + if engine.supports_gpu_direct + else "cpu" + ) buffer = TransferTensorBuffer( pool_size=pool_size, device=device, role_name=self._disagg_role.value ) @@ -651,7 +656,7 @@ def _prefetch_transfer_ready(self: Scheduler, msg: dict) -> tuple: self._transfer_manager.register_prealloc_as_receive(request_id, slot) # Load tensors on transfer_stream (non-blocking) - local_device = f"cuda:{self.worker.local_rank}" + local_device = f"{current_platform.device_type}:{self.worker.local_rank}" tensors, load_event = self._transfer_manager.load_tensors_async( request_id, manifest, @@ -778,7 +783,9 @@ def _broadcast_req_to_all_ranks(self: Scheduler, req: Req | None) -> Req | None: # (set via torch.cuda.set_device(local_rank) during init), which is # already the right physical GPU — the .to() is effectively a no-op # but makes the invariant explicit for future readers. - local_device = torch.device(f"cuda:{self.worker.local_rank}") + local_device = torch.device( + f"{current_platform.device_type}:{self.worker.local_rank}" + ) for key, value in list(tensor_fields.items()): if isinstance(value, torch.Tensor): tensor_fields[key] = value.to(local_device, non_blocking=True) @@ -847,7 +854,9 @@ def _disagg_prefetch_event_loop(self: Scheduler, role_name: str) -> None: ) # Wait for load to complete on compute stream if load_event is not None: - torch.cuda.current_stream().wait_event(load_event) + torch.get_device_module().current_stream().wait_event( + load_event + ) # Now safe to free the receive slot if prealloc_slot_id is not None: with self._transfer_manager._lock: @@ -1222,7 +1231,7 @@ def _handle_transfer_ready(self: Scheduler, msg: dict) -> None: self._transfer_manager.register_prealloc_as_receive(request_id, slot) # 1. Start load on transfer_stream (non-blocking) - local_device = f"cuda:{self.worker.local_rank}" + local_device = f"{current_platform.device_type}:{self.worker.local_rank}" tensors, load_event = self._transfer_manager.load_tensors_async( request_id, manifest, @@ -1239,7 +1248,7 @@ def _handle_transfer_ready(self: Scheduler, msg: dict) -> None: # 4. Wait for load before compute (GPU must see the data) if load_event is not None: - torch.cuda.current_stream().wait_event(load_event) + torch.get_device_module().current_stream().wait_event(load_event) # 5. Free receive slot after load completes (data is on compute GPU) if prealloc_slot_id is not None: diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/transport/buffer.py b/python/sglang/multimodal_gen/runtime/disaggregation/transport/buffer.py index e7a8fae8a5d0..49c3faeb5070 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/transport/buffer.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/transport/buffer.py @@ -104,7 +104,7 @@ def write_tensor( name: str, tensor: torch.Tensor, byte_offset: int = 0, - stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> int: """Copy a tensor into the pool slot. Returns bytes written.""" src_tensor = tensor.contiguous() @@ -122,7 +122,7 @@ def write_tensor( src_bytes = src_tensor.view(torch.uint8).reshape(-1) if stream is not None: - with torch.cuda.stream(stream): + with torch.get_device_module().stream(stream): dst.copy_(src_bytes, non_blocking=True) else: dst.copy_(src_bytes, non_blocking=True) @@ -136,7 +136,7 @@ def read_tensor( dtype: torch.dtype, byte_offset: int = 0, device: torch.device | str = "cpu", - stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> torch.Tensor: """Read a tensor from the pool slot. Returns a clone on target device.""" nbytes = 1 @@ -157,12 +157,12 @@ def read_tensor( if same_device: # Clone to decouple tensor lifetime from pool slot if stream is not None: - with torch.cuda.stream(stream): + with torch.get_device_module().stream(stream): return src.clone() return src.clone() if stream is not None: - with torch.cuda.stream(stream): + with torch.get_device_module().stream(stream): return src.to(device, non_blocking=True) return src.to(device, non_blocking=True) @@ -170,7 +170,7 @@ def write_tensors_from_gpu( self, handle: SlotHandle, tensors: dict[str, torch.Tensor | list[torch.Tensor] | None], - stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> dict[str, list[dict]]: """Batch-write GPU tensors into a slot. Returns a manifest for later reads.""" manifest: dict[str, list[dict]] = {} @@ -178,7 +178,7 @@ def write_tensors_from_gpu( # Ensure copy stream sees all prior compute kernels if stream is not None: - stream.wait_stream(torch.cuda.current_stream()) + stream.wait_stream(torch.get_device_module().current_stream()) for name, value in tensors.items(): if value is None: @@ -225,7 +225,7 @@ def read_tensors_from_manifest( handle: SlotHandle, manifest: dict[str, list[dict]], device: torch.device | str = "cpu", - stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> dict[str, torch.Tensor | list[torch.Tensor]]: """Batch-read tensors from a slot using a manifest.""" result: dict[str, torch.Tensor | list[torch.Tensor]] = {} diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/transport/codec.py b/python/sglang/multimodal_gen/runtime/disaggregation/transport/codec.py index 51664b6b448c..14667348aebd 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/transport/codec.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/transport/codec.py @@ -48,7 +48,7 @@ class TensorWrapper: """Expose a CPU-contiguous tensor's data buffer for zero-copy ZMQ send.""" def __init__(self, tensor: torch.Tensor): - if tensor.is_cuda: + if tensor.is_cuda or tensor.is_npu: tensor = tensor.cpu() if not tensor.is_contiguous(): tensor = tensor.contiguous() diff --git a/python/sglang/multimodal_gen/runtime/disaggregation/transport/manager.py b/python/sglang/multimodal_gen/runtime/disaggregation/transport/manager.py index 9647c0bb492a..b9241710b13b 100644 --- a/python/sglang/multimodal_gen/runtime/disaggregation/transport/manager.py +++ b/python/sglang/multimodal_gen/runtime/disaggregation/transport/manager.py @@ -14,6 +14,7 @@ from sglang.multimodal_gen.runtime.disaggregation.transport.engine import ( BaseTransferEngine, ) +from sglang.multimodal_gen.runtime.platforms import current_platform logger = logging.getLogger(__name__) @@ -75,7 +76,7 @@ def stage_tensors( request_id: str, tensor_fields: dict[str, torch.Tensor | list[torch.Tensor] | None], scalar_fields: dict | None = None, - stream: torch.cuda.Stream | None = None, + stream: torch.Stream | None = None, ) -> StagedTransfer | None: """Stage GPU tensors into the local TransferBuffer. Returns None on allocation failure.""" total_size = 0 @@ -112,8 +113,8 @@ def stage_tensors( if stream is not None: stream.synchronize() - elif torch.cuda.is_available(): - torch.cuda.synchronize() + elif torch.get_device_module().is_available(): + torch.get_device_module().synchronize() staged = StagedTransfer( request_id=request_id, @@ -137,8 +138,8 @@ def stage_tensors_async( request_id: str, tensor_fields: dict[str, torch.Tensor | list[torch.Tensor] | None], scalar_fields: dict | None = None, - stream: torch.cuda.Stream | None = None, - ) -> tuple[StagedTransfer | None, torch.cuda.Event | None]: + stream: torch.Stream | None = None, + ) -> tuple[StagedTransfer | None, torch.Event | None]: """Stage GPU tensors, returning a CUDA event instead of blocking. Caller MUST wait on the event before reading buffer data. @@ -177,11 +178,11 @@ def stage_tensors_async( d2h_event = None if stream is not None: - d2h_event = torch.cuda.Event() + d2h_event = torch.get_device_module().Event() d2h_event.record(stream) - elif torch.cuda.is_available(): - d2h_event = torch.cuda.Event() - d2h_event.record(torch.cuda.current_stream()) + elif torch.get_device_module().is_available(): + d2h_event = torch.get_device_module().Event() + d2h_event.record(torch.get_device_module().current_stream()) staged = StagedTransfer( request_id=request_id, @@ -204,9 +205,12 @@ def load_tensors_async( self, request_id: str, manifest: dict, - device: torch.device | str = "cuda", - stream: torch.cuda.Stream | None = None, - ) -> tuple[dict[str, torch.Tensor | list[torch.Tensor]], torch.cuda.Event | None]: + device: torch.device | str = current_platform.device_type, + stream: torch.Stream | None = None, + ) -> tuple[ + dict[str, torch.Tensor | list[torch.Tensor]], + torch.get_device_module().Event | None, + ]: """Load tensors from receive slot to GPU, returning a CUDA event. Caller MUST wait on the event before using the returned tensors. @@ -225,11 +229,11 @@ def load_tensors_async( load_event = None if stream is not None: - load_event = torch.cuda.Event() + load_event = torch.get_device_module().Event() load_event.record(stream) - elif torch.cuda.is_available(): - load_event = torch.cuda.Event() - load_event.record(torch.cuda.current_stream()) + elif torch.get_device_module().is_available(): + load_event = torch.get_device_module().Event() + load_event.record(torch.get_device_module().current_stream()) logger.debug( "TransferManager: loaded_async %d tensor fields for %s to %s", @@ -315,8 +319,8 @@ def load_tensors( self, request_id: str, manifest: dict, - device: torch.device | str = "cuda", - stream: torch.cuda.Stream | None = None, + device: torch.device | str = current_platform.device_type, + stream: torch.Stream | None = None, ) -> dict[str, torch.Tensor | list[torch.Tensor]]: """Load tensors from a receive slot into GPU memory.""" with self._lock: @@ -333,8 +337,8 @@ def load_tensors( if stream is not None: stream.synchronize() - elif torch.cuda.is_available(): - torch.cuda.synchronize() + elif torch.get_device_module().is_available(): + torch.get_device_module().synchronize() logger.debug( "TransferManager: loaded %d tensor fields for %s to %s", diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py b/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py index a5171d8e5f98..f5b2bbc35098 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py @@ -33,6 +33,9 @@ def add_multimodal_gen_serve_args(parser: argparse.ArgumentParser): def execute_serve_cmd(args: argparse.Namespace, unknown_args: list[str] | None = None): """The entry point for the serve command.""" server_args = ServerArgs.from_cli_args(args, unknown_args) + if not server_args.is_arg_explicitly_set("warmup"): + server_args.warmup = True + logger.info("Warmup is enabled by default for sglang serve.") dispatch_launch(server_args) diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py index c6c25ebf2e87..793e2c52b0eb 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sparse_linear_attn.py @@ -323,8 +323,8 @@ def forward( ) # Apply feature maps - query = self.feature_map_q(query).contiguous().to(self.dtype) # c_q - key = self.feature_map_k(key).contiguous().to(self.dtype) # c_k + query = self.feature_map_q(query).to(self.dtype) # c_q + key = self.feature_map_k(key).to(self.dtype) # c_k # Linear attention computation o_l = self._calc_linear_attention_with_torch(query, key, value) @@ -681,8 +681,8 @@ def forward( ########## SPARGE END ########## # Linear attention with feature maps - q_linear = self.feature_map_q(q).contiguous().to(self.dtype) - k_linear = self.feature_map_k(k).contiguous().to(self.dtype) + q_linear = self.feature_map_q(q).to(self.dtype) + k_linear = self.feature_map_k(k).to(self.dtype) o_l = self._calc_linear_attention_with_torch(q_linear, k_linear, v) # Project linear attention output and combine diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py index 2ee9a17dfad9..abe11b207cb7 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py @@ -159,6 +159,8 @@ class VideoSparseAttentionMetadata(AttentionMetadata): reverse_tile_partition_indices: torch.LongTensor variable_block_sizes: torch.LongTensor non_pad_index: torch.LongTensor + untile_combined_index: torch.LongTensor + tile_buf: torch.Tensor | None = None # adaption for FastWan2.1-T2V-1.3B-Diffusers # Sequence lengths for the forward batch @@ -211,6 +213,7 @@ def build( # type: ignore non_pad_index = get_non_pad_index( variable_block_sizes, math.prod(VSA_TILE_SIZE) ) + untile_combined_index = non_pad_index[reverse_tile_partition_indices] return VideoSparseAttentionMetadata( current_timestep=current_timestep, @@ -222,6 +225,7 @@ def build( # type: ignore reverse_tile_partition_indices=reverse_tile_partition_indices, variable_block_sizes=variable_block_sizes, non_pad_index=non_pad_index, + untile_combined_index=untile_combined_index, ) @@ -244,58 +248,52 @@ def __init__( def tile( self, x: torch.Tensor, - num_tiles: list[int], - tile_partition_indices: torch.LongTensor, - non_pad_index: torch.LongTensor, + attn_metadata: VideoSparseAttentionMetadata, ) -> torch.Tensor: + num_tiles = attn_metadata.num_tiles t_padded_size = num_tiles[0] * VSA_TILE_SIZE[0] h_padded_size = num_tiles[1] * VSA_TILE_SIZE[1] w_padded_size = num_tiles[2] * VSA_TILE_SIZE[2] - - x_padded = torch.zeros( - ( - x.shape[0], - t_padded_size * h_padded_size * w_padded_size, - x.shape[-2], - x.shape[-1], - ), - device=x.device, - dtype=x.dtype, + target_shape = ( + x.shape[0], + t_padded_size * h_padded_size * w_padded_size, + x.shape[-2], + x.shape[-1], ) - x_padded[:, non_pad_index] = x[:, tile_partition_indices] - return x_padded + + buf = attn_metadata.tile_buf + if ( + buf is None + or buf.shape != target_shape + or buf.dtype != x.dtype + or buf.device != x.device + ): + buf = torch.zeros(target_shape, device=x.device, dtype=x.dtype) + attn_metadata.tile_buf = buf + + buf[:, attn_metadata.non_pad_index] = x[:, attn_metadata.tile_partition_indices] + return buf def untile( self, x: torch.Tensor, - reverse_tile_partition_indices: torch.LongTensor, - non_pad_index: torch.LongTensor, + untile_combined_index: torch.LongTensor, ) -> torch.Tensor: - x = x[:, non_pad_index][:, reverse_tile_partition_indices] - return x + return x[:, untile_combined_index] def preprocess_qkv( self, qkv: torch.Tensor, attn_metadata: VideoSparseAttentionMetadata, ) -> torch.Tensor: - return self.tile( - qkv, - attn_metadata.num_tiles, - attn_metadata.tile_partition_indices, - attn_metadata.non_pad_index, - ) + return self.tile(qkv, attn_metadata) def postprocess_output( self, output: torch.Tensor, attn_metadata: VideoSparseAttentionMetadata, ) -> torch.Tensor: - return self.untile( - output, - attn_metadata.reverse_tile_partition_indices, - attn_metadata.non_pad_index, - ) + return self.untile(output, attn_metadata.untile_combined_index) def forward( # type: ignore[override] self, diff --git a/python/sglang/multimodal_gen/runtime/layers/layernorm.py b/python/sglang/multimodal_gen/runtime/layers/layernorm.py index 8ed772d7093a..d4ae19ee11bb 100755 --- a/python/sglang/multimodal_gen/runtime/layers/layernorm.py +++ b/python/sglang/multimodal_gen/runtime/layers/layernorm.py @@ -387,14 +387,42 @@ def extra_repr(self) -> str: # NOTE(will): Needed to match behavior of diffusers and wan2.1 even while using # FSDP's MixedPrecisionPolicy class FP32LayerNorm(nn.LayerNorm): + def _cached_fp32_param( + self, attr: str, param: torch.Tensor | None, device: torch.device + ) -> torch.Tensor | None: + if param is None: + return None + + # Keep autograd semantics identical to the old path. The diffusion + # runtime enters here for inference, where grad is disabled. + if torch.is_grad_enabled(): + return param.float().to(device=device) + + key = ( + param.data_ptr(), + param._version, + param.device, + device, + param.dtype, + ) + cache = self.__dict__.get(attr) + if cache is not None and cache[0] == key: + return cache[1] + + fp32_param = param.detach().to(device=device, dtype=torch.float32) + self.__dict__[attr] = (key, fp32_param) + return fp32_param + def forward(self, inputs: torch.Tensor) -> torch.Tensor: origin_dtype = inputs.dtype device = inputs.device + weight = self._cached_fp32_param("_weight_fp32_cache", self.weight, device) + bias = self._cached_fp32_param("_bias_fp32_cache", self.bias, device) return F.layer_norm( inputs.float(), self.normalized_shape, - self.weight.float().to(device=device) if self.weight is not None else None, - self.bias.float().to(device=device) if self.bias is not None else None, + weight, + bias, self.eps, ).to(origin_dtype) diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py index 2f3d7c33e84f..1a3b76e983a7 100755 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py @@ -66,6 +66,29 @@ def _prepare_nvfp4_weight_bytes( return ((weight >> 4) | (weight << 4)).contiguous() +def _swizzled_nvfp4_scales_to_linear(scales: torch.Tensor) -> torch.Tensor: + """Convert FlashInfer/CUTLASS-swizzled FP4 scales back to row-major layout.""" + scale_ndim = scales.ndim + if scale_ndim == 2: + scales = scales.unsqueeze(0) + assert scales.ndim == 3 + + B, M, K = scales.shape + M_padded = round_up(M, 128) + K_padded = round_up(K, 4) + if M != M_padded or K != K_padded: + padded = torch.zeros( + (B, M_padded, K_padded), dtype=scales.dtype, device=scales.device + ) + padded[:B, :M, :K] = scales + scales = padded + + linear = scales.reshape(B, M_padded // 128, K_padded // 4, 32, 4, 4) + linear = linear.permute(0, 1, 4, 3, 2, 5).contiguous() + linear = linear.reshape(B, M_padded, K_padded)[:, :M, :K] + return linear.squeeze(0) if scale_ndim == 2 else linear + + def _require_flashinfer(): if flashinfer is None: raise RuntimeError( @@ -203,6 +226,7 @@ def __init__( packed_modules_mapping: Optional[Dict[str, List[str]]] = None, checkpoint_uses_packed_qkv: bool = False, swap_weight_nibbles: bool = False, + checkpoint_weight_scale_layout: str = "linear", ) -> None: super().__init__(exclude_modules, packed_modules_mapping) self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized @@ -214,6 +238,7 @@ def __init__( self.group_size = group_size self.checkpoint_uses_packed_qkv = checkpoint_uses_packed_qkv self.swap_weight_nibbles = swap_weight_nibbles + self.checkpoint_weight_scale_layout = checkpoint_weight_scale_layout @classmethod def get_name(cls) -> str: @@ -311,6 +336,9 @@ def from_config(cls, config: Dict[str, Any]) -> ModelOptFp4Config: packed_modules_mapping=config.get("packed_modules_mapping"), checkpoint_uses_packed_qkv=config.get("checkpoint_uses_packed_qkv", False), swap_weight_nibbles=swap_weight_nibbles, + checkpoint_weight_scale_layout=config.get( + "checkpoint_weight_scale_layout", "linear" + ), ) def get_quant_method(self, layer: torch.nn.Module, prefix: str): @@ -405,7 +433,7 @@ def apply( class ModelOptFp4LinearMethod(LinearMethodBase): - """NVFP4 linear method using CUTLASS FP4 GEMM.""" + """NVFP4 linear method using the selected FP4 GEMM backend.""" def __init__(self, quant_config: ModelOptFp4Config): self.quant_config = quant_config @@ -504,13 +532,18 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.quant_config, "swap_weight_nibbles", False ), ) + scales = layer.weight_scale + if ( + getattr(self.quant_config, "checkpoint_weight_scale_layout", "linear") + == "swizzled" + ): + scales = _swizzled_nvfp4_scales_to_linear(scales) _, flashinfer_backend = _get_fp4_gemm_op() if flashinfer_backend == "trtllm": flashinfer_ops = _require_flashinfer() weight, _ = pad_nvfp4_weight(w_swapped, n_alignment=128, k_alignment=0) - scales = layer.weight_scale if scales.shape[0] != weight.shape[0]: pad_n = weight.shape[0] - scales.shape[0] scales = torch.nn.functional.pad(scales, (0, 0, 0, pad_n)) @@ -550,7 +583,6 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.weights_padding_cols = weights_padding_cols copy_or_rebind_param(layer, "weight", weight) - scales = layer.weight_scale scale_ndim = scales.ndim if scale_ndim == 2: scales = scales.unsqueeze(0) diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py index 2ff38095ef2e..9c21d44d1fa8 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/vae_loader.py @@ -5,7 +5,6 @@ import torch.nn as nn from safetensors.torch import load_file as safetensors_load_file -from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.models import ModelConfig from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( ComponentLoader, @@ -18,6 +17,7 @@ from sglang.multimodal_gen.runtime.models.registry import ModelRegistry from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( get_diffusers_component_config, ) @@ -25,6 +25,7 @@ from sglang.multimodal_gen.utils import PRECISION_TO_TYPE logger = init_logger(__name__) +VAE_CHANNELS_LAST_3D_ENV = "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D" def _backfill_ltx2_audio_vae_latent_stats( @@ -59,6 +60,30 @@ def _convert_conv3d_weights_to_channels_last_3d(module: nn.Module) -> int: return num_converted +def _should_use_channels_last_3d( + server_args: ServerArgs | None, component_name: str +) -> bool: + if component_name not in ( + "vae", + "video_vae", + ) or not (current_platform.is_cuda() or current_platform.is_rocm()): + return False + + override = os.getenv(VAE_CHANNELS_LAST_3D_ENV) + if override is not None and override.strip().lower() != "auto": + return get_bool_env_var(VAE_CHANNELS_LAST_3D_ENV) + + if server_args is None: + return False + + pipeline_name = server_args.pipeline_config.__class__.__name__ + if pipeline_name.startswith("QwenImage"): + return True + if "Wan" in pipeline_name and server_args.num_gpus == 1: + return True + return False + + class VAELoader(ComponentLoader): """Shared loader for (video/audio) VAE modules.""" @@ -120,11 +145,7 @@ def load_customized( trust_remote_code=server_args.trust_remote_code, ) vae = vae.to(device=target_device, dtype=vae_dtype) - if ( - component_name in ("vae", "video_vae") - and torch.cuda.is_available() - and getattr(envs, "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", False) - ): + if _should_use_channels_last_3d(server_args, component_name): n = _convert_conv3d_weights_to_channels_last_3d(vae) if n > 0: logger.info( @@ -167,11 +188,7 @@ def load_customized( if unexpected_keys: logger.warning("VAE unexpected keys: %s", unexpected_keys) - if ( - component_name in ("vae", "video_vae") - and torch.cuda.is_available() - and getattr(envs, "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", False) - ): + if _should_use_channels_last_3d(server_args, component_name): n = _convert_conv3d_weights_to_channels_last_3d(vae) if n > 0: logger.info("VAE: converted %d Conv3d weights to channels_last_3d", n) diff --git a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py index 204608c24349..58f167c94298 100644 --- a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py @@ -94,6 +94,17 @@ def _merge_modelopt_fp4_configs( inferred_config.swap_weight_nibbles = getattr( inferred_config, "swap_weight_nibbles", False ) or getattr(existing_config, "swap_weight_nibbles", False) + existing_scale_layout = getattr( + existing_config, "checkpoint_weight_scale_layout", "linear" + ) + inferred_scale_layout = getattr( + inferred_config, "checkpoint_weight_scale_layout", "linear" + ) + inferred_config.checkpoint_weight_scale_layout = ( + existing_scale_layout + if inferred_scale_layout == "linear" and existing_scale_layout != "linear" + else inferred_scale_layout + ) if getattr(inferred_config, "group_size", None) is None: inferred_config.group_size = getattr(existing_config, "group_size", None) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py index 7501257d9032..c5e25d073a3c 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py @@ -25,6 +25,7 @@ ) from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.nvtx_pytorch_hooks import DiffusionNvtxHooks logger = init_logger(__name__) @@ -148,6 +149,10 @@ def __init__( self._current_use_index: int = -1 self._active_use: ComponentUse | None = None self._active_use_module: nn.Module | None = None + self._active_nvtx_key: tuple[str, str, str | None] | None = None + self._nvtx_hooks_by_use_key: dict[ + tuple[str, str, str | None], tuple[int, DiffusionNvtxHooks] + ] = {} self._prefetched_use_keys: set[tuple[str, str, str | None]] = set() self._custom_strategies: dict[str, ComponentResidencyStrategy] = dict( pipeline.component_residency_strategies @@ -161,6 +166,7 @@ def enabled(self) -> bool: def refresh_pipeline(self, pipeline: ComponentResidencyPipeline) -> None: custom_strategies = dict(pipeline.component_residency_strategies) if pipeline is not self.pipeline: + self._remove_nvtx_hooks() self.strategy_for.cache_clear() self._should_keep_single_dit.cache_clear() self._active_use = None @@ -200,6 +206,7 @@ def begin_request( ) self._active_use = None self._active_use_module = None + self._disable_active_nvtx() self._current_use_index = -1 self._prefetched_use_keys.clear() self._uses_seen.clear() @@ -241,11 +248,11 @@ def after_stage(self, stage_index: int) -> None: return self._trace("stage_exit", detail=f"index={stage_index}") - def before_use(self, use: ComponentUse) -> None: + def before_use(self, use: ComponentUse, module: nn.Module | None = None) -> None: """component use-site starts""" if not self.enabled: return - self.begin_use(use) + self.begin_use(use, module=module) def begin_use(self, use: ComponentUse, module: nn.Module | None = None) -> None: """Begin one sequential component use interval. this is idempotent @@ -255,8 +262,19 @@ def begin_use(self, use: ComponentUse, module: nn.Module | None = None) -> None: 3. Wait until the current component is ready, then prefetch the next heavy use. """ if self._active_use is not None and self._same_use(self._active_use, use): + if self._use_key(self._active_use) != self._use_key(use): + self._mark_current_use(use) + self._active_use = use + self.state.current_use = use + self._enable_nvtx_for_use( + use, + module + or self._active_use_module + or self.get_module(use.component_name), + ) return if self._active_use is not None: + self._disable_active_nvtx() # finish previous active use self._finish_use( self._active_use, @@ -267,9 +285,10 @@ def begin_use(self, use: ComponentUse, module: nn.Module | None = None) -> None: self._active_use_module = None self.state.current_use = None self._mark_current_use(use) - self._prepare_forward_use(use, module=module) + module = self._prepare_forward_use(use, module=module) self._active_use = use self._active_use_module = module + self._enable_nvtx_for_use(use, module) self._prefetch_next_memory_intensive_use() def end_use(self, use: ComponentUse, module: nn.Module | None = None) -> None: @@ -281,6 +300,7 @@ def end_use(self, use: ComponentUse, module: nn.Module | None = None) -> None: """ if self._active_use is None or not self._same_use(self._active_use, use): return + self._disable_active_nvtx() self._finish_use( self._active_use, module=self._active_use_module or module, @@ -323,6 +343,20 @@ def ensure_ready(self, use: ComponentUse, module: nn.Module | None = None) -> No return self._prepare_forward_use(use, module=module) + def remove_nvtx_hooks_for_module(self, module: nn.Module | None) -> None: + """Detach NVTX hooks before a component object is deleted or replaced.""" + if module is None: + return + module_id = id(module) + for key, (registered_id, hooks) in list(self._nvtx_hooks_by_use_key.items()): + if registered_id != module_id: + continue + if self._active_nvtx_key == key: + hooks.set_enabled(False) + self._active_nvtx_key = None + hooks.remove_hooks() + del self._nvtx_hooks_by_use_key[key] + def prefetch_checkpoint(self, anchor: ComponentUse | None = None) -> None: """Give the manager a timeline overlap point. @@ -341,6 +375,7 @@ def finish_active_use(self, *, prefetch_next: bool = True) -> None: if self._active_use is None: return active_use = self._active_use + self._disable_active_nvtx() self._finish_use( active_use, module=self._active_use_module, @@ -354,12 +389,12 @@ def finish_active_use(self, *, prefetch_next: bool = True) -> None: def _prepare_forward_use( self, use: ComponentUse, module: nn.Module | None = None - ) -> None: + ) -> nn.Module | None: """Prepare a component that is about to run and wait until it is ready.""" module = module or self.get_module(use.component_name) if module is None: self._trace("skip_missing", use) - return + return None strategy = self.strategy_for(use.component_name, module) self._uses_seen[use.component_name] = use self.state.current_use = use @@ -367,6 +402,66 @@ def _prepare_forward_use( strategy.prepare_for_use(module, use, self.state) self._trace("wait", use, strategy, module) strategy.wait_for_use(module, use, self.state) + return module + + def _enable_nvtx_for_use( + self, use: ComponentUse, module: nn.Module | None = None + ) -> None: + if ( + not self.server_args.enable_layerwise_nvtx_marker + or self.state.batch_is_warmup + or not isinstance(module, nn.Module) + ): + self._disable_active_nvtx() + return + + key = self._use_key(use) + if self._active_nvtx_key != key: + self._disable_active_nvtx() + + module_id = id(module) + existing = self._nvtx_hooks_by_use_key.get(key) + if existing is None or existing[0] != module_id: + if existing is not None: + existing[1].remove_hooks() + self._nvtx_hooks_by_use_key.pop(key, None) + hooks = DiffusionNvtxHooks() + prefix = self._nvtx_prefix_for_use(use) + total = hooks.register_hooks(module, prefix=prefix) + if total == 0: + return + logger.debug( + "[component_residency] Registered NVTX hooks for %s on %d submodules", + prefix, + total, + ) + self._nvtx_hooks_by_use_key[key] = (module_id, hooks) + else: + hooks = existing[1] + + hooks.set_enabled(True) + self._active_nvtx_key = key + + def _disable_active_nvtx(self) -> None: + if self._active_nvtx_key is None: + return + existing = self._nvtx_hooks_by_use_key.get(self._active_nvtx_key) + if existing is not None: + existing[1].set_enabled(False) + self._active_nvtx_key = None + + def _remove_nvtx_hooks(self) -> None: + self._disable_active_nvtx() + for _, hooks in self._nvtx_hooks_by_use_key.values(): + hooks.remove_hooks() + self._nvtx_hooks_by_use_key.clear() + + @staticmethod + def _nvtx_prefix_for_use(use: ComponentUse) -> str: + parts = [use.stage_name, use.component_name] + if use.phase is not None and use.phase != use.component_name: + parts.append(use.phase) + return ".".join(parts) def _prefetch_use(self, use: ComponentUse) -> None: """Prepare a future component opportunistically without waiting. diff --git a/python/sglang/multimodal_gen/runtime/managers/scheduler.py b/python/sglang/multimodal_gen/runtime/managers/scheduler.py index a51de4f8afaa..02b51718ef48 100644 --- a/python/sglang/multimodal_gen/runtime/managers/scheduler.py +++ b/python/sglang/multimodal_gen/runtime/managers/scheduler.py @@ -156,6 +156,7 @@ def __init__( # warmup progress tracking self._warmup_total = 0 self._warmup_processed = 0 + self._logged_server_ready_after_warmup = False self.prepare_server_warmup_reqs() @@ -296,6 +297,11 @@ def _log_warmup_result(self, output_batch: OutputBatch, is_warmup: bool) -> None f"Warmup req processed in {GREEN}%.2f{RESET} seconds", total_duration_s, ) + if not self._logged_server_ready_after_warmup and ( + self._warmup_total <= 0 or self._warmup_processed >= self._warmup_total + ): + logger.info("The server is fired up and ready to roll!") + self._logged_server_ready_after_warmup = True else: if self._warmup_total > 0: logger.info( diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py index 89f5c14e971a..782e9902897a 100755 --- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py @@ -624,6 +624,8 @@ def __init__( added_kv_proj_dim: int | None = None, supported_attention_backends: set[AttentionBackendEnum] | None = None, prefix: str = "", + attention_type: str = "original", + sla_topk: float = 0.0, quant_config: QuantizationConfig | None = None, ): super().__init__() diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py index ee4750f3d747..a944bcb9948f 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py @@ -20,9 +20,7 @@ ) -# Scripting this brings model speed up 1.4x -@torch.jit.script -def snake(x, alpha): +def _snake(x, alpha): shape = x.shape x = x.reshape(shape[0], shape[1], -1) x = x + (alpha + 1e-9).reciprocal() * torch.sin(alpha * x).pow(2) @@ -30,12 +28,27 @@ def snake(x, alpha): return x +# Scripting this brings model speed up 1.4x +snake = torch.jit.script(_snake) + + +# ROCm HIPRTC can fail to compile the scripted bf16 Snake kernel. +def _should_use_eager_snake_on_rocm_bf16(x: torch.Tensor, alpha: torch.Tensor) -> bool: + return ( + torch.version.hip is not None + and (x.is_cuda or alpha.is_cuda) + and (x.dtype == torch.bfloat16 or alpha.dtype == torch.bfloat16) + ) + + class Snake1d(nn.Module): def __init__(self, channels): super().__init__() self.alpha = nn.Parameter(torch.ones(1, channels, 1)) def forward(self, x): + if _should_use_eager_snake_on_rocm_bf16(x, self.alpha): + return _snake(x, self.alpha) return snake(x, self.alpha) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_common_utils.py b/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_common_utils.py index 3d6e86ef0061..4d4fbde43c26 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_common_utils.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_common_utils.py @@ -7,6 +7,26 @@ from sglang.multimodal_gen.runtime.platforms import current_platform +def _channels_last_3d_supported_by_platform() -> bool: + return hasattr(torch, "channels_last_3d") and ( + current_platform.is_cuda() or current_platform.is_rocm() + ) + + +def _conv3d_weight_is_channels_last_3d(weight: torch.Tensor) -> bool: + return ( + weight.dim() == 5 + and _channels_last_3d_supported_by_platform() + and weight.is_contiguous(memory_format=torch.channels_last_3d) + ) + + +def match_conv3d_input_format(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + if x.dim() == 5 and _conv3d_weight_is_channels_last_3d(weight): + return x.contiguous(memory_format=torch.channels_last_3d) + return x + + class AvgDown3D(nn.Module): def __init__( self, @@ -151,6 +171,7 @@ def forward(self, x, cache_x=None): x = ( x if current_platform.is_amp_supported() else x.to(self.weight.dtype) ) # casting needed if amp isn't supported + x = match_conv3d_input_format(x, self.weight) return super().forward(x) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_dist_utils.py b/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_dist_utils.py index 5c2f5af32032..16f490172116 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_dist_utils.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/parallel/wan_dist_utils.py @@ -18,6 +18,7 @@ WanRMS_norm, WanUpsample, attention_block_forward, + match_conv3d_input_format, mid_block_forward, resample_forward, residual_block_forward, @@ -91,10 +92,21 @@ def split_for_parallel_decode( return x, expected_height +def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor: + if ( + x.dim() == 5 + and hasattr(torch, "channels_last_3d") + and x.is_contiguous(memory_format=torch.channels_last_3d) + and not x.is_contiguous() + ): + return x.contiguous() + return x + + def gather_and_trim_height(x: torch.Tensor, expected_height: int | None): if expected_height is None: return x - x = get_sp_group().all_gather(x, dim=-2) + x = get_sp_group().all_gather(_maybe_contiguous_for_sp_gather(x), dim=-2) if x.shape[-2] != expected_height: x = x[..., :expected_height, :].contiguous() return x @@ -323,6 +335,7 @@ def forward(self, x, cache_x=None): x_padded = x_padded[..., shift:, :] global_start += shift + x_padded = match_conv3d_input_format(x_padded, self.weight) out = super().forward(x_padded) if self.height_halo_size == 0: @@ -484,7 +497,7 @@ def __init__(self, dim) -> None: def forward(self, x): if self.world_size > 1: - x = self.sp_group.all_gather(x, dim=-2) + x = self.sp_group.all_gather(_maybe_contiguous_for_sp_gather(x), dim=-2) x = x.contiguous() x = attention_block_forward(self, x) if self.world_size > 1: diff --git a/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py index c227ef89ac6f..5842f65c66a7 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan3d_pipeline.py @@ -19,6 +19,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.hunyuan3d import ( Hunyuan3D2PipelineConfig, ) +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.loader.fsdp_load import ( load_model_from_full_model_state_dict, set_default_torch_dtype, @@ -58,6 +59,21 @@ class Hunyuan3D2Pipeline(ComposedPipelineBase): "hy3dshape_image_processor", ] + def validate_disagg_role(self, role: RoleType) -> None: + if role == RoleType.MONOLITHIC: + return + config = self.server_args.pipeline_config + if not isinstance(config, Hunyuan3D2PipelineConfig): + raise TypeError( + "Hunyuan3D2Pipeline requires Hunyuan3D2PipelineConfig, " + f"got {type(config)}" + ) + if config.paint_enable: + raise ValueError( + "Hunyuan3D2Pipeline only supports shape-only disaggregation. " + "Disable paint_enable when launching encoder/denoiser/decoder roles." + ) + def _load_config(self) -> dict[str, Any]: return { "_class_name": self.pipeline_name, @@ -357,6 +373,8 @@ def initialize_pipeline(self, server_args: ServerArgs): def create_pipeline_stages(self, server_args: ServerArgs): config = server_args.pipeline_config assert isinstance(config, Hunyuan3D2PipelineConfig) + latent_shape = tuple(config.vae_config.arch_config.latent_shape) + guidance_embed = bool(config.dit_config.arch_config.guidance_embed) # Shape: 4 stages self.add_stage( @@ -364,10 +382,10 @@ def create_pipeline_stages(self, server_args: ServerArgs): stage=Hunyuan3DShapeBeforeDenoisingStage( image_processor=self.get_module("hy3dshape_image_processor"), conditioner=self.get_module("hy3dshape_conditioner"), - vae=self.get_module("hy3dshape_vae"), - model=self.get_module("hy3dshape_model"), scheduler=self.get_module("hy3dshape_scheduler"), config=config, + latent_shape=latent_shape, + guidance_embed=guidance_embed, ), ) self.add_stage( diff --git a/python/sglang/multimodal_gen/runtime/pipelines/mova_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/mova_pipeline.py index 57e1c5029632..8707a657461a 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/mova_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/mova_pipeline.py @@ -63,7 +63,12 @@ def create_pipeline_stages(self, server_args: ServerArgs) -> None: self.add_stage(InputValidationStage()) self.add_standard_text_encoding_stage() if getattr(self.get_module("video_dit"), "require_vae_embedding", True): - self.add_stage(ImageVAEEncodingStage(vae=self.get_module("video_vae"))) + self.add_stage( + ImageVAEEncodingStage( + vae=self.get_module("video_vae"), + component_name="video_vae", + ) + ) self.add_stage( MOVALatentPreparationStage( audio_vae=self.get_module("audio_vae"), diff --git a/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py b/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py index c61c31864f98..ad1c0504a1ba 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/qwen_image.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from diffusers.image_processor import VaeImageProcessor +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( ComposedPipelineBase, @@ -115,9 +116,10 @@ class QwenImageLayeredPipeline(QwenImageEditPipeline): ] def create_pipeline_stages(self, server_args: ServerArgs): - self.add_stage( - QwenImageLayeredBeforeDenoisingStage( + def create_before_denoising_stage(): + return QwenImageLayeredBeforeDenoisingStage( vae=self.get_module("vae"), + text_encoder=None, tokenizer=self.get_module("tokenizer"), processor=self.get_module("processor"), transformer=self.get_module("transformer"), @@ -128,6 +130,11 @@ def create_pipeline_stages(self, server_args: ServerArgs): server_args.pipeline_config.text_encoder_precisions[0] ], ) + + self.add_stage_factory( + RoleType.ENCODER, + create_before_denoising_stage, + "QwenImageLayeredBeforeDenoisingStage", ) self.add_standard_timestep_preparation_stage( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py index 4995b679bd85..3bac67e46b96 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py @@ -99,6 +99,7 @@ def __init__( """ self.server_args = server_args self._disagg_role = server_args.disagg_role + self.validate_disagg_role(self._disagg_role) self.model_path: str = model_path self._stages: list[PipelineStage] = [] @@ -107,17 +108,26 @@ def __init__( self.executor = executor or self.build_executor(server_args=server_args) self.component_residency_manager: ComponentResidencyManager | None = None - if required_config_modules is not None: - self._required_config_modules = required_config_modules - - if self._required_config_modules is None: + base_required_config_modules = ( + required_config_modules + if required_config_modules is not None + else self._required_config_modules + ) + if base_required_config_modules is None: raise NotImplementedError("Subclass must set _required_config_modules") + self._required_config_modules = list(base_required_config_modules) + self._extra_config_module_map = dict(self._extra_config_module_map) # Filter modules based on disaggregation role if self._disagg_role != RoleType.MONOLITHIC: original_modules = list(self._required_config_modules) + task_name = self.server_args.pipeline_config.task_type.name.lower() self._required_config_modules = filter_modules_for_role( - self._required_config_modules, self._disagg_role + self._required_config_modules, + self._disagg_role, + extra_allowed_modules=self._get_extra_allowed_modules_for_role( + self._disagg_role, task_name + ), ) skipped = set(original_modules) - set(self._required_config_modules) if skipped: @@ -202,6 +212,44 @@ def initialize_pipeline(self, server_args: ServerArgs): """ return + def validate_disagg_role(self, role: RoleType) -> None: + """Validate whether the requested disaggregation role is supported.""" + return + + def _get_extra_allowed_modules_for_role( + self, role: RoleType, task_name: str + ) -> set[str]: + role_to_pipeline_modules: dict[RoleType, dict[str, set[str]]] = { + RoleType.ENCODER: { + "Flux2Pipeline": {"vae"}, + "Flux2KleinPipeline": {"vae"}, + "QwenImageEditPipeline": {"vae"}, + "QwenImageEditPlusPipeline": {"vae"}, + "QwenImageLayeredPipeline": {"vae", "transformer"}, + "GlmImagePipeline": {"vae", "transformer"}, + "WanImageToVideoPipeline": {"vae"}, + "WanImageToVideoDmdPipeline": {"vae"}, + "MOVA": {"video_vae", "audio_vae"}, + "MOVAPipeline": {"video_vae", "audio_vae"}, + }, + RoleType.DENOISER: {}, + RoleType.DECODER: {}, + } + extra_allowed_modules = set( + role_to_pipeline_modules.get(role, {}).get(self.pipeline_name, set()) + ) + + if role == RoleType.DENOISER and task_name == "ti2v": + if self.pipeline_name in { + "WanImageToVideoPipeline", + "WanImageToVideoDmdPipeline", + }: + extra_allowed_modules.add("vae") + elif self.pipeline_name == "LTX2Pipeline": + extra_allowed_modules.update({"vae", "audio_vae"}) + + return extra_allowed_modules + # --- Config-name → pipeline_config attribute mapping --- _CONFIG_ATTR_MAP: dict[str, str] = { "vae": "vae_config", @@ -495,6 +543,22 @@ def load_modules( self.memory_usages, round(current_platform.get_available_gpu_memory(), 2), ) + total_consumed_gb = sum( + usage + for usage in self.memory_usages.values() + if isinstance(usage, (int, float)) + ) + available_after_gb = current_platform.get_available_gpu_memory() + logger.debug( + "Module load summary: required_modules=%s loaded_modules=%s " + "memory_usages_gb=%s total_consumed_gb=%.2f " + "available_after_gb=%.2f", + list(required_modules), + list(loaded_components.keys()), + self.memory_usages, + total_consumed_gb, + available_after_gb, + ) return loaded_components @@ -502,6 +566,24 @@ def load_modules( def _infer_stage_name(stage: PipelineStage) -> str: return stage.__class__.__name__ + def _should_add_stage_for_role( + self, + role_affinity: RoleType, + stage_name: str, + ) -> bool: + if self._disagg_role == RoleType.MONOLITHIC: + return True + if role_affinity == self._disagg_role: + return True + + logger.info( + "Disagg role=%s: skipping stage %s (affinity=%s)", + self._disagg_role.value, + stage_name, + role_affinity.value, + ) + return False + def _profile_stage_name(self, stage: PipelineStage, stage_name: str) -> str: class_name = stage.__class__.__name__ if any(existing.__class__.__name__ == class_name for existing in self._stages): @@ -513,22 +595,13 @@ def add_stage( ) -> "ComposedPipelineBase": assert self.modules is not None, "No modules are registered" + if stage_name is None: + stage_name = self._infer_stage_name(stage) # Filter stages based on disaggregation role - if self._disagg_role != RoleType.MONOLITHIC: - if stage.role_affinity != self._disagg_role: - if stage_name is None: - stage_name = self._infer_stage_name(stage) - logger.info( - "Disagg role=%s: skipping stage %s (affinity=%s)", - self._disagg_role.value, - stage_name, - stage.role_affinity.value, - ) - return self + if not self._should_add_stage_for_role(stage.role_affinity, stage_name): + return self - if stage_name is None: - stage_name = self._infer_stage_name(stage) if stage_name in self._stage_name_mapping: raise ValueError(f"Duplicate stage name detected: {stage_name}") @@ -538,6 +611,17 @@ def add_stage( self._stage_name_mapping[stage_name] = stage return self + def add_stage_factory( + self, + role_affinity: RoleType, + stage_factory: Callable[[], PipelineStage], + stage_name: str, + ) -> "ComposedPipelineBase": + assert self.modules is not None, "No modules are registered" + if not self._should_add_stage_for_role(role_affinity, stage_name): + return self + return self.add_stage(stage_factory(), stage_name) + def add_stages( self, stages: list[PipelineStage | tuple[PipelineStage, str]] ) -> "ComposedPipelineBase": @@ -579,12 +663,12 @@ def add_standard_text_encoding_stage( def add_standard_timestep_preparation_stage( self, scheduler_key: str = "scheduler", - prepare_extra_kwargs: list[Callable] | None = [], + prepare_extra_kwargs: list[Callable] | None = None, ) -> "ComposedPipelineBase": return self.add_stage( TimestepPreparationStage( scheduler=self.get_module(scheduler_key), - prepare_extra_set_timesteps_kwargs=prepare_extra_kwargs, + prepare_extra_set_timesteps_kwargs=list(prepare_extra_kwargs or []), ), ) @@ -606,43 +690,57 @@ def add_standard_denoising_stage( transformer_2_key: str | None = "transformer_2", scheduler_key: str = "scheduler", vae_key: str | None = "vae", + stage_name: str = "denoising_stage", ) -> "ComposedPipelineBase": - kwargs = { - "transformer": self.get_module(transformer_key), - "scheduler": self.get_module(scheduler_key), - } - - if transformer_2_key: - transformer_2 = self.get_module(transformer_2_key, None) - if transformer_2 is not None: - kwargs["transformer_2"] = transformer_2 - - if vae_key: - vae = self.get_module(vae_key, None) - if vae is not None: - kwargs["vae"] = vae - kwargs["pipeline"] = self - - return self.add_stage(DenoisingStage(**kwargs)) + def create_stage() -> PipelineStage: + kwargs = { + "transformer": self.get_module(transformer_key), + "scheduler": self.get_module(scheduler_key), + } + + if transformer_2_key: + transformer_2 = self.get_module(transformer_2_key, None) + if transformer_2 is not None: + kwargs["transformer_2"] = transformer_2 + + if vae_key: + vae = self.get_module(vae_key, None) + if vae is not None: + kwargs["vae"] = vae + kwargs["pipeline"] = self + + return DenoisingStage(**kwargs) + + return self.add_stage_factory( + RoleType.DENOISER, + create_stage, + stage_name, + ) def add_standard_decoding_stage( self, vae_key: str = "vae", + stage_name: str = "decoding_stage", ) -> "ComposedPipelineBase": - return self.add_stage( - DecodingStage( + def create_stage() -> PipelineStage: + return DecodingStage( vae=self.get_module(vae_key), pipeline=self, component_name=vae_key, - ), + ) + + return self.add_stage_factory( + RoleType.DECODER, + create_stage, + stage_name, ) def add_standard_t2i_stages( self, include_input_validation: bool = True, - prepare_extra_timestep_kwargs: list[Callable] | None = [], + prepare_extra_timestep_kwargs: list[Callable] | None = None, ) -> "ComposedPipelineBase": if include_input_validation: @@ -671,7 +769,7 @@ def add_standard_ti2i_stages( prompt_text_encoder_key: str = "text_encoder", image_vae_key: str = "vae", image_vae_stage_kwargs: dict[str, Any] | None = None, - prepare_extra_timestep_kwargs: list[Callable] | None = [], + prepare_extra_timestep_kwargs: list[Callable] | None = None, ) -> "ComposedPipelineBase": if include_input_validation: self.add_stage( @@ -696,7 +794,10 @@ def add_standard_ti2i_stages( self.add_stage( ImageVAEEncodingStage( vae=self.get_module(image_vae_key), - **(image_vae_stage_kwargs or {}), + **{ + "component_name": image_vae_key, + **(image_vae_stage_kwargs or {}), + }, ), ) @@ -723,8 +824,9 @@ def add_standard_ti2v_stages( image_vae_encoding_position: Literal[ "before_timestep", "after_latent" ] = "before_timestep", - prepare_extra_timestep_kwargs: list[Callable] | None = [], + prepare_extra_timestep_kwargs: list[Callable] | None = None, denoising_stage_factory: Callable[[], PipelineStage] | None = None, + denoising_stage_name: str = "denoising_stage", ) -> "ComposedPipelineBase": if include_input_validation: self.add_stage( @@ -750,7 +852,10 @@ def add_standard_ti2v_stages( self.add_stage( ImageVAEEncodingStage( vae=self.get_module(image_vae_key), - **(image_vae_stage_kwargs or {}), + **{ + "component_name": image_vae_key, + **(image_vae_stage_kwargs or {}), + }, ) ) @@ -762,7 +867,10 @@ def add_standard_ti2v_stages( self.add_stage( ImageVAEEncodingStage( vae=self.get_module(image_vae_key), - **(image_vae_stage_kwargs or {}), + **{ + "component_name": image_vae_key, + **(image_vae_stage_kwargs or {}), + }, ) ) elif image_vae_encoding_position != "before_timestep": @@ -773,7 +881,11 @@ def add_standard_ti2v_stages( if denoising_stage_factory is None: self.add_standard_denoising_stage() else: - self.add_stage(denoising_stage_factory()) + self.add_stage_factory( + RoleType.DENOISER, + denoising_stage_factory, + denoising_stage_name, + ) self.add_standard_decoding_stage() return self diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py index acaac0f6804d..48ffee232c4d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py @@ -4,7 +4,6 @@ import torch -from sglang.multimodal_gen.runtime.distributed import get_sp_group from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_cfg_group, get_classifier_free_guidance_rank, @@ -33,26 +32,6 @@ class ParallelExecutor(PipelineExecutor): """ - def collect_from_main(self, batches: list[Req]): - - # TODO: fix this condition - if self.server_args.sp_degree != 1: - sp_group = get_sp_group() - batches = broadcast_pyobj( - batches, - sp_group.rank, - sp_group.cpu_group, - src=sp_group.ranks[0], - ) - - if self.server_args.enable_cfg_parallel: - batches = broadcast_pyobj( - batches, - self.worker.cfg_group.rank, - self.worker.cfg_cpu_group, - src=self.worker.cfg_group.ranks[0], - ) - def _execute_stages( self, stages: List[PipelineStage], @@ -68,8 +47,9 @@ def _execute_stages( cfg_group = get_cfg_group() group = get_world_group() - self.begin_component_residency_request(stages, batch, server_args) - try: + use_nvtx = self._should_use_stage_nvtx(batch, server_args) + + with self._component_residency_request(stages, batch, server_args): # TODO: decide when to gather on main when CFG_PARALLEL -> MAIN_RANK_ONLY for stage_index, stage in enumerate(stages): paradigm = stage.parallelism_type @@ -77,11 +57,14 @@ def _execute_stages( if paradigm == StageParallelismType.MAIN_RANK_ONLY: if rank == 0: # Only main rank executes, others just wait - self.before_stage(stage, stage_index, batch, server_args) - batch = self.run_stage_with_context( - stage, batch, server_args, run_stage + batch = self._run_stage_with_executor_hooks( + stage, + stage_index, + batch, + server_args, + run_stage, + use_nvtx, ) - self.after_stage(stage_index) torch.distributed.barrier() elif paradigm == StageParallelismType.CFG_PARALLEL: @@ -95,28 +78,37 @@ def _execute_stages( ) if rank != 0: batch = broadcasted_list[0] - self.before_stage(stage, stage_index, batch, server_args) - batch = self.run_stage_with_context( - stage, batch, server_args, run_stage + batch = self._run_stage_with_executor_hooks( + stage, + stage_index, + batch, + server_args, + run_stage, + use_nvtx, ) - self.after_stage(stage_index) torch.distributed.barrier() elif paradigm == StageParallelismType.REPLICATED: - self.before_stage(stage, stage_index, batch, server_args) - batch = self.run_stage_with_context( - stage, batch, server_args, run_stage + batch = self._run_stage_with_executor_hooks( + stage, + stage_index, + batch, + server_args, + run_stage, + use_nvtx, ) - self.after_stage(stage_index) elif paradigm == StageParallelismType.MAIN_RANK_ONLY_AND_SEND_TO_OTHERS: if rank == 0: # Only main rank executes, others just wait - self.before_stage(stage, stage_index, batch, server_args) - batch = self.run_stage_with_context( - stage, batch, server_args, run_stage + batch = self._run_stage_with_executor_hooks( + stage, + stage_index, + batch, + server_args, + run_stage, + use_nvtx, ) - self.after_stage(stage_index) torch.distributed.barrier() # Send batch to other ranks @@ -127,8 +119,6 @@ def _execute_stages( if rank != 0: batch = broadcasted_list[0] torch.distributed.barrier() - finally: - self.finish_component_residency_request() return batch def execute( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py index e4960df973e8..be94ccde5780 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py @@ -7,7 +7,7 @@ import contextlib from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Any, Callable, List import torch @@ -16,6 +16,7 @@ from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.nvtx_pytorch_hooks import maybe_nvtx_range from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler from sglang.multimodal_gen.runtime.utils.profiler import SGLDiffusionProfiler @@ -53,7 +54,7 @@ def __init__(self, server_args): def begin_component_residency_request( self, stages: List["PipelineStage"], - batch: Req, + batch: Any, server_args: ServerArgs, ) -> None: self.component_residency_manager.begin_request(stages, batch, server_args) @@ -62,7 +63,7 @@ def before_stage( self, stage: "PipelineStage", stage_index: int, - batch: Req, + batch: Any, server_args: ServerArgs, ) -> None: stage.set_component_residency_manager(self.component_residency_manager) @@ -76,6 +77,56 @@ def after_stage(self, stage_index: int) -> None: def finish_component_residency_request(self) -> None: self.component_residency_manager.finish_request() + @contextlib.contextmanager + def _component_residency_request( + self, + stages: List["PipelineStage"], + payload: Any, + server_args: ServerArgs, + ): + self.begin_component_residency_request(stages, payload, server_args) + try: + yield + finally: + self.finish_component_residency_request() + + @staticmethod + def _is_warmup_payload(payload: Any) -> bool: + if isinstance(payload, list): + return bool(payload) and all( + getattr(item, "is_warmup", False) for item in payload + ) + return getattr(payload, "is_warmup", False) + + def _should_use_stage_nvtx(self, payload: Any, server_args: ServerArgs) -> bool: + return server_args.enable_layerwise_nvtx_marker and not self._is_warmup_payload( + payload + ) + + def _run_stage_with_executor_hooks( + self, + stage: "PipelineStage", + stage_index: int, + payload: Any, + server_args: ServerArgs, + run_stage: Callable[["PipelineStage", Any], Any], + use_nvtx: bool, + ) -> Any: + stage_name = stage._component_stage_name() + self.before_stage(stage, stage_index, payload, server_args) + with maybe_nvtx_range(f"stage_{stage_name}", use_nvtx): + payload = self.run_stage_with_context( + stage, payload, server_args, run_stage + ) + self.after_stage(stage_index) + return payload + + @staticmethod + def _step_stage_profiler() -> None: + profiler = SGLDiffusionProfiler.get_instance() + if profiler: + profiler.step_stage() + def execute_with_profiling( self, stages: List["PipelineStage"], diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py index d190360804e5..f9ece80daa6e 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py @@ -9,7 +9,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor import ( PipelineExecutor, - SGLDiffusionProfiler, ) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req from sglang.multimodal_gen.runtime.pipelines_core.stages import PipelineStage @@ -29,19 +28,19 @@ def _run_profile_all_stages( run_stage: Callable[[PipelineStage, Any], Any], ) -> Any: """Execute all pipeline stages sequentially and step the profiler.""" - self.begin_component_residency_request(stages, payload, server_args) - try: + + use_nvtx = self._should_use_stage_nvtx(payload, server_args) + with self._component_residency_request(stages, payload, server_args): for stage_index, stage in enumerate(stages): - self.before_stage(stage, stage_index, payload, server_args) - payload = self.run_stage_with_context( - stage, payload, server_args, run_stage + payload = self._run_stage_with_executor_hooks( + stage, + stage_index, + payload, + server_args, + run_stage, + use_nvtx, ) - self.after_stage(stage_index) - profiler = SGLDiffusionProfiler.get_instance() - if profiler: - profiler.step_stage() - finally: - self.finish_component_residency_request() + self._step_stage_profiler() return payload def run_profile_all_stages( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py index d7979cff3981..5c902c406899 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/lora_pipeline.py @@ -4,7 +4,7 @@ import os from collections import defaultdict from collections.abc import Hashable -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from typing import Any import torch @@ -19,6 +19,9 @@ wrap_with_lora_layer, ) from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + is_layerwise_offloaded_module, +) from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( ComposedPipelineBase, ) @@ -170,10 +173,6 @@ def _temporarily_disable_offload( Yields: List of modules that had offload disabled. """ - from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( - is_layerwise_offloaded_module, - ) - module_names = [] if target_modules is not None: # Extract module names from target_modules @@ -215,6 +214,22 @@ def _temporarily_disable_offload( for module in offload_disabled_modules: module.enable_offload() + def _needs_lora_weight_update_context( + self, + target_modules: list[tuple[str, dict[str, BaseLayerWithLoRA]]], + merge_weights_by_module: dict[str, bool], + ) -> bool: + + for module_name, lora_layers_dict in target_modules: + if merge_weights_by_module[module_name]: + return True + if any(layer.merged for layer in lora_layers_dict.values()): + return True + module = self.modules.get(module_name) + if module is not None and is_layerwise_offloaded_module(module): + return True + return False + def convert_module_lora_layers( self, module: torch.nn.Module, @@ -578,6 +593,58 @@ def _apply_lora_to_layers( ) return adapted_count + def _reactivate_cached_dynamic_lora_layers( + self, + lora_layers: dict[str, BaseLayerWithLoRA], + lora_nicknames: list[str], + lora_paths: list[str | None], + strengths: list[float], + ) -> int | None: + """ + Re-enable a previously applied dynamic LoRA without rebuilding per-layer state. + + Dynamic LoRA keeps adapter tensors on the wrapped layers. When a later stage only + disables them and the next stage asks for the same single adapter again, toggling + `disable_lora` is enough; the stored A/B tensors, rank, alpha, and strength still + describe the requested adapter. + """ + if len(lora_nicknames) != 1: + return None + + nickname = lora_nicknames[0] + strength = strengths[0] + adapter = self.lora_adapters.get(nickname) + if adapter is None: + return None + path = lora_paths[0] or self.loaded_adapter_paths.get(nickname) + if path is None: + return None + + active_count = 0 + for name, layer in lora_layers.items(): + if layer.merged or len(layer.lora_weights_list) != 1: + return None + has_adapter = name + ".lora_A" in adapter and name + ".lora_B" in adapter + if not has_adapter: + continue + if ( + layer.lora_A is None + or layer.lora_B is None + or layer.lora_path != path + or layer.strength != strength + ): + return None + active_count += 1 + + if active_count == 0: + return None + + for name, layer in lora_layers.items(): + has_adapter = name + ".lora_A" in adapter and name + ".lora_B" in adapter + layer.disable_lora = not has_adapter + + return active_count + def is_lora_effective(self, target: str = "all") -> bool: """ Check if LoRA is currently effective for the specified target. @@ -756,58 +823,79 @@ def set_lora( if not target_modules: continue - # Disable layerwise offload if enabled: load all layers to GPU - # the LoRA weights merging process requires weights being on device - with self._temporarily_disable_offload(target_modules=target_modules): - tgt_nicknames = [lora_nicknames[i] for i in idx_list] - tgt_paths = [lora_paths[i] for i in idx_list] - tgt_strengths = [strengths[i] for i in idx_list] - - merged_name = ( - ",".join(tgt_nicknames) - if len(tgt_nicknames) > 1 - else tgt_nicknames[0] - ) + tgt_nicknames = [lora_nicknames[i] for i in idx_list] + tgt_paths = [lora_paths[i] for i in idx_list] + tgt_strengths = [strengths[i] for i in idx_list] + + merged_name = ( + ",".join(tgt_nicknames) if len(tgt_nicknames) > 1 else tgt_nicknames[0] + ) - # Skip if LoRA configuration matches exactly (including order and strength) - # Since all modules for the same target apply the same config, checking one is sufficient - first_module_name, first_lora_layers_dict = target_modules[0] - first_effective_merge_weights = self._should_merge_lora_for_layers( - first_module_name, first_lora_layers_dict, merge_mode + # Skip if LoRA configuration matches exactly (including order and strength) + # Since all modules for the same target apply the same config, checking one is sufficient + first_module_name, first_lora_layers_dict = target_modules[0] + first_effective_merge_weights = self._should_merge_lora_for_layers( + first_module_name, first_lora_layers_dict, merge_mode + ) + if not first_effective_merge_weights and len(tgt_nicknames) > 1: + raise ValueError( + "Dynamic LoRA currently supports only one adapter per target. " + "Use merge_mode='merge' for multiple adapters." ) - if not first_effective_merge_weights and len(tgt_nicknames) > 1: - raise ValueError( - "Dynamic LoRA currently supports only one adapter per target. " - "Use merge_mode='merge' for multiple adapters." + + merge_weights_by_module = {} + for module_name, lora_layers_dict in target_modules: + merge_weights_by_module[module_name] = ( + first_effective_merge_weights + if module_name == first_module_name + else self._should_merge_lora_for_layers( + module_name, lora_layers_dict, merge_mode ) - if self._check_lora_config_matches( - first_module_name, - tgt_nicknames, - tgt_strengths, - first_effective_merge_weights, - adapter_updated, - ): - logger.info("LoRA configuration matches exactly, skipping") - continue + ) + + if self._check_lora_config_matches( + first_module_name, + tgt_nicknames, + tgt_strengths, + first_effective_merge_weights, + adapter_updated, + ): + logger.info("LoRA configuration matches exactly, skipping") + continue + + # merged LoRA and offloaded modules update backing weights; dynamic + # reactivation only toggles wrapper metadata when cached tensors match + if self._needs_lora_weight_update_context( + target_modules, merge_weights_by_module + ): + weight_update_context = self._temporarily_disable_offload( + target_modules=target_modules + ) + else: + weight_update_context = nullcontext() + with weight_update_context: # Apply LoRA to modules for this target for module_name, lora_layers_dict in target_modules: - effective_merge_weights = ( - first_effective_merge_weights - if module_name == first_module_name - else self._should_merge_lora_for_layers( - module_name, lora_layers_dict, merge_mode + effective_merge_weights = merge_weights_by_module[module_name] + count = None + if not effective_merge_weights and not adapter_updated: + count = self._reactivate_cached_dynamic_lora_layers( + lora_layers_dict, + tgt_nicknames, + tgt_paths, + tgt_strengths, + ) + if count is None: + count = self._apply_lora_to_layers( + lora_layers_dict, + tgt_nicknames, + tgt_paths, + rank, + tgt_strengths, + clear_existing=True, + merge_weights=effective_merge_weights, ) - ) - count = self._apply_lora_to_layers( - lora_layers_dict, - tgt_nicknames, - tgt_paths, - rank, - tgt_strengths, - clear_existing=True, - merge_weights=effective_merge_weights, - ) adapted_count += count self.cur_adapter_name[module_name] = merged_name self.cur_adapter_path[module_name] = ",".join( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py index 1044e95fc3a2..c004ad728c5c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py @@ -59,6 +59,10 @@ class PipelineStage(StageDedupMixin, ABC): for a specific part of the process, such as prompt encoding, latent preparation, etc. """ + # Class-level default so subclasses that override __init__ without + # calling super().__init__() still see a consistent explicit-range gate. + _current_use_nvtx: bool = False + def __init__(self): self.server_args = get_global_server_args() self._component_residency_manager = None @@ -133,6 +137,12 @@ def _component_stage_name(self, stage_name: str | None = None) -> str: ) def _active_component_stage_name(self) -> str: + """Stage name reported by the residency manager. + + Only valid between ``before_stage`` and ``after_stage``; outside + that window the manager state still holds the previous stage's + name. Use :meth:`_component_stage_name` for the static identity. + """ manager = getattr(self, "_component_residency_manager", None) manager_state = getattr(manager, "state", None) manager_stage_name = getattr(manager_state, "stage_name", None) @@ -206,6 +216,26 @@ def component_uses( """Declares component uses of current stage for unified residency scheduling.""" return [] + def _apply_nvtx_gate(self, is_warmup: bool) -> bool: + """Resolve the per-request NVTX gate for explicit stage ranges. + + Layerwise module hooks are registered at component use-sites by + ``ComponentResidencyManager``. Stages use this value only for + explicit ``maybe_nvtx_range`` blocks. + """ + use_nvtx = self.server_args.enable_layerwise_nvtx_marker and not is_warmup + self._current_use_nvtx = use_nvtx + return use_nvtx + + @property + def current_use_nvtx(self) -> bool: + """Last resolved ``use_nvtx`` value from :meth:`_apply_nvtx_gate`. + + ``forward`` implementations can read this to gate explicit + ``maybe_nvtx_range`` blocks without re-evaluating the flag. + """ + return self._current_use_nvtx + # Default role affinity: ENCODER. Override in subclasses for DENOISING/DECODER. @property def role_affinity(self) -> RoleType: @@ -297,16 +327,23 @@ def __call__( logger.error("Input verification failed for %s: %s", stage_name, str(e)) raise - # Execute the actual stage logic with unified profiling - with StageProfiler( - stage_name, - logger=logger, - metrics=batch.metrics, - log_stage_start_end=not batch.is_warmup - and not (self.server_args and self.server_args.comfyui_mode), - perf_dump_path_provided=batch.perf_dump_path is not None, - ): - result = self.forward(batch, server_args) + # Resolve the NVTX gate once per call. Component-level hooks are + # attached by the residency manager at the actual component use-site. + self._apply_nvtx_gate(batch.is_warmup) + + # Execute the actual stage logic with unified profiling. + try: + with StageProfiler( + stage_name, + logger=logger, + metrics=batch.metrics, + log_stage_start_end=not batch.is_warmup + and not (self.server_args and self.server_args.comfyui_mode), + perf_dump_path_provided=batch.perf_dump_path is not None, + ): + result = self.forward(batch, server_args) + finally: + self._current_use_nvtx = False # Post-execution output verification try: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index 1f598d0d9eee..4b533ed900dc 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -98,6 +98,7 @@ ) from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.nvtx_pytorch_hooks import maybe_nvtx_range from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler from sglang.multimodal_gen.runtime.utils.profiler import SGLDiffusionProfiler from sglang.multimodal_gen.utils import PRECISION_TO_TYPE, dict_to_3d_list @@ -221,12 +222,20 @@ def _infer_transformer_attention_backend(self) -> AttentionBackendEnum | None: if not backends: return None if len(backends) > 1: + sparse_backends = {backend for backend in backends if backend.is_sparse} + selected_backend = ( + sorted(sparse_backends, key=lambda backend: backend.name)[0] + if sparse_backends + else sorted(backends, key=lambda backend: backend.name)[0] + ) logger.warning( "Multiple transformer attention backends detected: %s. " - "Using one backend for denoising metadata.", + "Using %s for denoising metadata.", sorted(backend.name.lower() for backend in backends), + selected_backend.name.lower(), ) - return sorted(backends, key=lambda backend: backend.name)[0] + return selected_backend + return next(iter(backends)) def component_uses( self, server_args: ServerArgs, stage_name: str | None = None @@ -796,6 +805,7 @@ def _before_denoising_loop( """Prepare scheduler state before entering the shared denoising loop.""" self._reset_scheduler_loop_state(ctx.scheduler) ctx.scheduler.set_begin_index(0) + self._init_cfg_gate_state(ctx, batch, server_args) def _reset_scheduler_loop_state(self, scheduler) -> None: if hasattr(scheduler, "_step_index"): @@ -816,6 +826,55 @@ def _reset_scheduler_loop_state(self, scheduler) -> None: if hasattr(scheduler, "timestep_list"): scheduler.timestep_list = [None] * solver_order + def _init_cfg_gate_state( + self, ctx: DenoisingContext, batch: Req, server_args: ServerArgs + ) -> None: + """Initialize optional CFG residual reuse for the current denoising loop.""" + fraction = envs.SGLANG_DIFFUSION_CFG_GATE_STEP + if not 0.0 <= fraction <= 1.0: + raise ValueError( + "SGLANG_DIFFUSION_CFG_GATE_STEP must be between 0.0 and 1.0, " + f"got {fraction}." + ) + + num_steps = len(ctx.timesteps) + requested = fraction < 1.0 and batch.do_classifier_free_guidance + active = requested and not server_args.enable_cfg_parallel + gate_step = int(num_steps * fraction) if active else num_steps + 1 + ctx.extra["cfg_gate_state"] = { + "fraction": fraction, + "requested": requested, + "active": active, + "gate_step": gate_step, + "delta": None, + "model_id": None, + "fresh_uncond": 0, + "reused": 0, + "invalidations": 0, + } + + if ctx.is_warmup or get_world_group().local_rank != 0: + return + + if active: + logger.info( + "CFG gating enabled: reuse unconditioned residual after step %d/%d " + "(fraction=%.3f).", + gate_step, + num_steps, + fraction, + ) + if batch.guidance_rescale > 0: + logger.warning( + "CFG gating is enabled with guidance_rescale=%s; benchmark image " + "quality before using this setting in production.", + batch.guidance_rescale, + ) + elif requested: + logger.info( + "CFG gating requested but skipped because CFG parallel is enabled." + ) + def _get_transformer_attr(self, name: str) -> Any: seen: set[int] = set() stack = [self.transformer] @@ -912,8 +971,13 @@ def _run_denoising_step( ) -> None: """Run one scheduler-backed denoising step in the shared base path. - Model-specific stages should override this instead of the whole loop whenever possible to achieve better performance + Model-specific stages should override this instead of the whole loop + whenever possible to achieve better performance. Overrides that bypass + ``_predict_noise_with_cfg`` / ``ctx.scheduler.step`` will lose the + inner ``predict_noise`` / ``scheduler_step`` NVTX markers emitted + below; mirror them in the override if those markers are needed. """ + use_nvtx = self.current_use_nvtx # 1. Prepare latent inputs in the model's compute dtype. latent_model_input = ctx.latents.to(ctx.target_dtype) if batch.image_latent is not None: @@ -940,31 +1004,34 @@ def _run_denoising_step( ) # 4. Run the model prediction path, including CFG when enabled. - noise_pred = self._predict_noise_with_cfg( - current_model=step.current_model, - latent_model_input=latent_model_input, - timestep=timestep, - batch=batch, - timestep_index=step.step_index, - attn_metadata=step.attn_metadata, - target_dtype=ctx.target_dtype, - current_guidance_scale=step.current_guidance_scale, - cfg_policy=ctx.cfg_policy, - server_args=server_args, - guidance=ctx.guidance, - latents=ctx.latents, - ) + with maybe_nvtx_range("predict_noise", use_nvtx): + noise_pred = self._predict_noise_with_cfg( + current_model=step.current_model, + latent_model_input=latent_model_input, + timestep=timestep, + batch=batch, + timestep_index=step.step_index, + attn_metadata=step.attn_metadata, + target_dtype=ctx.target_dtype, + current_guidance_scale=step.current_guidance_scale, + cfg_policy=ctx.cfg_policy, + cfg_gate_state=ctx.extra.get("cfg_gate_state"), + server_args=server_args, + guidance=ctx.guidance, + latents=ctx.latents, + ) if server_args.comfyui_mode: batch.noise_pred = noise_pred # 5. Advance the scheduler state with the predicted noise. - ctx.latents = ctx.scheduler.step( - model_output=noise_pred, - timestep=step.t_device, - sample=ctx.latents, - **ctx.extra_step_kwargs, - return_dict=False, - )[0] + with maybe_nvtx_range("scheduler_step", use_nvtx): + ctx.latents = ctx.scheduler.step( + model_output=noise_pred, + timestep=step.t_device, + sample=ctx.latents, + **ctx.extra_step_kwargs, + return_dict=False, + )[0] # 6. Re-apply any model-specific latent constraints after the update. ctx.latents = self.post_forward_for_ti2v_task( @@ -992,6 +1059,7 @@ def _finalize_denoising_loop( self, ctx: DenoisingContext, batch: Req, server_args: ServerArgs ) -> None: """Finalize the shared loop by handing state to post-denoising processing.""" + self._log_cfg_gate_summary(ctx, batch) self._post_denoising_loop( batch=batch, latents=ctx.latents, @@ -1001,6 +1069,28 @@ def _finalize_denoising_loop( is_warmup=ctx.is_warmup, ) + def _log_cfg_gate_summary(self, ctx: DenoisingContext, batch: Req) -> None: + state = ctx.extra.get("cfg_gate_state") + if ( + not state + or not state["requested"] + or ctx.is_warmup + or get_world_group().local_rank != 0 + ): + return + + logger.info( + "CFG gating summary: fraction=%.3f, gate_step=%d/%d, " + "fresh_uncond=%d, reused=%d, invalidations=%d, guidance_rescale=%s.", + state["fraction"], + state["gate_step"], + len(ctx.timesteps), + state["fresh_uncond"], + state["reused"], + state["invalidations"], + batch.guidance_rescale, + ) + def _post_denoising_loop( self, batch: Req, @@ -1062,6 +1152,10 @@ def _post_denoising_loop( "Memory before deallocating transformer: %s", torch.mps.current_allocated_memory(), ) + if self._component_residency_manager is not None: + self._component_residency_manager.remove_nvtx_hooks_for_module( + self.transformer + ) del self.transformer if pipeline is not None and "transformer" in pipeline.modules: del pipeline.modules["transformer"] @@ -1167,7 +1261,7 @@ def _manage_dit_use_site( preferred_ready_after_request=component_name == "transformer", memory_intensive=True, ) - manager.begin_use(use) + manager.begin_use(use, module=current_model) def _select_and_manage_model( self, @@ -1252,19 +1346,33 @@ def forward( # to avoid device-sync caused by timestep comparison timesteps_cpu = ctx.timesteps.cpu() num_timesteps = timesteps_cpu.shape[0] - with torch.autocast( - device_type=current_platform.device_type, - dtype=ctx.target_dtype, - enabled=ctx.autocast_enabled, + # Re-resolve the explicit-range gate so the per-step markers + # below honor this request's is_warmup state. Layer hooks are + # registered by the residency manager at the use-site. + use_nvtx = self._apply_nvtx_gate(ctx.is_warmup) + + with ( + torch.autocast( + device_type=current_platform.device_type, + dtype=ctx.target_dtype, + enabled=ctx.autocast_enabled, + ), + maybe_nvtx_range("denoising_loop", use_nvtx), ): with self.progress_bar(total=ctx.num_inference_steps) as progress_bar: for step_index, t_host in enumerate(timesteps_cpu): - with StageProfiler( - f"denoising_step_{step_index}", - logger=logger, - metrics=batch.metrics, - perf_dump_path_provided=batch.perf_dump_path is not None, - record_as_step=True, + # Use ``:.4g`` so flow-matching schedulers (e.g. FLUX) that + # use non-integer timesteps keep their precision in markers. + step_marker = f"denoising_step_{step_index}_t{t_host.item():.4g}" + with ( + maybe_nvtx_range(step_marker, use_nvtx), + StageProfiler( + f"denoising_step_{step_index}", + logger=logger, + metrics=batch.metrics, + perf_dump_path_provided=batch.perf_dump_path is not None, + record_as_step=True, + ), ): step = self._prepare_step_state( ctx, @@ -1390,6 +1498,7 @@ def _predict_noise_with_cfg( target_dtype, current_guidance_scale, cfg_policy: CFGPolicy, + cfg_gate_state: dict[str, Any] | None, server_args: ServerArgs, guidance: torch.Tensor, latents: torch.Tensor, @@ -1421,6 +1530,48 @@ def predict_fn(branch): ) return _unwrap(pred_t) + if ( + cfg_gate_state + and cfg_gate_state["active"] + and not server_args.enable_cfg_parallel + and type(cfg_policy) is CFGPolicy + and len(cfg_policy.branches) == 2 + and cfg_policy.branches[0].is_conditional + and not cfg_policy.branches[1].is_conditional + ): + model_id = id(current_model) + if cfg_gate_state["model_id"] not in (None, model_id): + cfg_gate_state["delta"] = None + cfg_gate_state["invalidations"] += 1 + + pos_pred = predict_fn(cfg_policy.branches[0]) + pos_t = _wrap(pos_pred) + delta_t = cfg_gate_state["delta"] + can_reuse = ( + timestep_index >= cfg_gate_state["gate_step"] + and delta_t is not None + and len(pos_t) == len(delta_t) + ) + + if can_reuse: + neg_pred = _unwrap(tuple(p - d for p, d in zip(pos_t, delta_t))) + cfg_gate_state["reused"] += 1 + else: + neg_pred = predict_fn(cfg_policy.branches[1]) + neg_t = _wrap(neg_pred) + cfg_gate_state["delta"] = tuple( + p.detach() - n.detach() for p, n in zip(pos_t, neg_t) + ) + cfg_gate_state["model_id"] = model_id + cfg_gate_state["fresh_uncond"] += 1 + + return cfg_policy.combine( + [pos_pred, neg_pred], + batch, + cfg_scale, + server_args.pipeline_config, + ) + if server_args.enable_cfg_parallel: if ( len(cfg_policy.branches) == 2 @@ -1474,12 +1625,16 @@ def _build_attn_metadata( self.attn_backend.get_enum() == AttentionBackendEnum.SLIDING_TILE_ATTN or self.attn_backend.get_enum() == AttentionBackendEnum.VIDEO_SPARSE_ATTN ): + attention_backend_config = server_args.attention_backend_config or {} + vsa_sparsity = attention_backend_config.get( + "VSA_sparsity", attention_backend_config.get("sparsity", 0.0) + ) attn_metadata = self.attn_metadata_builder.build( current_timestep=i, raw_latent_shape=batch.raw_latent_shape[2:5], patch_size=server_args.pipeline_config.dit_config.patch_size, STA_param=batch.STA_param, - VSA_sparsity=server_args.attention_backend_config.VSA_sparsity, + VSA_sparsity=vsa_sparsity, device=get_local_torch_device(), ) elif ( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/hunyuan3d_shape.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/hunyuan3d_shape.py index d8832925bcef..0ef0f1fca025 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/hunyuan3d_shape.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/hunyuan3d_shape.py @@ -128,18 +128,18 @@ def __init__( self, image_processor: Any, conditioner: Any, - vae: Any, - model: Any, scheduler: Any, config: Hunyuan3D2PipelineConfig, + latent_shape: tuple[int, ...], + guidance_embed: bool, ) -> None: super().__init__() self.image_processor = image_processor self.conditioner = conditioner - self.vae = vae - self.model = model self.scheduler = scheduler self.config = config + self.latent_shape = latent_shape + self.guidance_embed = guidance_embed def _validate_input(self, batch: Req, server_args: ServerArgs) -> None: if batch.image_path is None: @@ -160,10 +160,41 @@ def _validate_input(self, batch: Req, server_args: ServerArgs) -> None: def _prepare_latents(self, batch_size, dtype, device, generator, scheduler): from diffusers.utils.torch_utils import randn_tensor - shape = (batch_size, *self.vae.latent_shape) + shape = (batch_size, *self.latent_shape) latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) return latents * getattr(scheduler, "init_noise_sigma", 1.0) + def _find_conditioner_dtype(self, items_fn_name: str) -> torch.dtype | None: + items_fn = getattr(self.conditioner, items_fn_name, None) + if not callable(items_fn): + return None + try: + for item in items_fn(): + if isinstance(item, torch.Tensor) and torch.is_floating_point(item): + return item.dtype + except TypeError as exc: + logger.warning( + "Failed to inspect Hunyuan3D conditioner %s() for runtime dtype; " + "falling back to the sample tensor dtype. error=%s", + items_fn_name, + exc, + ) + return None + + def _resolve_runtime_dtype( + self, sample_tensor: torch.Tensor | None = None + ) -> torch.dtype: + for items_fn_name in ("parameters", "buffers"): + dtype = self._find_conditioner_dtype(items_fn_name) + if dtype is not None: + return dtype + + if isinstance(sample_tensor, torch.Tensor) and torch.is_floating_point( + sample_tensor + ): + return sample_tensor.dtype + return torch.float32 + def forward(self, batch: Req, server_args: ServerArgs) -> Req: # 1. Input validation self._validate_input(batch, server_args) @@ -173,14 +204,14 @@ def forward(self, batch: Req, server_args: ServerArgs) -> Req: image = cond_inputs.pop("image") device = self.device - dtype = next(self.model.parameters()).dtype + dtype = self._resolve_runtime_dtype( + image if isinstance(image, torch.Tensor) else None + ) image = _move_to_device(image, device, dtype) cond_inputs = _move_to_device(cond_inputs, device, dtype) # 3. Conditioning with CFG - do_cfg = batch.guidance_scale >= 0 and not ( - hasattr(self.model, "guidance_embed") and self.model.guidance_embed is True - ) + do_cfg = batch.guidance_scale >= 0 and not self.guidance_embed cond = self.conditioner(image=image, **cond_inputs) if do_cfg: @@ -216,7 +247,7 @@ def cat_recursive(a, b): latents = self._prepare_latents(batch_size, dtype, device, generator, scheduler) guidance = None - if hasattr(self.model, "guidance_embed") and self.model.guidance_embed is True: + if self.guidance_embed: guidance = torch.tensor( [batch.guidance_scale] * batch_size, device=device, dtype=dtype ) @@ -416,6 +447,12 @@ def __init__(self, vae: Any, config: Hunyuan3D2PipelineConfig) -> None: self.vae = vae self.config = config + @property + def role_affinity(self): + from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType + + return RoleType.DECODER + def forward(self, batch: Req, server_args: ServerArgs) -> Req: if self.config.shape_mc_algo is not None: try: @@ -473,6 +510,12 @@ def __init__(self, config: Hunyuan3D2PipelineConfig) -> None: super().__init__() self.config = config + @property + def role_affinity(self): + from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType + + return RoleType.DECODER + def _get_output_paths(self, batch: Req) -> tuple[str, str]: output_path = batch.output_file_path() or os.path.join( batch.output_path, "output.obj" diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py index a309f304a185..1c64c94c829d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/image_encoding.py @@ -803,9 +803,15 @@ class ImageVAEEncodingStage(PipelineStage): "vae_image_sizes", ) - def __init__(self, vae: ParallelTiledVAE, **kwargs) -> None: + def __init__( + self, + vae: ParallelTiledVAE, + component_name: str = "vae", + **kwargs, + ) -> None: super().__init__() self.vae: ParallelTiledVAE = vae + self.component_name = component_name def component_uses( self, server_args: ServerArgs, stage_name: str | None = None @@ -815,7 +821,7 @@ def component_uses( return [ ComponentUse( stage_name, - "vae", + self.component_name, target_dtype=vae_dtype, ) ] @@ -851,7 +857,10 @@ def forward( vae_dtype != torch.float32 ) and not server_args.disable_autocast - with self.use_declared_component(component_name="vae", module=self.vae) as vae: + with self.use_declared_component( + component_name=self.component_name, + module=self.vae, + ) as vae: assert vae is not None self.vae = vae diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py index 7fed04ffe215..e0ba25fd0cad 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py @@ -13,6 +13,7 @@ import torch import torch.nn.functional as F +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, @@ -105,6 +106,10 @@ def __init__(self, transformer, scheduler): self.transformer = transformer self.scheduler = scheduler + @property + def role_affinity(self) -> RoleType: + return RoleType.DENOISER + @property def parallelism_type(self): return StageParallelismType.REPLICATED diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py index 8c0e14e629c6..06b0ab4ab3b1 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py @@ -21,6 +21,7 @@ from diffusers.utils.torch_utils import randn_tensor from tqdm.auto import tqdm +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.distributed import ( get_local_torch_device, get_world_group, @@ -187,6 +188,10 @@ def component_uses( ) return uses + @property + def role_affinity(self) -> RoleType: + return RoleType.DENOISER + @property def parallelism_type(self) -> StageParallelismType: if get_global_server_args().enable_cfg_parallel: @@ -240,6 +245,13 @@ def _maybe_enable_torch_compile(self, module: nn.Module, server_args: ServerArgs """ if not server_args.enable_torch_compile or not isinstance(module, nn.Module): return + if current_platform.is_hip(): + logger.warning( + "Skipping torch.compile for %s on ROCm because the current " + "HIPRTC/Inductor path can emit invalid bf16 kernels.", + module.__class__.__name__, + ) + return compile_kwargs: dict[str, object] = {"fullgraph": False, "dynamic": None} if current_platform.is_npu(): @@ -954,6 +966,10 @@ def component_uses( ComponentUse(stage_name, "audio_vae"), ] + @property + def role_affinity(self) -> RoleType: + return RoleType.DECODER + @property def parallelism_type(self) -> StageParallelismType: if get_global_server_args().enable_cfg_parallel: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py index 315c260892ff..d5dece9f4ede 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py @@ -21,6 +21,31 @@ logger = init_logger(__name__) +def _resolve_text_encoder_dtype( + text_encoder: object, fallback: torch.dtype = torch.bfloat16 +) -> torch.dtype: + module_dtype = getattr(text_encoder, "dtype", None) + if isinstance(module_dtype, torch.dtype): + return module_dtype + + for tensor_source in ("parameters", "buffers"): + tensors = getattr(text_encoder, tensor_source, None) + if not callable(tensors): + continue + try: + for tensor in tensors(): + if isinstance(tensor, torch.Tensor) and torch.is_floating_point(tensor): + return tensor.dtype + except TypeError as exc: + logger.warning( + "Failed to inspect text encoder %s() for dtype: %s", + tensor_source, + exc, + ) + + return fallback + + def _seq_lens_from_optional_mask( prompt_embeds: torch.Tensor, prompt_embeds_mask: torch.Tensor | None ) -> list[int]: @@ -125,6 +150,7 @@ class QwenImageLayeredBeforeDenoisingStage(PipelineStage): def __init__( self, vae, + text_encoder, tokenizer, processor, transformer, @@ -137,14 +163,14 @@ def __init__( self.vae = vae.to(dtype=vae_dtype) self.vae_dtype = vae_dtype self.text_encoder_dtype = text_encoder_dtype - from transformers import Qwen2_5_VLForConditionalGeneration + if text_encoder is None: + from transformers import Qwen2_5_VLForConditionalGeneration - self.text_encoder = ( - Qwen2_5_VLForConditionalGeneration.from_pretrained( + text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( model_path, subfolder="text_encoder" ) - .to(get_local_torch_device()) - .to(dtype=self.text_encoder_dtype) + self.text_encoder = text_encoder.to( + device=get_local_torch_device(), dtype=self.text_encoder_dtype ) self.tokenizer = tokenizer self.processor = processor @@ -186,9 +212,15 @@ def component_uses( stage_name = self._component_stage_name(stage_name) return [ ComponentUse( - stage_name, "qwen_layered_text_encoder", target_dtype=torch.bfloat16 + stage_name, + "text_encoder", + target_dtype=self.text_encoder_dtype, + ), + ComponentUse( + stage_name, + "vae", + target_dtype=self.vae_dtype, ), - ComponentUse(stage_name, "vae", target_dtype=torch.bfloat16), ] # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._extract_masked_hidden @@ -232,7 +264,7 @@ def _get_qwen_prompt_embeds( device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None, ): - dtype = dtype or self.text_encoder.dtype + dtype = dtype or _resolve_text_encoder_dtype(self.text_encoder) prompt = [prompt] if isinstance(prompt, str) else prompt @@ -482,7 +514,7 @@ def forward( prompt = batch.prompt with self.use_declared_component( - component_name="qwen_layered_text_encoder", + component_name="text_encoder", module=self.text_encoder, ) as text_encoder: assert text_encoder is not None diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py index 09f60d2b7e83..c332c1652f44 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py @@ -9,6 +9,7 @@ import inspect from dataclasses import dataclass +from functools import lru_cache from typing import Any import torch @@ -34,6 +35,18 @@ logger = init_logger(__name__) +@lru_cache(maxsize=1) +def get_model_default_negative_prompt( + model_path: str, backend: Any, model_id: str | None +): + from sglang.multimodal_gen.registry import get_model_info + + model_info = get_model_info(model_path, backend=backend, model_id=model_id) + if model_info is None: + return None + return model_info.sampling_param_cls().negative_prompt + + @dataclass(frozen=True) class TextEncodingFingerprint: prompt: Any @@ -104,69 +117,195 @@ def component_uses( def get_or_compute_negative_text_embedding( self, batch: Req, server_args: ServerArgs, all_indices: list[int] ): + """Get the cached text embedding result or compute + + this is a one-slot cache for the model-default negative prompt: + most requests don't override the negative prompt, the cache hit rate is considerably high + """ negative_cache_key = self._build_negative_text_cache_key( batch, server_args, all_indices ) - use_negative_cache = not batch.is_warmup - cached_negative = None - if use_negative_cache: - cached_negative = ( - self._negative_text_cache_value - if self._negative_text_cache_key == negative_cache_key - else None - ) - if cached_negative is None: - ( - neg_embeds_list, - neg_masks_list, - neg_pooler_embeds_list, - neg_embeds_masks_list, - neg_seq_lens_list, - ) = self.encode_text( - batch.negative_prompt, - server_args, - encoder_index=all_indices, - return_attention_mask=True, - ) + cached_negative = self._get_cached_negative_text_embedding(negative_cache_key) + if cached_negative is not None: + return cached_negative - if use_negative_cache: - self._negative_text_cache_key = negative_cache_key - self._negative_text_cache_value = ( - tuple(neg_embeds_list), - tuple(neg_masks_list), - tuple(neg_pooler_embeds_list), - tuple(neg_embeds_masks_list), - tuple(neg_seq_lens_list), - ) - else: - ( - neg_embeds_list, - neg_masks_list, - neg_pooler_embeds_list, - neg_embeds_masks_list, - neg_seq_lens_list, - ) = cached_negative - return ( - neg_embeds_list, - neg_masks_list, - neg_pooler_embeds_list, - neg_embeds_masks_list, - neg_seq_lens_list, + negative_text_outputs = self.encode_text( + batch.negative_prompt, + server_args, + encoder_index=all_indices, + return_attention_mask=True, + ) + self._maybe_cache_negative_text_embedding( + negative_cache_key, negative_text_outputs + ) + return negative_text_outputs + + def _should_cache_negative_text_embedding( + self, batch: Req, server_args: ServerArgs + ) -> bool: + if not batch.is_warmup: + return True + return self._uses_model_default_negative_prompt(batch, server_args) + + def _get_cached_negative_text_embedding(self, negative_cache_key): + if negative_cache_key is None: + return None + if self._negative_text_cache_key == negative_cache_key: + return self._negative_text_cache_value + return None + + def _maybe_cache_negative_text_embedding( + self, + negative_cache_key, + negative_text_outputs, + ) -> None: + + # skip caching if None + if negative_cache_key is None: + return + self._negative_text_cache_key = negative_cache_key + self._negative_text_cache_value = tuple( + tuple(value) for value in negative_text_outputs ) def _build_negative_text_cache_key( self, batch: Req, server_args: ServerArgs, encoder_indices: list[int] ): + """if the current req doesn't worth caching, returns None""" + # skip if we don't cache for current req + if not self._should_cache_negative_text_embedding(batch, server_args): + return None + # Negative text encoding changes when the template or max length changes, # even if the visible negative prompt string is the same. return ( - server_args.pipeline_class_name, tuple(encoder_indices), self.freeze_for_dedup(batch.negative_prompt), self.freeze_for_dedup(batch.prompt_template), batch.max_sequence_length, ) + def _uses_model_default_negative_prompt( + self, batch: Req, server_args: ServerArgs + ) -> bool: + default_negative_prompt = self._get_model_default_negative_prompt(server_args) + if default_negative_prompt is None: + return False + return self._normalize_negative_prompt_for_default_match( + batch.negative_prompt + ) == self._normalize_negative_prompt_for_default_match(default_negative_prompt) + + def _get_model_default_negative_prompt(self, server_args: ServerArgs) -> str | None: + return get_model_default_negative_prompt( + server_args.model_path, + server_args.backend, + server_args.model_id, + ) + + @staticmethod + def _normalize_negative_prompt_for_default_match(value): + if isinstance(value, str) and not value.isspace(): + return value.strip() + return value + + def _append_positive_text_outputs( + self, + batch: Req, + prompt_embeds_list, + prompt_masks_list, + pooler_embeds_list, + prompt_embeds_masks_list, + prompt_seq_lens_list, + ) -> None: + for pe in prompt_embeds_list: + batch.prompt_embeds.append(pe) + + for pe in pooler_embeds_list: + batch.pooled_embeds.append(pe) + + if batch.prompt_attention_mask is None: + batch.prompt_attention_mask = [] + for am in prompt_masks_list: + batch.prompt_attention_mask.append(am) + + batch.prompt_embeds_mask = [] + batch.prompt_seq_lens = [] + for mask in prompt_embeds_masks_list: + batch.prompt_embeds_mask.append(mask) + for seq_lens in prompt_seq_lens_list: + batch.prompt_seq_lens.append(seq_lens) + + def _append_negative_text_outputs( + self, + batch: Req, + prompt_embeds_list, + neg_embeds_list, + neg_masks_list, + neg_pooler_embeds_list, + neg_embeds_masks_list, + neg_seq_lens_list, + ) -> None: + assert batch.negative_prompt_embeds is not None + + # a single negative prompt can be shared across positive prompts + target_batch_sizes = [pe.shape[0] for pe in prompt_embeds_list] + + def align_negative_batch_dim( + tensor: torch.Tensor, target_batch: int, name: str + ) -> torch.Tensor: + if tensor.shape[0] == target_batch: + return tensor + if tensor.shape[0] == 1 and target_batch > 1: + return tensor.expand(target_batch, *tensor.shape[1:]) + raise ValueError( + f"{name} batch dimension mismatch: got {tensor.shape[0]}, expected 1 or {target_batch}" + ) + + def align_negative_seq_lens( + seq_lens: list[int], target_batch: int, name: str + ) -> list[int]: + if len(seq_lens) == target_batch: + return [int(x) for x in seq_lens] + if len(seq_lens) == 1 and target_batch > 1: + return [int(seq_lens[0])] * target_batch + raise ValueError( + f"{name} batch dimension mismatch: got {len(seq_lens)}, expected 1 or {target_batch}" + ) + + for idx, ne in enumerate(neg_embeds_list): + target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] + ne = align_negative_batch_dim(ne, target_batch, "negative_prompt_embeds") + batch.negative_prompt_embeds.append(ne) + + for idx, pe in enumerate(neg_pooler_embeds_list): + target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] + pe = align_negative_batch_dim(pe, target_batch, "negative_pooled_embeds") + batch.neg_pooled_embeds.append(pe) + if batch.negative_attention_mask is None: + batch.negative_attention_mask = [] + for idx, nm in enumerate(neg_masks_list): + target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] + nm = align_negative_batch_dim( + nm, target_batch, "negative_attention_mask" + ) + batch.negative_attention_mask.append(nm) + + batch.negative_prompt_embeds_mask = [] + batch.negative_prompt_seq_lens = [] + for idx, nm in enumerate(neg_embeds_masks_list): + target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] + nm = align_negative_batch_dim( + nm, target_batch, "negative_prompt_embeds_mask" + ) + batch.negative_prompt_embeds_mask.append(nm) + for idx, seq_lens in enumerate(neg_seq_lens_list): + target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] + batch.negative_prompt_seq_lens.append( + align_negative_seq_lens( + seq_lens, target_batch, "negative_prompt_seq_lens" + ) + ) + @torch.no_grad() def forward( self, @@ -204,25 +343,6 @@ def forward( max_length=max_seq_length, ) - for pe in prompt_embeds_list: - batch.prompt_embeds.append(pe) - - for pe in pooler_embeds_list: - batch.pooled_embeds.append(pe) - - if batch.prompt_attention_mask is None: - batch.prompt_attention_mask = [] - for am in prompt_masks_list: - batch.prompt_attention_mask.append(am) - - batch.prompt_embeds_mask = [] - batch.prompt_seq_lens = [] - for mask in prompt_embeds_masks_list: - batch.prompt_embeds_mask.append(mask) - for seq_lens in prompt_seq_lens_list: - batch.prompt_seq_lens.append(seq_lens) - - # Encode negative prompt if CFG is enabled if batch.do_classifier_free_guidance: assert isinstance(batch.negative_prompt, str) ( @@ -235,72 +355,26 @@ def forward( batch, server_args, all_indices ) - assert batch.negative_prompt_embeds is not None - - # A single negative prompt can be shared across positive prompts. - target_batch_sizes = [pe.shape[0] for pe in prompt_embeds_list] - - def align_negative_batch_dim( - tensor: torch.Tensor, target_batch: int, name: str - ) -> torch.Tensor: - if tensor.shape[0] == target_batch: - return tensor - if tensor.shape[0] == 1 and target_batch > 1: - return tensor.expand(target_batch, *tensor.shape[1:]) - raise ValueError( - f"{name} batch dimension mismatch: got {tensor.shape[0]}, expected 1 or {target_batch}" - ) - - def align_negative_seq_lens( - seq_lens: list[int], target_batch: int, name: str - ) -> list[int]: - if len(seq_lens) == target_batch: - return [int(x) for x in seq_lens] - if len(seq_lens) == 1 and target_batch > 1: - return [int(seq_lens[0])] * target_batch - raise ValueError( - f"{name} batch dimension mismatch: got {len(seq_lens)}, expected 1 or {target_batch}" - ) - - for idx, ne in enumerate(neg_embeds_list): - target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] - ne = align_negative_batch_dim( - ne, target_batch, "negative_prompt_embeds" - ) - batch.negative_prompt_embeds.append(ne) - - for idx, pe in enumerate(neg_pooler_embeds_list): - target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] - pe = align_negative_batch_dim( - pe, target_batch, "negative_pooled_embeds" - ) - batch.neg_pooled_embeds.append(pe) - if batch.negative_attention_mask is None: - batch.negative_attention_mask = [] - for idx, nm in enumerate(neg_masks_list): - target_batch = target_batch_sizes[ - min(idx, len(target_batch_sizes) - 1) - ] - nm = align_negative_batch_dim( - nm, target_batch, "negative_attention_mask" - ) - batch.negative_attention_mask.append(nm) + self._append_positive_text_outputs( + batch, + prompt_embeds_list, + prompt_masks_list, + pooler_embeds_list, + prompt_embeds_masks_list, + prompt_seq_lens_list, + ) - batch.negative_prompt_embeds_mask = [] - batch.negative_prompt_seq_lens = [] - for idx, nm in enumerate(neg_embeds_masks_list): - target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] - nm = align_negative_batch_dim( - nm, target_batch, "negative_prompt_embeds_mask" - ) - batch.negative_prompt_embeds_mask.append(nm) - for idx, seq_lens in enumerate(neg_seq_lens_list): - target_batch = target_batch_sizes[min(idx, len(target_batch_sizes) - 1)] - batch.negative_prompt_seq_lens.append( - align_negative_seq_lens( - seq_lens, target_batch, "negative_prompt_seq_lens" - ) - ) + # Encode negative prompt if CFG is enabled + if batch.do_classifier_free_guidance: + self._append_negative_text_outputs( + batch, + prompt_embeds_list, + neg_embeds_list, + neg_masks_list, + neg_pooler_embeds_list, + neg_embeds_masks_list, + neg_seq_lens_list, + ) return batch @@ -322,7 +396,9 @@ def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResul result.add_check( "negative_prompt", batch.negative_prompt, - lambda x: not batch.do_classifier_free_guidance or V.string_not_none(x), + lambda x: not batch.do_classifier_free_guidance + or V.string_not_none(x) + or isinstance(x, str), ) result.add_check( "do_classifier_free_guidance", @@ -353,7 +429,7 @@ def _manage_text_encoder_use(self, encoder_index: int) -> None: # TODO: Keep this begin-only interval until manager supports explicit # declared-use interval grouping. Wrapping each encoder call separately # can offload between positive and negative prompt encoding. - manager.before_use(use) + manager.begin_use(use, module=self.text_encoders[encoder_index]) def _forward_text_encoder(self, text_encoder, encoder_forward_kwargs): if not getattr(text_encoder, "uses_sglang_forward_context", True): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py index 5d3c78eeee3f..da08f28eaba1 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/timestep_preparation.py @@ -60,13 +60,13 @@ class TimestepPreparationStage(PipelineStage): def __init__( self, scheduler, - prepare_extra_set_timesteps_kwargs: list[ - Callable[[Req, ServerArgs], Tuple[str, Any]] - ] = [], + prepare_extra_set_timesteps_kwargs: ( + list[Callable[[Req, ServerArgs], Tuple[str, Any]]] | None + ) = None, ) -> None: super().__init__() self.scheduler = scheduler - self.prepare_extra_set_timesteps_kwargs = ( + self.prepare_extra_set_timesteps_kwargs = list( prepare_extra_set_timesteps_kwargs or [] ) diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index 32b8212a896e..cd83f78beb56 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -124,7 +124,7 @@ def get_modelopt_fp4_quantize_op(cls) -> Callable | None: @lru_cache(maxsize=1) def get_modelopt_flashinfer_fp4_backend(cls) -> str: backend = envs.SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND - default_backend = "cudnn" if cls.is_blackwell() else "auto" + default_backend = "trtllm" if backend is None: return default_backend @@ -151,35 +151,23 @@ def get_modelopt_flashinfer_fp4_backend(cls) -> str: @lru_cache(maxsize=1) def get_modelopt_fp4_gemm_op(cls) -> tuple[Callable | None, str | None]: requested_backend = envs.SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND - prefer_flashinfer = requested_backend is not None - - # TODO: Remove this explicit FlashInfer preference once the sm100 CUTLASS - # LargeM dispatch grows a validated fallback for Blackwell NVFP4 shapes - # such as Wan2.2's large-M attention projections. - if prefer_flashinfer: - try: - from flashinfer import mm_fp4 as flashinfer_mm_fp4 - - return flashinfer_mm_fp4, cls.get_modelopt_flashinfer_fp4_backend() - except ImportError: - logger.warning( - "Requested SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=%r " - "but flashinfer.mm_fp4 is unavailable. Falling back to " - "cutlass.", - requested_backend, - ) try: - from sgl_kernel import cutlass_scaled_fp4_mm as cutlass_fp4_gemm + from flashinfer import mm_fp4 as flashinfer_mm_fp4 - return cutlass_fp4_gemm, None + return flashinfer_mm_fp4, cls.get_modelopt_flashinfer_fp4_backend() except ImportError: - pass + logger.warning( + "Requested SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND=%r " + "but flashinfer.mm_fp4 is unavailable. Falling back to " + "cutlass.", + requested_backend or "flashinfer_trtllm (default)", + ) try: - from flashinfer import mm_fp4 as flashinfer_mm_fp4 + from sgl_kernel import cutlass_scaled_fp4_mm as cutlass_fp4_gemm - return flashinfer_mm_fp4, cls.get_modelopt_flashinfer_fp4_backend() + return cutlass_fp4_gemm, None except ImportError: return None, None diff --git a/python/sglang/multimodal_gen/runtime/server_args.py b/python/sglang/multimodal_gen/runtime/server_args.py index 2ab9f66a5d2b..8314bf369fc1 100644 --- a/python/sglang/multimodal_gen/runtime/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args.py @@ -219,6 +219,9 @@ class ServerArgs(DisaggArgsMixin): # Compilation enable_torch_compile: bool = False + # NVTX profiling + enable_layerwise_nvtx_marker: bool = False + # warmup warmup: bool = False warmup_resolutions: list[str] = None @@ -1190,6 +1193,17 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: + "However, will likely cause precision drifts. See (https://github.com/pytorch/pytorch/issues/145213)", ) + parser.add_argument( + "--enable-layerwise-nvtx-marker", + action=StoreBoolean, + default=ServerArgs.enable_layerwise_nvtx_marker, + help="Enable layerwise NVTX markers for profiling with Nsight Systems. " + "Adds NVTX ranges around each pipeline stage, the denoising loop, " + "every denoising step, the predict_noise / scheduler_step " + "sub-operations, and every transformer submodule forward (recursive). " + "Warmup steps are excluded to keep captured traces clean.", + ) + # warmup parser.add_argument( "--warmup", diff --git a/python/sglang/multimodal_gen/runtime/utils/nvtx_pytorch_hooks.py b/python/sglang/multimodal_gen/runtime/utils/nvtx_pytorch_hooks.py new file mode 100644 index 000000000000..2b020fcd5b12 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/utils/nvtx_pytorch_hooks.py @@ -0,0 +1,214 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""PyTorch hooks for layerwise NVTX profiling in SGLang Diffusion. + +Mirrors the structure of ``sglang.srt.utils.nvtx_pytorch_hooks.PytHooks`` +but uses a compact ``{name} in={shapes}`` marker format that is well-suited +to DiT transformer blocks. See +``sglang.srt.utils.nvtx_pytorch_hooks`` for the LLM-runtime equivalent +that emits a richer per-layer parameter dict. +""" + +from __future__ import annotations + +import contextlib +from collections.abc import Iterator +from typing import Any + +import torch +import torch.cuda.nvtx as nvtx +from torch.utils.hooks import RemovableHandle + +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +# Module types that are too lightweight to warrant their own NVTX range. +# Skipping them keeps the captured timeline readable. +_DEFAULT_SKIP_TYPES: tuple[type, ...] = ( + torch.nn.Identity, + torch.nn.Dropout, + torch.nn.Dropout1d, + torch.nn.Dropout2d, + torch.nn.Dropout3d, +) + + +@contextlib.contextmanager +def maybe_nvtx_range(name: str, enabled: bool = True) -> Iterator[None]: + """Context manager that wraps a block of work in an NVTX range. + + Calls ``range_push`` / ``range_pop`` directly rather than going through + :func:`torch.cuda.nvtx.range`, which would otherwise interpret ``name`` as a + ``str.format`` template (so a literal ``{`` in the marker would raise + ``KeyError``). The ``range_pop`` is invoked from the ``finally`` clause, so + exceptions raised inside the ``with`` block cannot leak a half-open range. + + When ``enabled`` is ``False`` the function is a zero-cost no-op, suitable + for use under a per-request gate (e.g. warmup exclusion). + """ + if not enabled: + yield + return + nvtx.range_push(name) + try: + yield + finally: + nvtx.range_pop() + + +class DiffusionNvtxHooks: + """Register NVTX markers around each submodule forward pass. + + Each registered module emits an NVTX range covering its forward pass. + The range name encodes the qualified module name and the input tensor + shapes for downstream identification in Nsight Systems. + + Hook handles are retained so they can be removed via :meth:`remove_hooks`; + the same instance must not be reused across multiple model instances. + """ + + def __init__(self, skip_types: tuple[type, ...] = _DEFAULT_SKIP_TYPES) -> None: + self._skip_types = skip_types + self._module_to_name_map: dict[torch.nn.Module, str] = {} + self._hook_handles: list[RemovableHandle] = [] + # Caller must explicitly enable via ``set_enabled``. Default off + # so a forward that bypasses the component-use gate (e.g. an early + # warmup pass) cannot accidentally pollute the captured timeline. + self._enabled: bool = False + + def register_hooks( + self, + model: torch.nn.Module, + prefix: str = "", + ) -> int: + """Walk ``model`` and attach forward pre/post hooks to every module. + + Args: + model: Root module to instrument. + prefix: Optional name prefix prepended to every emitted range. + + Returns: + Number of modules instrumented. + + Notes: + Weight-tied or otherwise duplicated module instances are + skipped (the first occurrence wins) so each forward pass + produces exactly one NVTX range. + """ + instrumented = 0 + for name, module in model.named_modules(prefix=prefix): + if isinstance(module, self._skip_types): + continue + # Skip duplicate module instances (e.g., weight-tied layers). + # The check must happen before hook registration to avoid + # double-emitting NVTX ranges on the second occurrence. + if module in self._module_to_name_map: + logger.debug( + "NVTX: module %s already registered as '%s', skipping '%s'", + type(module).__name__, + self._module_to_name_map[module], + name, + ) + continue + self._module_to_name_map[module] = name + self._hook_handles.append( + module.register_forward_pre_hook( + self._forward_pre_hook, with_kwargs=True + ) + ) + # ``always_call=True`` (PyTorch 2.0+) guarantees the post-hook + # still fires when ``forward`` raises, so an OOM or assertion + # inside the wrapped module cannot leak a half-open NVTX range. + self._hook_handles.append( + module.register_forward_hook(self._forward_hook, always_call=True) + ) + instrumented += 1 + return instrumented + + def remove_hooks(self) -> None: + """Remove every hook registered by this instance. + + Safe to call multiple times; subsequent calls are no-ops. The + bookkeeping is cleared in a ``finally`` so a misbehaving + ``handle.remove()`` cannot leave the instance with stale + handles or name-map entries. + """ + try: + for handle in self._hook_handles: + handle.remove() + finally: + self._hook_handles.clear() + self._module_to_name_map.clear() + + def set_enabled(self, enabled: bool) -> None: + """Toggle whether the registered hooks emit NVTX ranges. + + When disabled, both the pre- and post-hooks early-return, so each + forward produces a matched (push, pop) pair of "no-ops" — no range + leak and no half-open range across the toggle. + """ + self._enabled = enabled + + # ------------------------------------------------------------------ hooks + + def _forward_pre_hook( + self, + module: torch.nn.Module, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> None: + if not self._enabled: + return + name = self._module_to_name_map.get(module, "unknown") + shapes = _collect_input_shapes(args, kwargs) + marker = f"{name} in={shapes}" if shapes else name + nvtx.range_push(marker) + + def _forward_hook( + self, + module: torch.nn.Module, + _args: Any, + _output: Any, + ) -> None: + if not self._enabled: + return + nvtx.range_pop() + + +def _collect_input_shapes( + args: tuple[Any, ...], kwargs: dict[str, Any] | None = None +) -> list[list[int]]: + """Best-effort extraction of input tensor shapes for marker labels. + + Walks positional ``args`` and keyword ``kwargs`` values, recursing into + lists and tuples (so DiT inputs like ``image_rotary_emb=(cos, sin)`` are + captured). Non-tensor scalars, ``None``, dicts, and arbitrary objects are + silently skipped. + """ + shapes: list[list[int]] = [] + _append_tensor_shapes(args, shapes) + if kwargs: + _append_tensor_shapes(tuple(kwargs.values()), shapes) + return shapes + + +def _append_tensor_shapes(items: Any, shapes: list[list[int]]) -> None: + if isinstance(items, torch.Tensor): + shapes.append(list(items.size())) + return + if isinstance(items, (list, tuple)): + for item in items: + _append_tensor_shapes(item, shapes) diff --git a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py index cfd4eb68476b..168227e7b84e 100644 --- a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py @@ -438,14 +438,22 @@ def _build_nvfp4_config_from_safetensors_files( "group_size": group_size, "ignore": exclude_modules, "checkpoint_uses_packed_qkv": checkpoint_uses_packed_qkv, + # The official FLUX.2 mixed NVFP4 export is detected by its + # packed QKV tensors and stores block scales in the + # FlashInfer/CUTLASS-swizzled layout. SGLang-converted + # transformer repos keep the linear layout. + "checkpoint_weight_scale_layout": ( + "swizzled" if checkpoint_uses_packed_qkv else "linear" + ), } ) logger.info( - "Built NVFP4 quant config from %d safetensors: group_size=%d, %d excluded modules, packed_qkv=%s", + "Built NVFP4 quant config from %d safetensors: group_size=%d, %d excluded modules, packed_qkv=%s, scale_layout=%s", len(files_with_nvfp4_signal), group_size, len(exclude_modules), checkpoint_uses_packed_qkv, + getattr(result, "checkpoint_weight_scale_layout", "linear"), ) return result except Exception as e: diff --git a/python/sglang/multimodal_gen/test/run_suite_musa.py b/python/sglang/multimodal_gen/test/run_suite_musa.py index b8cd91e22961..5ec408ef008c 100644 --- a/python/sglang/multimodal_gen/test/run_suite_musa.py +++ b/python/sglang/multimodal_gen/test/run_suite_musa.py @@ -22,11 +22,13 @@ SUITES = { "1-gpu-musa": [ - "musa/test_server_a_musa.py", - "musa/test_server_b_musa.py", + "musa/test_server_1_gpu_musa.py", + ], + "1-gpu-musa-nightly": [ + "musa/test_server_1_gpu_musa_nightly.py", ], "2-gpu-musa": [ - "musa/test_server_2_gpu_a_musa.py", + "musa/test_server_2_gpu_musa.py", ], } diff --git a/python/sglang/multimodal_gen/test/server/accuracy_hooks.py b/python/sglang/multimodal_gen/test/server/accuracy_hooks.py index a495904d8a0e..09ca8f17d0e1 100644 --- a/python/sglang/multimodal_gen/test/server/accuracy_hooks.py +++ b/python/sglang/multimodal_gen/test/server/accuracy_hooks.py @@ -38,6 +38,7 @@ DEFAULT_TRANSFORMER_POOLED_CHANNELS = 768 DEFAULT_VAE_LATENT_CHANNELS = 16 DEFAULT_VAE_LATENT_SPATIAL_SIZE = 32 +DEFAULT_VAE_VIDEO_LATENT_FRAMES = 3 LARGE_CHANNEL_LAYOUT_THRESHOLD = 128 @@ -610,17 +611,35 @@ def _infer_vae_latent_channels(model: nn.Module) -> int: def _build_vae_hook_inputs( case: Any, model: nn.Module, device: str, ref_model: Optional[nn.Module] = None ) -> Inputs: - del case, ref_model + del ref_model latent_channels = _infer_vae_latent_channels(model) + model_path = getattr(getattr(case, "server_args", None), "model_path", "").lower() + modality = getattr(getattr(case, "server_args", None), "modality", None) + use_wan_video_latent = ( + modality == "video" + and "wan" in model_path + and any(isinstance(module, nn.Conv3d) for module in model.modules()) + ) + shape = ( + ( + 1, + latent_channels, + DEFAULT_VAE_VIDEO_LATENT_FRAMES, + DEFAULT_VAE_LATENT_SPATIAL_SIZE, + DEFAULT_VAE_LATENT_SPATIAL_SIZE, + ) + if use_wan_video_latent + else ( + 1, + latent_channels, + DEFAULT_VAE_LATENT_SPATIAL_SIZE, + DEFAULT_VAE_LATENT_SPATIAL_SIZE, + ) + ) rng = _DeterministicRNG() return { "z": rng.randn( - ( - 1, - latent_channels, - DEFAULT_VAE_LATENT_SPATIAL_SIZE, - DEFAULT_VAE_LATENT_SPATIAL_SIZE, - ), + shape, device, torch.bfloat16, ) diff --git a/python/sglang/multimodal_gen/test/server/accuracy_utils.py b/python/sglang/multimodal_gen/test/server/accuracy_utils.py index cd544e6b9f51..6a2dcb6bf1f8 100644 --- a/python/sglang/multimodal_gen/test/server/accuracy_utils.py +++ b/python/sglang/multimodal_gen/test/server/accuracy_utils.py @@ -760,12 +760,12 @@ def _run_staged_native_component_accuracy_case( ref = ref.to(device=device, dtype=torch.bfloat16).eval() if component == ComponentType.VAE: - from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.loader.component_loaders.vae_loader import ( _convert_conv3d_weights_to_channels_last_3d, + _should_use_channels_last_3d, ) - if torch.cuda.is_available() and envs.SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D: + if _should_use_channels_last_3d(runtime_server_args, "vae"): _convert_conv3d_weights_to_channels_last_3d(ref) ref_call = profile.prepare_reference_call(ref, inputs) ref_autocast = ( diff --git a/python/sglang/multimodal_gen/test/server/component_accuracy.py b/python/sglang/multimodal_gen/test/server/component_accuracy.py index 1e97beb7668a..8f6a26561bd2 100644 --- a/python/sglang/multimodal_gen/test/server/component_accuracy.py +++ b/python/sglang/multimodal_gen/test/server/component_accuracy.py @@ -2,6 +2,7 @@ import gc import os +from contextlib import contextmanager from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple @@ -74,6 +75,69 @@ logger = init_logger(__name__) MIN_MATCH_RATIO = float(os.getenv("SGLANG_DIFFUSION_WEIGHT_MATCH_RATIO", "0.98")) +VAE_CHANNELS_LAST_3D_ENV = "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D" +VAE_CHANNELS_LAST_3D_PARITY_THRESHOLD = float( + os.getenv("SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D_PARITY_THRESHOLD", "0.999") +) + + +@contextmanager +def _temporary_vae_channels_last_3d(enabled: bool): + previous = os.environ.get(VAE_CHANNELS_LAST_3D_ENV) + os.environ[VAE_CHANNELS_LAST_3D_ENV] = "true" if enabled else "false" + try: + yield + finally: + if previous is None: + os.environ.pop(VAE_CHANNELS_LAST_3D_ENV, None) + else: + os.environ[VAE_CHANNELS_LAST_3D_ENV] = previous + + +@dataclass +class Conv3dLayoutStats: + calls: int = 0 + channels_last_input_calls: int = 0 + channels_last_weight_calls: int = 0 + mixed_layout_calls: int = 0 + + +@contextmanager +def _record_conv3d_layouts(): + stats = Conv3dLayoutStats() + original_conv3d = torch.nn.functional.conv3d + + def wrapped_conv3d(input, weight, *args, **kwargs): + if ( + isinstance(input, torch.Tensor) + and isinstance(weight, torch.Tensor) + and input.dim() == 5 + and weight.dim() == 5 + and hasattr(torch, "channels_last_3d") + ): + input_channels_last = input.is_contiguous( + memory_format=torch.channels_last_3d + ) + weight_channels_last = weight.is_contiguous( + memory_format=torch.channels_last_3d + ) + else: + input_channels_last = False + weight_channels_last = False + + stats.calls += 1 + stats.channels_last_input_calls += int(input_channels_last) + stats.channels_last_weight_calls += int(weight_channels_last) + stats.mixed_layout_calls += int( + weight_channels_last and not input_channels_last + ) + return original_conv3d(input, weight, *args, **kwargs) + + torch.nn.functional.conv3d = wrapped_conv3d + try: + yield stats + finally: + torch.nn.functional.conv3d = original_conv3d @dataclass(frozen=True) @@ -584,3 +648,114 @@ def load_component_pair( ) return sgl_component.eval(), ref_component.eval(), str(device) + + @staticmethod + def run_vae_channels_last_3d_parity( + case: DiffusionTestCase, + num_gpus: int, + ) -> None: + component = ComponentType.VAE + spec = COMPONENT_SPECS[component] + hub_id = case.server_args.model_path + component_selection = select_component_source( + hub_id, + case.server_args.extras, + component, + spec.model_index_keys, + ) + sgl_args = build_accuracy_server_args( + component_selection.base_model_id, + component_selection.base_model_root, + case, + component, + num_gpus, + component_selection.component_paths, + ) + + baseline_vae = None + channels_last_vae = None + try: + initialize_parallel_runtime(sgl_args) + set_global_server_args(sgl_args) + device = get_local_torch_device() + + with _temporary_vae_channels_last_3d(False): + baseline_vae = _load_sglang_component( + component_selection.source_path, + sgl_args, + component, + spec.reference_library, + ).to(device=device, dtype=torch.bfloat16) + + with _temporary_vae_channels_last_3d(True): + channels_last_vae = _load_sglang_component( + component_selection.source_path, + sgl_args, + component, + spec.reference_library, + ).to(device=device, dtype=torch.bfloat16) + + baseline_vae.eval() + channels_last_vae.eval() + + profile = resolve_component_native_profile(component) + inputs = profile.build_inputs( + case, baseline_vae, str(device), channels_last_vae + ) + baseline_call = profile.prepare_sglang_call(baseline_vae, inputs) + channels_last_call = profile.prepare_sglang_call(channels_last_vae, inputs) + + with torch.no_grad(): + with _record_conv3d_layouts() as baseline_layout: + baseline_raw = AccuracyEngine._execute_with_native_hook( + baseline_call + ) + with _record_conv3d_layouts() as channels_last_layout: + channels_last_raw = AccuracyEngine._execute_with_native_hook( + channels_last_call + ) + + baseline_out = profile.normalize_sglang_output(baseline_raw) + channels_last_out = profile.normalize_sglang_output(channels_last_raw) + + AccuracyEngine.check_accuracy( + channels_last_out, + baseline_out, + f"{case.id}_vae_channels_last_3d", + VAE_CHANNELS_LAST_3D_PARITY_THRESHOLD, + ) + + logger.info( + "[%s_vae_channels_last_3d] Conv3d layout baseline: calls=%d, " + "input_cl3d=%d, weight_cl3d=%d, mixed=%d | channels_last: " + "calls=%d, input_cl3d=%d, weight_cl3d=%d, mixed=%d", + case.id, + baseline_layout.calls, + baseline_layout.channels_last_input_calls, + baseline_layout.channels_last_weight_calls, + baseline_layout.mixed_layout_calls, + channels_last_layout.calls, + channels_last_layout.channels_last_input_calls, + channels_last_layout.channels_last_weight_calls, + channels_last_layout.mixed_layout_calls, + ) + if channels_last_layout.calls == 0: + raise RuntimeError( + f"{case.id}: VAE channels_last_3d guard did not execute Conv3d" + ) + if channels_last_layout.channels_last_weight_calls == 0: + raise RuntimeError( + f"{case.id}: VAE channels_last_3d guard did not see channels_last_3d Conv3d weights" + ) + if channels_last_layout.mixed_layout_calls: + raise RuntimeError( + f"{case.id}: {channels_last_layout.mixed_layout_calls} Conv3d calls used " + "channels_last_3d weights with non-channels_last_3d inputs" + ) + finally: + if baseline_vae is not None: + del baseline_vae + if channels_last_vae is not None: + del channels_last_vae + AccuracyEngine.reset_parallel_runtime() + AccuracyEngine.clear_memory() diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 0d3a8e44a739..daa4e541da03 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -36,6 +36,7 @@ DEFAULT_FLUX_1_DEV_MODEL_NAME_FOR_TEST, DEFAULT_FLUX_2_DEV_MODEL_NAME_FOR_TEST, DEFAULT_FLUX_2_KLEIN_4B_MODEL_NAME_FOR_TEST, + DEFAULT_FLUX_2_KLEIN_BASE_4B_MODEL_NAME_FOR_TEST, DEFAULT_JOYAI_IMAGE_EDIT_MODEL_NAME_FOR_TEST, DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST, DEFAULT_QWEN_IMAGE_EDIT_2509_MODEL_NAME_FOR_TEST, @@ -123,6 +124,15 @@ ), T2I_sampling_params, ), + DiffusionTestCase( + "flux_2_klein_base_image_t2i", + DiffusionServerArgs( + model_path=DEFAULT_FLUX_2_KLEIN_BASE_4B_MODEL_NAME_FOR_TEST, + ), + T2I_sampling_params, + run_consistency_check=False, + run_component_accuracy_check=False, + ), # TODO: replace with a faster model to test the --dit-layerwise-offload # TODO: currently, we don't support sending more than one request in test, and setting `num_outputs_per_prompt` to 2 doesn't guarantee the denoising be executed twice, # so we do one warmup and send one request instead @@ -388,7 +398,6 @@ "hunyuan3d_shape_gen", DiffusionServerArgs( model_path="tencent/Hunyuan3D-2", - enable_warmup=False, ), HUNYUAN3D_SHAPE_sampling_params, run_consistency_check=False, diff --git a/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json b/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json index 464544501fcb..e71766e3731c 100644 --- a/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json +++ b/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json @@ -1,140 +1,316 @@ { - "metadata":{ - "model":"Diffusion Server", - "hardware":"CI S5000 pool", - "description":"Reference numbers captured from the CI diffusion server baseline run" + "metadata": { + "model": "Diffusion Server", + "hardware": "CI S5000 pool", + "description": "Reference numbers captured from the CI diffusion server baseline run" }, - "scenarios":{ - "qwen_image_t2i_musa":{ - "stages_ms":{ - "InputValidationStage":0.09, - "TextEncodingStage":658.4, - "LatentPreparationStage":0.33, - "TimestepPreparationStage":24.39, - "DenoisingStage":36196.6, - "DecodingStage":40.44 + "scenarios": { + "zimage_image_t2i_musa": { + "stages_ms": { + "InputValidationStage": 0.08, + "TextEncodingStage": 3761.4, + "LatentPreparationStage": 4.06, + "TimestepPreparationStage": 21.94, + "DenoisingStage": 4873.8, + "DecodingStage": 252.49 + }, + "denoise_step_ms": { + "0": 3281.35, + "2": 209.69, + "3": 199.26, + "5": 200.42, + "6": 207.8, + "8": 205.19 + }, + "expected_e2e_ms": 12161.85, + "expected_avg_denoise_ms": 540.67, + "expected_median_denoise_ms": 205.19, + "estimated_full_test_time_s": 64.6 + }, + "qwen_image_layered_i2i_musa": { + "stages_ms": { + "QwenImageLayeredBeforeDenoisingStage": 287.12, + "TimestepPreparationStage": 0.01, + "DenoisingStage": 79962.68, + "DecodingStage": 232.86 + }, + "denoise_step_ms": { + "0": 1377.49, + "1": 1608.67, + "2": 1606.57, + "3": 1601.06, + "4": 1606.65, + "5": 1605.0, + "6": 1596.31, + "7": 1609.13, + "8": 1599.48, + "9": 1601.81, + "10": 1606.7, + "11": 1601.7, + "12": 1607.68, + "13": 1606.42, + "14": 1599.13, + "15": 1611.78, + "16": 1598.31, + "17": 1600.97, + "18": 1611.38, + "19": 1598.6, + "20": 1599.72, + "21": 1608.24, + "22": 1599.4, + "23": 1613.52, + "24": 1600.56, + "25": 1605.91, + "26": 1605.47, + "27": 1598.85, + "28": 1607.93, + "29": 1603.65, + "30": 1598.64, + "31": 1607.1, + "32": 1595.55, + "33": 1608.09, + "34": 1606.47, + "35": 1596.18, + "36": 1599.88, + "37": 1607.12, + "38": 1595.63, + "39": 1612.07, + "40": 1596.19, + "41": 1602.35, + "42": 1604.17, + "43": 1598.26, + "44": 1602.67, + "45": 1611.54, + "46": 1599.01, + "47": 1619.72, + "48": 1593.45, + "49": 1602.33 + }, + "expected_e2e_ms": 80490.53, + "expected_avg_denoise_ms": 1599.09, + "expected_median_denoise_ms": 1602.51, + "estimated_full_test_time_s": 159.4 + }, + "fast_hunyuan_video_musa": { + "stages_ms": { + "InputValidationStage": 0.08, + "TextEncodingStage": 4146.14, + "TimestepPreparationStage": 18.33, + "LatentPreparationStage": 2.42, + "DenoisingStage": 17236.59, + "DecodingStage": 7019.29 }, - "denoise_step_ms":{ - "0":666.68, - "1":732.33, - "2":721.29, - "3":729.27, - "4":725.05, - "5":721.71, - "6":722.22, - "7":725.93, - "8":724.94, - "9":724.14, - "10":730.43, - "11":719.92, - "12":726.24, - "13":722.04, - "14":727.68, - "15":720.31, - "16":721.75, - "17":725.65, - "18":720.23, - "19":724.12, - "20":726.35, - "21":723.27, - "22":731.58, - "23":724.97, - "24":721.48, - "25":722.0, - "26":722.37, - "27":719.81, - "28":721.64, - "29":724.81, - "30":723.9, - "31":725.42, - "32":719.86, - "33":728.04, - "34":728.55, - "35":723.13, - "36":722.0, - "37":730.11, - "38":724.06, - "39":728.35, - "40":728.04, - "41":726.62, - "42":728.47, - "43":728.11, - "44":728.59, - "45":721.5, - "46":724.59, - "47":729.26, - "48":726.05, - "49":721.13 + "denoise_step_ms": { + "0": 5462.17, + "1": 2338.96, + "2": 2355.67, + "3": 2353.45, + "4": 2359.64, + "5": 2363.04 }, - "expected_e2e_ms":37190.98, - "expected_avg_denoise_ms":723.72, - "expected_median_denoise_ms":724.7 + "expected_e2e_ms": 32799.37, + "expected_avg_denoise_ms": 2872.16, + "expected_median_denoise_ms": 2357.66, + "estimated_full_test_time_s": 111.3 }, - "wan2_1_t2v_1.3b_musa":{ - "stages_ms":{ - "InputValidationStage":0.12, - "TextEncodingStage":1097.75, - "LatentPreparationStage":0.24, - "TimestepPreparationStage":5.66, - "DenoisingStage":47399.84, - "DecodingStage":946.08, - "per_frame_generation":null + "qwen_image_t2i_musa": { + "stages_ms": { + "InputValidationStage": 0.09, + "TextEncodingStage": 658.4, + "LatentPreparationStage": 0.33, + "TimestepPreparationStage": 24.39, + "DenoisingStage": 36196.6, + "DecodingStage": 40.44 }, - "denoise_step_ms":{ - "0":783.06, - "1":970.52, - "2":939.72, - "3":947.58, - "4":941.44, - "5":955.26, - "6":960.39, - "7":951.84, - "8":959.68, - "9":953.33, - "10":940.87, - "11":958.5, - "12":952.7, - "13":933.4, - "14":952.0, - "15":951.6, - "16":947.04, - "17":939.28, - "18":956.88, - "19":960.1, - "20":949.73, - "21":954.77, - "22":959.98, - "23":947.37, - "24":957.51, - "25":953.39, - "26":953.73, - "27":959.57, - "28":942.59, - "29":958.05, - "30":952.76, - "31":952.76, - "32":950.6, - "33":948.76, - "34":957.53, - "35":940.86, - "36":958.11, - "37":940.9, - "38":949.1, - "39":951.81, - "40":948.61, - "41":957.28, - "42":951.41, - "43":953.09, - "44":955.69, - "45":941.93, - "46":952.96, - "47":953.5, - "48":939.25, - "49":942.69 + "denoise_step_ms": { + "0": 666.68, + "1": 732.33, + "2": 721.29, + "3": 729.27, + "4": 725.05, + "5": 721.71, + "6": 722.22, + "7": 725.93, + "8": 724.94, + "9": 724.14, + "10": 730.43, + "11": 719.92, + "12": 726.24, + "13": 722.04, + "14": 727.68, + "15": 720.31, + "16": 721.75, + "17": 725.65, + "18": 720.23, + "19": 724.12, + "20": 726.35, + "21": 723.27, + "22": 731.58, + "23": 724.97, + "24": 721.48, + "25": 722.0, + "26": 722.37, + "27": 719.81, + "28": 721.64, + "29": 724.81, + "30": 723.9, + "31": 725.42, + "32": 719.86, + "33": 728.04, + "34": 728.55, + "35": 723.13, + "36": 722.0, + "37": 730.11, + "38": 724.06, + "39": 728.35, + "40": 728.04, + "41": 726.62, + "42": 728.47, + "43": 728.11, + "44": 728.59, + "45": 721.5, + "46": 724.59, + "47": 729.26, + "48": 726.05, + "49": 721.13 }, - "expected_e2e_ms":50007.17, - "expected_avg_denoise_ms":947.83, - "expected_median_denoise_ms":952.35 + "expected_e2e_ms": 37190.98, + "expected_avg_denoise_ms": 723.72, + "expected_median_denoise_ms": 724.7, + "estimated_full_test_time_s": 137.0 + }, + "qwen_image_2512_t2i_musa": { + "stages_ms": { + "InputValidationStage": 0.07, + "TextEncodingStage": 849.1, + "LatentPreparationStage": 0.26, + "TimestepPreparationStage": 21.95, + "DenoisingStage": 31935.97, + "DecodingStage": 98.62 + }, + "denoise_step_ms": { + "0": 553.87, + "1": 639.78, + "2": 640.39, + "3": 638.78, + "4": 640.13, + "5": 639.69, + "6": 640.19, + "7": 639.67, + "8": 638.95, + "9": 640.06, + "10": 639.8, + "11": 640.81, + "12": 638.7, + "13": 640.16, + "14": 639.76, + "15": 640.17, + "16": 639.26, + "17": 640.33, + "18": 639.91, + "19": 640.47, + "20": 640.1, + "21": 639.81, + "22": 639.05, + "23": 639.7, + "24": 640.02, + "25": 640.67, + "26": 639.04, + "27": 640.21, + "28": 639.78, + "29": 643.32, + "30": 640.2, + "31": 639.54, + "32": 640.14, + "33": 640.14, + "34": 639.54, + "35": 639.22, + "36": 639.94, + "37": 639.87, + "38": 639.41, + "39": 639.6, + "40": 638.5, + "41": 639.5, + "42": 639.35, + "43": 641.43, + "44": 638.73, + "45": 639.74, + "46": 639.72, + "47": 639.72, + "48": 638.64, + "49": 638.72 + }, + "expected_e2e_ms": 32915.75, + "expected_avg_denoise_ms": 638.55, + "expected_median_denoise_ms": 640.05, + "estimated_full_test_time_s": 137.8 + }, + "wan2_1_t2v_1.3b_musa": { + "stages_ms": { + "InputValidationStage": 0.06, + "TextEncodingStage": 1381.91, + "LatentPreparationStage": 0.18, + "TimestepPreparationStage": 3.99, + "DenoisingStage": 23679.86, + "DecodingStage": 1059.53, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 331.71, + "1": 478.93, + "2": 481.37, + "3": 483.42, + "4": 476.27, + "5": 488.73, + "6": 486.79, + "7": 467.89, + "8": 465.47, + "9": 472.81, + "10": 478.41, + "11": 488.84, + "12": 474.68, + "13": 468.48, + "14": 483.79, + "15": 476.05, + "16": 483.77, + "17": 476.04, + "18": 484.27, + "19": 486.43, + "20": 483.04, + "21": 473.76, + "22": 464.09, + "23": 474.14, + "24": 470.76, + "25": 487.99, + "26": 477.77, + "27": 465.76, + "28": 483.96, + "29": 484.14, + "30": 471.99, + "31": 483.41, + "32": 486.66, + "33": 467.07, + "34": 478.04, + "35": 476.9, + "36": 462.4, + "37": 476.53, + "38": 485.65, + "39": 478.89, + "40": 465.22, + "41": 472.03, + "42": 479.65, + "43": 479.74, + "44": 479.48, + "45": 474.19, + "46": 464.47, + "47": 463.09, + "48": 463.77, + "49": 463.49 + }, + "expected_e2e_ms": 26134.51, + "expected_avg_denoise_ms": 473.44, + "expected_median_denoise_ms": 476.72, + "estimated_full_test_time_s": 101.5 }, "wan2_2_t2v_a14b_2gpu_musa": { "stages_ms": { @@ -259,6 +435,130 @@ "expected_e2e_ms": 138624.98, "expected_avg_denoise_ms": 2686.91, "expected_median_denoise_ms": 2691.24 + }, + "qwen_image_edit_t2i_musa": { + "stages_ms": { + "InputValidationStage": 35.9, + "ImageEncodingStage": 7483.38, + "ImageVAEEncodingStage": 160.82, + "LatentPreparationStage": 3.0, + "TimestepPreparationStage": 19.02, + "DenoisingStage": 84367.86, + "DecodingStage": 36.82 + }, + "denoise_step_ms": { + "0": 7612.62, + "1": 1567.7, + "2": 1559.81, + "3": 1570.75, + "4": 1568.7, + "5": 1572.44, + "6": 1565.71, + "7": 1559.33, + "8": 1567.63, + "9": 1566.83, + "10": 1560.77, + "11": 1561.03, + "12": 1565.61, + "13": 1560.43, + "14": 1562.47, + "15": 1562.22, + "16": 1564.1, + "17": 1571.28, + "18": 1563.26, + "19": 1561.37, + "20": 1559.04, + "21": 1556.07, + "22": 1577.83, + "23": 1564.54, + "24": 1564.28, + "25": 1573.79, + "26": 1572.94, + "27": 1568.81, + "28": 1568.73, + "29": 1571.71, + "30": 1557.83, + "31": 1568.7, + "32": 1570.3, + "33": 1567.36, + "34": 1566.47, + "35": 1567.2, + "36": 1560.98, + "37": 1563.43, + "38": 1570.74, + "39": 1568.01, + "40": 1560.57, + "41": 1572.64, + "42": 1564.01, + "43": 1566.34, + "44": 1569.09, + "45": 1573.18, + "46": 1566.5, + "47": 1567.04, + "48": 1570.02, + "49": 1559.91 + }, + "expected_e2e_ms": 92355.9, + "expected_avg_denoise_ms": 1687.04, + "expected_median_denoise_ms": 1566.67, + "estimated_full_test_time_s": 217.0 + }, + "qwen_image_edit_2509_ti2i_musa": { + "stages_ms": { + "InputValidationStage": 125.7, + "ImageEncodingStage": 1018.93, + "ImageVAEEncodingStage": 311.41, + "LatentPreparationStage": 0.24, + "TimestepPreparationStage": 33.39, + "DenoisingStage": 88792.03, + "DecodingStage": 320.64 + }, + "denoise_step_ms": { + "0": 1914.75, + "1": 2230.29, + "2": 2216.93, + "3": 2231.22, + "4": 2230.63, + "5": 2222.13, + "6": 2224.43, + "7": 2235.47, + "8": 2220.55, + "9": 2239.83, + "10": 2239.29, + "11": 2216.95, + "12": 2221.39, + "13": 2229.65, + "14": 2231.94, + "15": 2222.23, + "16": 2230.03, + "17": 2236.55, + "18": 2217.18, + "19": 2231.48, + "20": 2236.88, + "21": 2226.74, + "22": 2224.26, + "23": 2231.1, + "24": 2214.29, + "25": 2224.57, + "26": 2233.64, + "27": 2217.0, + "28": 2226.08, + "29": 2229.47, + "30": 2230.21, + "31": 2224.45, + "32": 2230.89, + "33": 2232.82, + "34": 2219.97, + "35": 2228.74, + "36": 2231.6, + "37": 2225.25, + "38": 2223.72, + "39": 2228.41 + }, + "expected_e2e_ms": 90612.84, + "expected_avg_denoise_ms": 2219.58, + "expected_median_denoise_ms": 2228.58, + "estimated_full_test_time_s": 220.8 } } } diff --git a/python/sglang/multimodal_gen/test/server/musa/test_server_a_musa.py b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa.py similarity index 69% rename from python/sglang/multimodal_gen/test/server/musa/test_server_a_musa.py rename to python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa.py index 7b4c4bc29f15..753fb8899b55 100644 --- a/python/sglang/multimodal_gen/test/server/musa/test_server_a_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa.py @@ -1,5 +1,5 @@ """ -MUSA-specific diffusion performance test (1-GPU). +MUSA-specific 1-GPU diffusion performance tests. """ from __future__ import annotations @@ -8,7 +8,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import ( - ONE_GPU_MUSA_CASES_A, + ONE_GPU_MUSA_CASES, ) from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 DiffusionServerBase, @@ -19,10 +19,10 @@ logger = init_logger(__name__) -class TestDiffusionServerOneGpuMusaImage(DiffusionServerBase): - """Performance tests for 1-GPU diffusion cases on MUSA""" +class TestDiffusionServerOneGpuMusa(DiffusionServerBase): + """Performance tests for 1-GPU diffusion cases on MUSA.""" - @pytest.fixture(params=ONE_GPU_MUSA_CASES_A, ids=lambda c: c.id) + @pytest.fixture(params=ONE_GPU_MUSA_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: """Provide a DiffusionTestCase for each 1-GPU MUSA test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/musa/test_server_b_musa.py b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa_nightly.py similarity index 57% rename from python/sglang/multimodal_gen/test/server/musa/test_server_b_musa.py rename to python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa_nightly.py index f961648fa0ca..7b0384fee700 100644 --- a/python/sglang/multimodal_gen/test/server/musa/test_server_b_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa_nightly.py @@ -1,5 +1,5 @@ """ -MUSA-specific diffusion performance test (1-GPU). +MUSA-specific 1-GPU diffusion performance tests for nightly suite. """ from __future__ import annotations @@ -8,7 +8,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import ( - ONE_GPU_MUSA_CASES_B, + ONE_GPU_NIGHTLY_MUSA_CASES, ) from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 DiffusionServerBase, @@ -19,10 +19,10 @@ logger = init_logger(__name__) -class TestDiffusionServerOneGpuMusaVideo(DiffusionServerBase): - """Performance tests for 1-GPU diffusion cases on MUSA""" +class TestDiffusionServerOneGpuMusaNightly(DiffusionServerBase): + """Performance tests for 1-GPU diffusion cases on MUSA (nightly-only).""" - @pytest.fixture(params=ONE_GPU_MUSA_CASES_B, ids=lambda c: c.id) + @pytest.fixture(params=ONE_GPU_NIGHTLY_MUSA_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: - """Provide a DiffusionTestCase for each 1-GPU MUSA test.""" + """Provide a DiffusionTestCase for each 1-GPU MUSA nightly test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_a_musa.py b/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_musa.py similarity index 76% rename from python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_a_musa.py rename to python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_musa.py index 48909e1a2325..bacf613dcc29 100644 --- a/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_a_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_musa.py @@ -1,5 +1,5 @@ """ -MUSA-specific 2-GPU diffusion performance test. +MUSA-specific 2-GPU diffusion performance tests. """ from __future__ import annotations @@ -8,7 +8,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import ( - TWO_GPU_MUSA_CASES_A, + TWO_GPU_MUSA_CASES, ) from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 DiffusionServerBase, @@ -19,10 +19,10 @@ logger = init_logger(__name__) -class TestDiffusionServerTwoGpuMusaA(DiffusionServerBase): +class TestDiffusionServerTwoGpuMusa(DiffusionServerBase): """Performance tests for 2-GPU diffusion cases on MUSA.""" - @pytest.fixture(params=TWO_GPU_MUSA_CASES_A, ids=lambda c: c.id) + @pytest.fixture(params=TWO_GPU_MUSA_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: """Provide a DiffusionTestCase for each 2-GPU MUSA test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py b/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py index ca8363349452..c3019dedbd66 100644 --- a/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py @@ -1,32 +1,49 @@ from __future__ import annotations +from dataclasses import replace +from functools import lru_cache + from sglang.multimodal_gen.test.server.testcase_configs import ( T2V_PROMPT, DiffusionSamplingParams, DiffusionServerArgs, DiffusionTestCase, + MULTI_FRAME_I2I_sampling_params, + MULTI_IMAGE_TI2I_sampling_params, T2I_sampling_params, + T2V_sampling_params, + TI2I_sampling_params, TI2V_sampling_params, ) -ONE_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [ + +@lru_cache(maxsize=None) +def hf_cached_model(repo_id: str) -> str: + """Resolve an HF repo id to the local cache snapshot prepared on MUSA runners.""" + from huggingface_hub import snapshot_download + + return snapshot_download(repo_id, local_files_only=True) + + +MUSA_TI2I_sampling_params = replace( + TI2I_sampling_params, + image_path="/hf-cache/hub/musa-test-assets/TI2I_Qwen_Image_Edit_Input.jpg", +) + +ONE_GPU_MUSA_CASES: list[DiffusionTestCase] = [ DiffusionTestCase( "qwen_image_t2i_musa", DiffusionServerArgs( - model_path="Qwen/Qwen-Image", + model_path=hf_cached_model("Qwen/Qwen-Image"), modality="image", ), T2I_sampling_params, run_consistency_check=False, ), -] - - -ONE_GPU_MUSA_CASES_B: list[DiffusionTestCase] = [ DiffusionTestCase( "wan2_1_t2v_1.3b_musa", DiffusionServerArgs( - model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + model_path=hf_cached_model("Wan-AI/Wan2.1-T2V-1.3B-Diffusers"), modality="video", custom_validator="video", ), @@ -38,11 +55,75 @@ ] -TWO_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [ +NIGHTLY_1_GPU_MUSA_CASES: list[DiffusionTestCase] = [ + DiffusionTestCase( + "zimage_image_t2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Tongyi-MAI/Z-Image-Turbo"), + modality="image", + ), + T2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_layered_i2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-Layered"), + modality="image", + ), + MULTI_FRAME_I2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "fast_hunyuan_video_musa", + DiffusionServerArgs( + model_path=hf_cached_model("FastVideo/FastHunyuan-diffusers"), + modality="video", + custom_validator="video", + ), + T2V_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_2512_t2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-2512"), + modality="image", + ), + T2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_edit_t2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-Edit"), + modality="image", + ), + MUSA_TI2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_edit_2509_ti2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-Edit-2509"), + modality="image", + ), + MULTI_IMAGE_TI2I_sampling_params, + run_consistency_check=False, + ), +] + + +ONE_GPU_NIGHTLY_MUSA_CASES: list[DiffusionTestCase] = ( + ONE_GPU_MUSA_CASES + NIGHTLY_1_GPU_MUSA_CASES +) + + +TWO_GPU_MUSA_CASES: list[DiffusionTestCase] = [ DiffusionTestCase( "wan2_1_i2v_14b_480P_2gpu_musa", DiffusionServerArgs( - model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + model_path=hf_cached_model("Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"), modality="video", custom_validator="video", num_gpus=2, diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines.json b/python/sglang/multimodal_gen/test/server/perf_baselines.json index aec8a7ffe809..80981acf96d9 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines.json @@ -321,6 +321,73 @@ "expected_median_denoise_ms": 39.47, "estimated_full_test_time_s": 120.5 }, + "flux_2_klein_base_image_t2i": { + "stages_ms": { + "InputValidationStage": 0.06, + "TextEncodingStage": 505.51, + "ImageVAEEncodingStage": 0.01, + "LatentPreparationStage": 1.14, + "TimestepPreparationStage": 53.76, + "DenoisingStage": 12365.93, + "DecodingStage": 11.74 + }, + "denoise_step_ms": { + "0": 72.0, + "1": 192.12, + "2": 217.22, + "3": 217.5, + "4": 257.12, + "5": 262.19, + "6": 246.18, + "7": 224.94, + "8": 252.71, + "9": 262.82, + "10": 246.37, + "11": 231.87, + "12": 253.34, + "13": 260.94, + "14": 242.16, + "15": 235.92, + "16": 254.31, + "17": 262.02, + "18": 243.92, + "19": 239.61, + "20": 253.47, + "21": 259.75, + "22": 245.48, + "23": 240.91, + "24": 253.84, + "25": 255.36, + "26": 248.31, + "27": 243.75, + "28": 250.5, + "29": 251.34, + "30": 246.87, + "31": 243.8, + "32": 249.8, + "33": 254.02, + "34": 247.4, + "35": 244.77, + "36": 252.03, + "37": 248.85, + "38": 249.74, + "39": 248.6, + "40": 252.29, + "41": 249.25, + "42": 249.85, + "43": 249.68, + "44": 252.65, + "45": 249.2, + "46": 249.32, + "47": 248.62, + "48": 252.87, + "49": 249.19 + }, + "expected_e2e_ms": 13075.51, + "expected_avg_denoise_ms": 243.34, + "expected_median_denoise_ms": 249.22, + "estimated_full_test_time_s": 124.4 + }, "layerwise_offload": { "stages_ms": { "TextEncodingStage": 176.59, @@ -2442,7 +2509,7 @@ "per_frame_generation": null }, "denoise_step_ms": { - "0": 179.26, + "0": 225.43, "1": 283.15, "2": 193.3, "3": 166.82, @@ -2534,7 +2601,7 @@ "32": 209.80 }, "expected_e2e_ms": 12000.0, - "expected_avg_denoise_ms": 238.8, + "expected_avg_denoise_ms": 299.01, "expected_median_denoise_ms": 246.85, "estimated_full_test_time_s": 170.0 }, diff --git a/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py b/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py index 1d41f9b0b0b4..67741bb6fd61 100644 --- a/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py +++ b/python/sglang/multimodal_gen/test/server/test_component_accuracy_1_gpu.py @@ -15,6 +15,15 @@ ) from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine +VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS = { + "wan2_1_t2v_1.3b", +} +VAE_CHANNELS_LAST_3D_PARITY_CASES = [ + case + for case in ACCURACY_ONE_GPU_CASES + if case.id in VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS +] + @pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES, ids=lambda case: case.id) class TestComponentAccuracy1GPU: @@ -63,3 +72,16 @@ def test_encoder_accuracy(self, case): case, case.server_args.num_gpus, ) + + +@pytest.mark.parametrize( + "case", VAE_CHANNELS_LAST_3D_PARITY_CASES, ids=lambda case: case.id +) +class TestVAEChannelsLast3DParity1GPU: + """1-GPU VAE guard for channels_last_3d drift.""" + + def test_vae_channels_last_3d_parity(self, case): + AccuracyEngine.run_vae_channels_last_3d_parity( + case, + case.server_args.num_gpus, + ) diff --git a/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py b/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py index 8e8d3dfaa86f..bb5c0a54aa77 100644 --- a/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py +++ b/python/sglang/multimodal_gen/test/server/test_component_accuracy_2_gpu.py @@ -15,6 +15,15 @@ ) from sglang.multimodal_gen.test.server.component_accuracy import AccuracyEngine +VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS = { + "wan2_2_i2v_a14b_2gpu", +} +VAE_CHANNELS_LAST_3D_PARITY_CASES = [ + case + for case in ACCURACY_TWO_GPU_CASES + if case.id in VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS +] + @pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES, ids=lambda case: case.id) class TestComponentAccuracy2GPU: @@ -63,3 +72,16 @@ def test_encoder_accuracy(self, case): case, case.server_args.num_gpus, ) + + +@pytest.mark.parametrize( + "case", VAE_CHANNELS_LAST_3D_PARITY_CASES, ids=lambda case: case.id +) +class TestVAEChannelsLast3DParity2GPU: + """2-GPU VAE guard for channels_last_3d drift.""" + + def test_vae_channels_last_3d_parity(self, case): + AccuracyEngine.run_vae_channels_last_3d_parity( + case, + case.server_args.num_gpus, + ) diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index 3635799f6e1d..eb1329aadd97 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -128,9 +128,6 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext: if server_args.lora_path: extra_args += f" --lora-path {server_args.lora_path}" - if server_args.enable_warmup: - extra_args += " --warmup" - # Strict ports: fail immediately if port is occupied instead of silently # picking another one (which causes the test client to connect to the wrong server). extra_args += " --strict-ports" diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index a8ff76c3e45b..e3b47a22b81b 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -186,7 +186,6 @@ class DiffusionServerArgs: dit_offload_prefetch_size: int | float | None = None enable_cache_dit: bool = False text_encoder_cpu_offload: bool = False - enable_warmup: bool = True extras: list[str] = field(default_factory=lambda: []) env_vars: dict[str, str] = field(default_factory=dict) @@ -452,10 +451,8 @@ def from_req_perf_record( MODELOPT_FLUX1_NVFP4_TRANSFORMER = "lmsys/flux1-dev-modelopt-nvfp4-sglang-transformer" MODELOPT_FLUX2_NVFP4_WEIGHTS = "black-forest-labs/FLUX.2-dev-NVFP4" MODELOPT_WAN22_NVFP4_MODEL = "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4" -MODELOPT_NVFP4_B200_ENV_VARS = {"SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND": "cudnn"} -MODELOPT_WAN22_NVFP4_B200_ENV_VARS = { - "SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND": "trtllm" -} +MODELOPT_NVFP4_B200_ENV_VARS = {} +MODELOPT_WAN22_NVFP4_B200_ENV_VARS = {} def _make_modelopt_ci_case( @@ -473,7 +470,6 @@ def _make_modelopt_ci_case( DiffusionServerArgs( model_path=model_path, modality=modality, - enable_warmup=False, extras=extras, env_vars=env_vars or {}, ), diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 29da29308c41..0e115293bb48 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -33,7 +33,7 @@ logger = init_logger(__name__) -SGL_TEST_FILES_CI_DATA_REVISION = "94eab4fcca6d4ddc77cdb3622f13033b61e81002" +SGL_TEST_FILES_CI_DATA_REVISION = "b7455318873fc5af399c8447b3bb0d9471a5084c" SGL_TEST_FILES_CONSISTENCY_GT_ROOT = ( "https://raw.githubusercontent.com/" f"sgl-project/ci-data/{SGL_TEST_FILES_CI_DATA_REVISION}/" diff --git a/python/sglang/multimodal_gen/test/unit/test_cfg_gating.py b/python/sglang/multimodal_gen/test/unit/test_cfg_gating.py new file mode 100644 index 000000000000..37f66c9c720b --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_cfg_gating.py @@ -0,0 +1,180 @@ +import os +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( + DenoisingStage, +) + + +class _PipelineConfig: + def get_classifier_free_guidance_scale(self, batch, current_guidance_scale): + return current_guidance_scale + + def slice_noise_pred(self, noise_pred, latents): + return noise_pred + + def postprocess_cfg_noise(self, batch, noise_pred, noise_pred_cond): + return noise_pred + + +class TestCFGGating(unittest.TestCase): + def _make_server_args(self, enable_cfg_parallel=False): + return SimpleNamespace( + enable_cfg_parallel=enable_cfg_parallel, + pipeline_config=_PipelineConfig(), + ) + + def _make_batch(self): + return SimpleNamespace( + cfg_normalization=0, + guidance_rescale=0, + do_classifier_free_guidance=True, + is_cfg_negative=False, + ) + + def _make_gate_state(self, gate_step=1, model_id=None, delta=None): + return { + "fraction": 0.5, + "requested": True, + "active": True, + "gate_step": gate_step, + "delta": delta, + "model_id": model_id, + "fresh_uncond": 0, + "reused": 0, + "invalidations": 0, + } + + def test_reuses_unconditional_residual_after_gate_step(self): + stage = DenoisingStage.__new__(DenoisingStage) + batch = self._make_batch() + server_args = self._make_server_args() + policy = CFGPolicy().build(batch, {}, {}, {}) + calls = [] + + def fake_predict_noise(**kwargs): + calls.append("neg" if batch.is_cfg_negative else "pos") + timestep = kwargs["timestep"] + timestep_value = float(timestep.item()) + offset = 0.25 if batch.is_cfg_negative else 1.25 + return torch.tensor([timestep_value + offset]) + + stage._predict_noise = fake_predict_noise + model = torch.nn.Identity() + latents = torch.zeros(1) + state = self._make_gate_state(gate_step=1) + + first = stage._predict_noise_with_cfg( + current_model=model, + latent_model_input=latents, + timestep=torch.tensor(0), + batch=batch, + timestep_index=0, + attn_metadata=None, + target_dtype=torch.float32, + current_guidance_scale=4.0, + cfg_policy=policy, + cfg_gate_state=state, + server_args=server_args, + guidance=None, + latents=latents, + ) + second = stage._predict_noise_with_cfg( + current_model=model, + latent_model_input=latents, + timestep=torch.tensor(1), + batch=batch, + timestep_index=1, + attn_metadata=None, + target_dtype=torch.float32, + current_guidance_scale=4.0, + cfg_policy=policy, + cfg_gate_state=state, + server_args=server_args, + guidance=None, + latents=latents, + ) + + self.assertTrue(torch.equal(first, torch.tensor([4.25]))) + self.assertTrue(torch.equal(second, torch.tensor([5.25]))) + self.assertEqual(calls, ["pos", "neg", "pos"]) + self.assertEqual(state["fresh_uncond"], 1) + self.assertEqual(state["reused"], 1) + self.assertEqual(state["invalidations"], 0) + + def test_model_switch_invalidates_cached_delta(self): + stage = DenoisingStage.__new__(DenoisingStage) + batch = self._make_batch() + server_args = self._make_server_args() + policy = CFGPolicy().build(batch, {}, {}, {}) + calls = [] + + def fake_predict_noise(**kwargs): + calls.append("neg" if batch.is_cfg_negative else "pos") + value = 3.0 if batch.is_cfg_negative else 10.0 + return torch.tensor([value]) + + stage._predict_noise = fake_predict_noise + old_model = torch.nn.Identity() + new_model = torch.nn.Identity() + latents = torch.zeros(1) + state = self._make_gate_state( + gate_step=0, + model_id=id(old_model), + delta=(torch.tensor([2.0]),), + ) + + output = stage._predict_noise_with_cfg( + current_model=new_model, + latent_model_input=latents, + timestep=torch.tensor(2), + batch=batch, + timestep_index=2, + attn_metadata=None, + target_dtype=torch.float32, + current_guidance_scale=2.0, + cfg_policy=policy, + cfg_gate_state=state, + server_args=server_args, + guidance=None, + latents=latents, + ) + + self.assertTrue(torch.equal(output, torch.tensor([17.0]))) + self.assertEqual(calls, ["pos", "neg"]) + self.assertEqual(state["model_id"], id(new_model)) + self.assertEqual(state["fresh_uncond"], 1) + self.assertEqual(state["reused"], 0) + self.assertEqual(state["invalidations"], 1) + + def test_cfg_parallel_disables_gate_state(self): + stage = DenoisingStage.__new__(DenoisingStage) + ctx = SimpleNamespace(timesteps=torch.arange(10), extra={}, is_warmup=True) + batch = self._make_batch() + server_args = self._make_server_args(enable_cfg_parallel=True) + + with patch.dict(os.environ, {"SGLANG_DIFFUSION_CFG_GATE_STEP": "0.5"}): + stage._init_cfg_gate_state(ctx, batch, server_args) + + self.assertTrue(ctx.extra["cfg_gate_state"]["requested"]) + self.assertFalse(ctx.extra["cfg_gate_state"]["active"]) + self.assertEqual(ctx.extra["cfg_gate_state"]["gate_step"], 11) + + def test_rejects_invalid_gate_fraction(self): + stage = DenoisingStage.__new__(DenoisingStage) + ctx = SimpleNamespace(timesteps=torch.arange(10), extra={}, is_warmup=True) + batch = self._make_batch() + server_args = self._make_server_args() + + with patch.dict(os.environ, {"SGLANG_DIFFUSION_CFG_GATE_STEP": "1.5"}): + with self.assertRaises(ValueError): + stage._init_cfg_gate_state(ctx, batch, server_args) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_disagg_roles.py b/python/sglang/multimodal_gen/test/unit/test_disagg_roles.py new file mode 100644 index 000000000000..6bc441937745 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_disagg_roles.py @@ -0,0 +1,641 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for disaggregation role-based module filtering.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.multimodal_gen.configs.pipeline_configs.hunyuan3d import ( + Hunyuan3D2PipelineConfig, +) +from sglang.multimodal_gen.runtime import server_args as server_args_module +from sglang.multimodal_gen.runtime.disaggregation.roles import ( + RoleType, + filter_modules_for_role, + get_module_role, +) +from sglang.multimodal_gen.runtime.pipelines.flux_2 import Flux2Pipeline +from sglang.multimodal_gen.runtime.pipelines.glm_image import GlmImagePipeline +from sglang.multimodal_gen.runtime.pipelines.hunyuan3d_pipeline import ( + Hunyuan3D2Pipeline, +) +from sglang.multimodal_gen.runtime.pipelines.ltx_2_pipeline import LTX2Pipeline +from sglang.multimodal_gen.runtime.pipelines.mova_pipeline import ( + MOVAPipeline, + MOVAPipelineAlias, +) +from sglang.multimodal_gen.runtime.pipelines.qwen_image import ( + QwenImageEditPipeline, + QwenImageLayeredPipeline, +) +from sglang.multimodal_gen.runtime.pipelines.wan_i2v_dmd_pipeline import ( + WanImageToVideoDmdPipeline, +) +from sglang.multimodal_gen.runtime.pipelines.wan_i2v_pipeline import ( + WanImageToVideoPipeline, +) +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising_av import ( + LTX2RefinementStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.hunyuan3d_shape import ( + Hunyuan3DShapeBeforeDenoisingStage, + Hunyuan3DShapeExportStage, + Hunyuan3DShapeSaveStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.image_encoding import ( + ImageVAEEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.helios_denoising import ( + HeliosChunkedDenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.mova import ( + MOVADecodingStage, + MOVADenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image_layered import ( + QwenImageLayeredBeforeDenoisingStage, + _resolve_text_encoder_dtype, +) +from sglang.multimodal_gen.runtime.server_args import set_global_server_args + + +class _GlobalStageArgsMixin: + def _install_stage_server_args(self, **kwargs): + server_args = SimpleNamespace( + comfyui_mode=False, + enable_torch_compile=False, + enable_cfg_parallel=False, + attention_backend=None, + **kwargs, + ) + set_global_server_args(server_args) + return server_args + + def setUp(self): + super().setUp() + self._prev_global_server_args = server_args_module._global_server_args + self._install_stage_server_args() + + def tearDown(self): + set_global_server_args(self._prev_global_server_args) + super().tearDown() + + +class TestRoleType(unittest.TestCase): + def test_from_string(self): + self.assertEqual(RoleType.from_string("monolithic"), RoleType.MONOLITHIC) + self.assertEqual(RoleType.from_string("encoder"), RoleType.ENCODER) + self.assertEqual(RoleType.from_string("denoiser"), RoleType.DENOISER) + self.assertEqual(RoleType.from_string("decoder"), RoleType.DECODER) + self.assertEqual(RoleType.from_string("ENCODER"), RoleType.ENCODER) + + def test_from_string_backward_compat(self): + self.assertEqual(RoleType.from_string("denoising"), RoleType.DENOISER) + + def test_from_string_invalid(self): + with self.assertRaises(ValueError): + RoleType.from_string("invalid") + + def test_choices(self): + choices = RoleType.choices() + self.assertIn("monolithic", choices) + self.assertIn("encoder", choices) + self.assertIn("denoiser", choices) + self.assertIn("denoising", choices) + self.assertIn("decoder", choices) + + +class TestGetModuleRole(unittest.TestCase): + def test_encoder_modules(self): + self.assertEqual(get_module_role("text_encoder"), RoleType.ENCODER) + self.assertEqual(get_module_role("text_encoder_2"), RoleType.ENCODER) + self.assertEqual(get_module_role("tokenizer"), RoleType.ENCODER) + self.assertEqual(get_module_role("tokenizer_2"), RoleType.ENCODER) + self.assertEqual(get_module_role("image_encoder"), RoleType.ENCODER) + self.assertEqual(get_module_role("image_processor"), RoleType.ENCODER) + self.assertEqual(get_module_role("connectors"), RoleType.ENCODER) + self.assertEqual(get_module_role("vision_language_encoder"), RoleType.ENCODER) + self.assertEqual(get_module_role("hy3dshape_conditioner"), RoleType.ENCODER) + self.assertEqual(get_module_role("hy3dshape_image_processor"), RoleType.ENCODER) + + def test_denoiser_modules(self): + self.assertEqual(get_module_role("transformer"), RoleType.DENOISER) + self.assertEqual(get_module_role("transformer_2"), RoleType.DENOISER) + self.assertEqual(get_module_role("video_dit"), RoleType.DENOISER) + self.assertEqual(get_module_role("video_dit_2"), RoleType.DENOISER) + self.assertEqual(get_module_role("audio_dit"), RoleType.DENOISER) + self.assertEqual(get_module_role("dual_tower_bridge"), RoleType.DENOISER) + self.assertEqual(get_module_role("hy3dshape_model"), RoleType.DENOISER) + + def test_decoder_modules(self): + self.assertEqual(get_module_role("vae"), RoleType.DECODER) + self.assertEqual(get_module_role("audio_vae"), RoleType.DECODER) + self.assertEqual(get_module_role("video_vae"), RoleType.DECODER) + self.assertEqual(get_module_role("vocoder"), RoleType.DECODER) + self.assertEqual(get_module_role("hy3dshape_vae"), RoleType.DECODER) + + def test_shared_modules(self): + self.assertIsNone(get_module_role("scheduler")) + self.assertIsNone(get_module_role("hy3dshape_scheduler")) + + +class TestFilterModulesForRole(unittest.TestCase): + WAN_MODULES = ["text_encoder", "tokenizer", "vae", "transformer", "scheduler"] + + def test_monolithic_keeps_all(self): + result = filter_modules_for_role(self.WAN_MODULES, RoleType.MONOLITHIC) + self.assertEqual(result, self.WAN_MODULES) + + def test_encoder_does_not_keep_decoder_modules_by_default(self): + result = filter_modules_for_role(self.WAN_MODULES, RoleType.ENCODER) + self.assertEqual(result, ["text_encoder", "tokenizer", "scheduler"]) + + def test_encoder_can_keep_explicit_cross_role_modules(self): + result = filter_modules_for_role( + self.WAN_MODULES, + RoleType.ENCODER, + extra_allowed_modules={"vae"}, + ) + self.assertEqual(result, ["text_encoder", "tokenizer", "vae", "scheduler"]) + + def test_denoiser_skips_encoders_and_vae(self): + result = filter_modules_for_role(self.WAN_MODULES, RoleType.DENOISER) + self.assertEqual(result, ["transformer", "scheduler"]) + + def test_decoder_keeps_vae_and_scheduler(self): + result = filter_modules_for_role(self.WAN_MODULES, RoleType.DECODER) + self.assertEqual(result, ["vae", "scheduler"]) + + +class TestFilterModulesLTX2(unittest.TestCase): + LTX2_MODULES = [ + "transformer", + "text_encoder", + "tokenizer", + "scheduler", + "vae", + "audio_vae", + "vocoder", + "connectors", + ] + + def test_decoder_includes_audio(self): + result = filter_modules_for_role(self.LTX2_MODULES, RoleType.DECODER) + self.assertEqual(result, ["scheduler", "vae", "audio_vae", "vocoder"]) + + def test_encoder_does_not_keep_decoder_modules_by_default(self): + result = filter_modules_for_role(self.LTX2_MODULES, RoleType.ENCODER) + self.assertEqual( + result, ["text_encoder", "tokenizer", "scheduler", "connectors"] + ) + + def test_denoiser_can_keep_ti2v_decoder_components(self): + result = filter_modules_for_role( + self.LTX2_MODULES, + RoleType.DENOISER, + extra_allowed_modules={"vae", "audio_vae"}, + ) + self.assertEqual(result, ["transformer", "scheduler", "vae", "audio_vae"]) + + +# Consolidated from test_pipeline_stage_role_filter.py. +class _FakePipeline(ComposedPipelineBase): + pipeline_name = "FakePipeline" + _required_config_modules = [] + + def initialize_pipeline(self, server_args): + pass + + def create_pipeline_stages(self, server_args) -> None: + pass + + +def _make_pipeline(role: RoleType) -> _FakePipeline: + pipeline = object.__new__(_FakePipeline) + pipeline.modules = {} + pipeline._stages = [] + pipeline._stage_name_mapping = {} + pipeline._disagg_role = role + return pipeline + + +class _FakeStage: + def __init__(self, role_affinity: RoleType): + self.role_affinity = role_affinity + self.registered_stage_name = None + self.profile_stage_name = None + + def set_registered_stage_name(self, stage_name: str) -> None: + self.registered_stage_name = stage_name + + def set_profile_stage_name(self, stage_name: str) -> None: + self.profile_stage_name = stage_name + + +class TestPipelineStageRoleFilter(unittest.TestCase): + def test_stage_factory_skips_without_constructing_for_other_role(self): + pipeline = _make_pipeline(RoleType.ENCODER) + + def should_not_construct(): + raise AssertionError("stage factory should have been skipped") + + pipeline.add_stage_factory( + RoleType.DENOISER, + should_not_construct, + "denoising_stage", + ) + + self.assertEqual(pipeline.stages, []) + + def test_stage_factory_constructs_for_matching_role(self): + pipeline = _make_pipeline(RoleType.DENOISER) + stage = _FakeStage(RoleType.DENOISER) + events = [] + + def create_stage(): + events.append("called") + return stage + + pipeline.add_stage_factory( + RoleType.DENOISER, + create_stage, + "denoising_stage", + ) + + self.assertEqual(events, ["called"]) + self.assertIs(pipeline.get_stage("denoising_stage"), stage) + self.assertEqual(stage.registered_stage_name, "denoising_stage") + + def test_encoder_role_does_not_construct_standard_denoising_stage(self): + pipeline = _make_pipeline(RoleType.ENCODER) + + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base.DenoisingStage", + side_effect=AssertionError("DenoisingStage should not be constructed"), + ): + pipeline.add_standard_denoising_stage() + + self.assertEqual(pipeline.stages, []) + + def test_encoder_role_does_not_construct_standard_decoding_stage(self): + pipeline = _make_pipeline(RoleType.ENCODER) + + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base.DecodingStage", + side_effect=AssertionError("DecodingStage should not be constructed"), + ): + pipeline.add_standard_decoding_stage() + + self.assertEqual(pipeline.stages, []) + + +# Consolidated from test_disagg_pipeline_alignment.py. +class TestPipelineSpecificExtraModules(unittest.TestCase): + def _get_extra_modules( + self, pipeline_cls, role: RoleType, task_name: str + ) -> set[str]: + pipeline = object.__new__(pipeline_cls) + return pipeline._get_extra_allowed_modules_for_role(role, task_name) + + def test_flux_encoder_keeps_vae(self): + extras = self._get_extra_modules(Flux2Pipeline, RoleType.ENCODER, "ti2i") + filtered = filter_modules_for_role( + Flux2Pipeline._required_config_modules, + RoleType.ENCODER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, {"vae"}) + self.assertEqual( + set(filtered), {"text_encoder", "tokenizer", "vae", "scheduler"} + ) + + def test_qwen_image_edit_encoder_keeps_vae(self): + extras = self._get_extra_modules( + QwenImageEditPipeline, RoleType.ENCODER, "ti2i" + ) + filtered = filter_modules_for_role( + QwenImageEditPipeline._required_config_modules, + RoleType.ENCODER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, {"vae"}) + self.assertEqual( + set(filtered), + {"processor", "scheduler", "text_encoder", "tokenizer", "vae"}, + ) + + def test_qwen_image_layered_encoder_keeps_required_cross_role_modules(self): + extras = self._get_extra_modules( + QwenImageLayeredPipeline, RoleType.ENCODER, "ti2i" + ) + filtered = filter_modules_for_role( + QwenImageLayeredPipeline._required_config_modules, + RoleType.ENCODER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, {"vae", "transformer"}) + self.assertNotIn( + "text_encoder", QwenImageLayeredPipeline._required_config_modules + ) + self.assertEqual( + set(filtered), + { + "vae", + "tokenizer", + "processor", + "transformer", + "scheduler", + }, + ) + + def test_glm_image_encoder_keeps_vae_and_transformer(self): + extras = self._get_extra_modules(GlmImagePipeline, RoleType.ENCODER, "ti2i") + filtered = filter_modules_for_role( + GlmImagePipeline._required_config_modules, + RoleType.ENCODER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, {"vae", "transformer"}) + self.assertEqual( + set(filtered), + { + "text_encoder", + "tokenizer", + "vae", + "vision_language_encoder", + "processor", + "transformer", + "scheduler", + }, + ) + + def test_wan_ti2v_denoiser_keeps_vae(self): + for pipeline_cls in (WanImageToVideoPipeline, WanImageToVideoDmdPipeline): + extras = self._get_extra_modules(pipeline_cls, RoleType.DENOISER, "ti2v") + filtered = filter_modules_for_role( + pipeline_cls._required_config_modules, + RoleType.DENOISER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, {"vae"}) + self.assertEqual(set(filtered), {"vae", "transformer", "scheduler"}) + + def test_ltx2_encoder_does_not_keep_decoder_modules(self): + extras = self._get_extra_modules(LTX2Pipeline, RoleType.ENCODER, "ti2v") + filtered = filter_modules_for_role( + LTX2Pipeline._required_config_modules, + RoleType.ENCODER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, set()) + self.assertEqual( + set(filtered), + {"text_encoder", "tokenizer", "scheduler", "connectors"}, + ) + + def test_ltx2_ti2v_denoiser_keeps_vae_and_audio_vae(self): + extras = self._get_extra_modules(LTX2Pipeline, RoleType.DENOISER, "ti2v") + filtered = filter_modules_for_role( + LTX2Pipeline._required_config_modules, + RoleType.DENOISER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, {"vae", "audio_vae"}) + self.assertEqual( + set(filtered), {"transformer", "scheduler", "vae", "audio_vae"} + ) + + def test_mova_encoder_keeps_video_and_audio_vaes(self): + extras = self._get_extra_modules(MOVAPipeline, RoleType.ENCODER, "i2v") + filtered = filter_modules_for_role( + MOVAPipeline._required_config_modules, + RoleType.ENCODER, + extra_allowed_modules=extras, + ) + self.assertEqual(extras, {"video_vae", "audio_vae"}) + self.assertEqual( + set(filtered), + {"video_vae", "audio_vae", "text_encoder", "tokenizer", "scheduler"}, + ) + + def test_mova_alias_uses_same_encoder_extras(self): + extras = self._get_extra_modules(MOVAPipelineAlias, RoleType.ENCODER, "i2v") + self.assertEqual(extras, {"video_vae", "audio_vae"}) + + +class TestQwenImageLayeredDtype(_GlobalStageArgsMixin, unittest.TestCase): + def test_text_encoder_dtype_uses_parameter_dtype_without_dtype_attr(self): + text_encoder = torch.nn.Linear(1, 1, bias=False).to(dtype=torch.bfloat16) + self.assertEqual( + _resolve_text_encoder_dtype(text_encoder), + torch.bfloat16, + ) + + def test_component_uses_keep_standard_text_encoder_and_configured_dtypes(self): + class _DummyVAE: + temperal_downsample = [] + z_dim = 16 + + def to(self, *args, **kwargs): + return self + + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image_layered.get_local_torch_device", + return_value=torch.device("cpu"), + ): + stage = QwenImageLayeredBeforeDenoisingStage( + vae=_DummyVAE(), + text_encoder=torch.nn.Linear(1, 1), + tokenizer=object(), + processor=object(), + transformer=object(), + scheduler=object(), + model_path="/unused", + vae_dtype=torch.float32, + text_encoder_dtype=torch.float16, + ) + + uses = stage.component_uses(SimpleNamespace(), "qwen_layered") + self.assertEqual( + [(use.component_name, use.target_dtype) for use in uses], + [("text_encoder", torch.float16), ("vae", torch.float32)], + ) + + +class TestImageVAEEncodingStageComponentName(_GlobalStageArgsMixin, unittest.TestCase): + def test_component_name_can_follow_non_default_vae_key(self): + stage = ImageVAEEncodingStage(vae=object(), component_name="video_vae") + server_args = SimpleNamespace( + pipeline_config=SimpleNamespace(vae_precision="bf16") + ) + + uses = stage.component_uses(server_args, "image_vae_encoding") + self.assertEqual(len(uses), 1) + self.assertEqual(uses[0].component_name, "video_vae") + self.assertEqual(uses[0].target_dtype, torch.bfloat16) + + +class TestStageAffinityAndValidation(_GlobalStageArgsMixin, unittest.TestCase): + def _make_hunyuan_pipeline( + self, role: RoleType, *, paint_enable: bool + ) -> Hunyuan3D2Pipeline: + pipeline = object.__new__(Hunyuan3D2Pipeline) + pipeline.server_args = self._install_stage_server_args( + pipeline_config=Hunyuan3D2PipelineConfig(paint_enable=paint_enable) + ) + pipeline._disagg_role = role + pipeline.modules = { + "hy3dshape_image_processor": object(), + "hy3dshape_conditioner": object(), + "hy3dshape_scheduler": object(), + "hy3dshape_model": torch.nn.Linear(1, 1), + "hy3dshape_vae": object(), + } + pipeline._stages = [] + pipeline._stage_name_mapping = {} + return pipeline + + def test_helios_denoising_stage_is_denoiser_affine(self): + stage = object.__new__(HeliosChunkedDenoisingStage) + self.assertEqual(stage.role_affinity, RoleType.DENOISER) + + def test_mova_denoising_stage_is_denoiser_affine(self): + stage = object.__new__(MOVADenoisingStage) + self.assertEqual(stage.role_affinity, RoleType.DENOISER) + + def test_mova_decoding_stage_is_decoder_affine(self): + stage = object.__new__(MOVADecodingStage) + self.assertEqual(stage.role_affinity, RoleType.DECODER) + + def test_mova_skips_torch_compile_on_rocm(self): + class _CompileTrackingModule(torch.nn.Module): + def __init__(self): + super().__init__() + self.compile_called = False + + def compile(self, *args, **kwargs): + self.compile_called = True + + stage = object.__new__(MOVADenoisingStage) + module = _CompileTrackingModule() + server_args = SimpleNamespace(enable_torch_compile=True) + + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.mova.current_platform.is_hip", + return_value=True, + ): + stage._maybe_enable_torch_compile(module, server_args) + + self.assertFalse(module.compile_called) + + def test_hunyuan3d_shape_only_disagg_accepts_non_monolithic_roles(self): + pipeline = self._make_hunyuan_pipeline(RoleType.ENCODER, paint_enable=False) + pipeline.validate_disagg_role(RoleType.ENCODER) + pipeline.validate_disagg_role(RoleType.MONOLITHIC) + + def test_hunyuan3d_disagg_rejects_paint_pipeline(self): + pipeline = self._make_hunyuan_pipeline(RoleType.ENCODER, paint_enable=True) + with self.assertRaisesRegex(ValueError, "shape-only disaggregation"): + pipeline.validate_disagg_role(RoleType.ENCODER) + + def test_hunyuan3d_shape_export_and_save_are_decoder_affine(self): + export_stage = Hunyuan3DShapeExportStage( + vae=object(), + config=Hunyuan3D2PipelineConfig(paint_enable=False), + ) + save_stage = Hunyuan3DShapeSaveStage( + config=Hunyuan3D2PipelineConfig(paint_enable=False), + ) + + self.assertEqual(export_stage.role_affinity, RoleType.DECODER) + self.assertEqual(save_stage.role_affinity, RoleType.DECODER) + + def test_hunyuan3d_stage_filtering_matches_shape_only_roles(self): + expected = { + RoleType.ENCODER: ["shape_before_denoising"], + RoleType.DENOISER: ["shape_denoising"], + RoleType.DECODER: ["shape_export", "shape_save"], + } + + for role, stage_names in expected.items(): + pipeline = self._make_hunyuan_pipeline(role, paint_enable=False) + pipeline.create_pipeline_stages(pipeline.server_args) + self.assertEqual(list(pipeline._stage_name_mapping.keys()), stage_names) + + def test_hunyuan3d_shape_stage_no_longer_stores_model_dtype(self): + pipeline = self._make_hunyuan_pipeline(RoleType.ENCODER, paint_enable=False) + pipeline.create_pipeline_stages(pipeline.server_args) + stage = pipeline._stage_name_mapping["shape_before_denoising"] + self.assertIsInstance(stage, Hunyuan3DShapeBeforeDenoisingStage) + self.assertFalse(hasattr(stage, "model_dtype")) + + def test_ltx2_refinement_stage_keeps_class_name_stage_key(self): + stage = object.__new__(LTX2RefinementStage) + self.assertEqual( + ComposedPipelineBase._infer_stage_name(stage), "LTX2RefinementStage" + ) + + +class TestHunyuan3DShapeStageRuntimeDtype(_GlobalStageArgsMixin, unittest.TestCase): + def test_conditioner_parameter_dtype_wins_over_sample_dtype(self): + conditioner = torch.nn.Linear(4, 4, bias=False).to(dtype=torch.float32) + stage = Hunyuan3DShapeBeforeDenoisingStage( + image_processor=object(), + conditioner=conditioner, + scheduler=SimpleNamespace(init_noise_sigma=1.0), + config=Hunyuan3D2PipelineConfig(), + latent_shape=(1, 2, 2), + guidance_embed=False, + ) + + self.assertEqual( + stage._resolve_runtime_dtype(torch.zeros(1, dtype=torch.float16)), + torch.float32, + ) + + def test_runtime_dtype_falls_back_to_sample_tensor_without_module_dtype(self): + stage = Hunyuan3DShapeBeforeDenoisingStage( + image_processor=object(), + conditioner=object(), + scheduler=SimpleNamespace(init_noise_sigma=1.0), + config=Hunyuan3D2PipelineConfig(), + latent_shape=(1, 2, 2), + guidance_embed=False, + ) + + self.assertEqual( + stage._resolve_runtime_dtype(torch.zeros(1, dtype=torch.bfloat16)), + torch.bfloat16, + ) + + def test_runtime_dtype_warns_and_falls_back_after_non_iterable_parameters(self): + conditioner = SimpleNamespace( + parameters=lambda: (_ for _ in ()).throw(TypeError("not iterable")), + buffers=lambda: iter(()), + ) + stage = Hunyuan3DShapeBeforeDenoisingStage( + image_processor=object(), + conditioner=conditioner, + scheduler=SimpleNamespace(init_noise_sigma=1.0), + config=Hunyuan3D2PipelineConfig(), + latent_shape=(1, 2, 2), + guidance_embed=False, + ) + + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.hunyuan3d_shape.logger.warning" + ) as mock_warning: + dtype = stage._resolve_runtime_dtype(torch.zeros(1, dtype=torch.float16)) + + self.assertEqual(dtype, torch.float16) + mock_warning.assert_called_once() + self.assertEqual(mock_warning.call_args.args[1], "parameters") + + +if __name__ == "__main__": + unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_fp32_layernorm.py b/python/sglang/multimodal_gen/test/unit/test_fp32_layernorm.py new file mode 100644 index 000000000000..ed4616e7dff6 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_fp32_layernorm.py @@ -0,0 +1,71 @@ +import pytest +import torch +import torch.nn.functional as F + +from sglang.multimodal_gen.runtime.layers.layernorm import FP32LayerNorm + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_fp32_layernorm_cache_matches_reference(): + norm = FP32LayerNorm(16, eps=1e-5).cuda().to(torch.bfloat16) + inputs = torch.randn(4, 16, device="cuda", dtype=torch.bfloat16) + + with torch.no_grad(): + actual = norm(inputs) + expected = F.layer_norm( + inputs.float(), + norm.normalized_shape, + norm.weight.float().to(device=inputs.device), + norm.bias.float().to(device=inputs.device), + norm.eps, + ).to(inputs.dtype) + + torch.testing.assert_close(actual, expected) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_fp32_layernorm_cache_reuses_converted_params(): + norm = FP32LayerNorm(16, eps=1e-5).cuda().to(torch.bfloat16) + inputs = torch.randn(4, 16, device="cuda", dtype=torch.bfloat16) + + with torch.no_grad(): + norm(inputs) + weight_cache = norm.__dict__["_weight_fp32_cache"] + bias_cache = norm.__dict__["_bias_fp32_cache"] + + norm(inputs) + + assert norm.__dict__["_weight_fp32_cache"][1] is weight_cache[1] + assert norm.__dict__["_bias_fp32_cache"][1] is bias_cache[1] + assert "_weight_fp32_cache" not in norm.state_dict() + assert "_bias_fp32_cache" not in norm.state_dict() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_fp32_layernorm_cache_invalidates_on_param_update(): + norm = FP32LayerNorm(16, eps=1e-5).cuda().to(torch.bfloat16) + inputs = torch.randn(4, 16, device="cuda", dtype=torch.bfloat16) + + with torch.no_grad(): + norm(inputs) + first_key, first_weight = norm.__dict__["_weight_fp32_cache"] + + norm.weight.add_(1.0) + norm(inputs) + second_key, second_weight = norm.__dict__["_weight_fp32_cache"] + + assert second_key != first_key + assert second_weight is not first_weight + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +def test_fp32_layernorm_grad_mode_preserves_autograd_path(): + norm = FP32LayerNorm(16, eps=1e-5).cuda().to(torch.bfloat16) + inputs = torch.randn(4, 16, device="cuda", dtype=torch.bfloat16, requires_grad=True) + + output = norm(inputs).float().sum() + output.backward() + + assert inputs.grad is not None + assert "_weight_fp32_cache" not in norm.__dict__ + assert "_bias_fp32_cache" not in norm.__dict__ diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py new file mode 100644 index 000000000000..e4e14d402935 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py @@ -0,0 +1,121 @@ +from collections import defaultdict +from contextlib import contextmanager, nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA +from sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline import LoRAPipeline + +_RANK_PATCH = "sglang.multimodal_gen.runtime.pipelines_core.lora_pipeline.dist.get_rank" + + +class _TestLoRAPipeline(LoRAPipeline): + def create_pipeline_stages(self, server_args): + return None + + +def _make_layer() -> BaseLayerWithLoRA: + return BaseLayerWithLoRA(torch.nn.Linear(2, 2, bias=False)) + + +def _make_pipeline(layer: BaseLayerWithLoRA) -> _TestLoRAPipeline: + pipeline = object.__new__(_TestLoRAPipeline) + pipeline.modules = {"transformer": torch.nn.Module()} + pipeline.server_args = SimpleNamespace(lora_merge_mode="dynamic") + pipeline.lora_initialized = True + pipeline.lora_adapters = defaultdict(dict) + pipeline.loaded_adapter_paths = {"adapter": "/adapter"} + pipeline.cur_adapter_name = {} + pipeline.cur_adapter_path = {} + pipeline.cur_adapter_strength = {} + pipeline.cur_adapter_config = {} + pipeline.lora_layers = {"linear": layer} + pipeline.lora_layers_transformer_2 = {} + pipeline.lora_layers_critic = {} + pipeline.is_lora_merged = {} + + pipeline.lora_adapters["adapter"]["linear.lora_A"] = torch.ones(1, 2) + pipeline.lora_adapters["adapter"]["linear.lora_B"] = torch.ones(2, 1) + return pipeline + + +def test_dynamic_lora_reactivates_cached_layers_without_weight_update_context(): + layer = _make_layer() + pipeline = _make_pipeline(layer) + context_calls = 0 + + @contextmanager + def counted_context(*args, **kwargs): + nonlocal context_calls + context_calls += 1 + yield [] + + pipeline._temporarily_disable_offload = counted_context + + with patch(_RANK_PATCH, return_value=0): + pipeline.set_lora( + "adapter", + "/adapter", + target="transformer", + strength=0.75, + merge_mode="dynamic", + ) + + first_lora_a = layer.lora_A + first_lora_b = layer.lora_B + assert context_calls == 0 + assert not layer.disable_lora + + pipeline._temporarily_disable_offload = lambda *args, **kwargs: nullcontext([]) + pipeline.deactivate_lora_weights("transformer") + assert layer.disable_lora + + def fail_apply(*args, **kwargs): + raise AssertionError("cached dynamic LoRA should not rebuild weights") + + context_calls = 0 + pipeline._temporarily_disable_offload = counted_context + pipeline._apply_lora_to_layers = fail_apply + + with patch(_RANK_PATCH, return_value=0): + pipeline.set_lora( + "adapter", + None, + target="transformer", + strength=0.75, + merge_mode="dynamic", + ) + + assert context_calls == 0 + assert not layer.disable_lora + assert layer.lora_A is first_lora_a + assert layer.lora_B is first_lora_b + + +def test_merged_lora_still_uses_weight_update_context(): + layer = _make_layer() + pipeline = _make_pipeline(layer) + context_calls = 0 + + @contextmanager + def counted_context(*args, **kwargs): + nonlocal context_calls + context_calls += 1 + yield [] + + pipeline._temporarily_disable_offload = counted_context + + with patch(_RANK_PATCH, return_value=0): + pipeline.set_lora( + "adapter", + "/adapter", + target="transformer", + strength=1.0, + merge_mode="merge", + ) + + assert context_calls == 1 + assert layer.merged + assert pipeline.is_lora_merged["transformer"] diff --git a/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py b/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py index 34aaccb03846..3f0edafb373e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py +++ b/python/sglang/multimodal_gen/test/unit/test_multi_output_grouping.py @@ -22,7 +22,9 @@ class CountingDedupStage(PipelineStage): deduplicated_extra_tensor_tree_output_keys = ("mu",) def __init__(self): - self.server_args = SimpleNamespace(comfyui_mode=True) + self.server_args = SimpleNamespace( + comfyui_mode=True, enable_layerwise_nvtx_marker=False + ) self.forward_calls = 0 def build_dedup_fingerprint(self, batch: Req, server_args): @@ -40,7 +42,9 @@ def forward(self, batch: Req, server_args) -> Req: class CountingLatentStage(LatentPreparationStage): def __init__(self): - self.server_args = SimpleNamespace(comfyui_mode=True) + self.server_args = SimpleNamespace( + comfyui_mode=True, enable_layerwise_nvtx_marker=False + ) self.prepare_group_calls = 0 self.forward_calls = 0 diff --git a/python/sglang/multimodal_gen/test/unit/test_nvtx_pytorch_hooks.py b/python/sglang/multimodal_gen/test/unit/test_nvtx_pytorch_hooks.py new file mode 100644 index 000000000000..5483ceef5d24 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_nvtx_pytorch_hooks.py @@ -0,0 +1,374 @@ +"""Unit tests for ``sglang.multimodal_gen.runtime.utils.nvtx_pytorch_hooks``. + +These tests cover the CPU-only surface: the ``maybe_nvtx_range`` helper, +``DiffusionNvtxHooks.register_hooks`` / ``remove_hooks`` lifecycle, and the +shape-collection helper. The actual ``nvtx.range_push`` / ``range_pop`` calls +require CUDA and are exercised end-to-end by Nsight-Systems profiling runs. +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentResidencyManager, + ComponentUse, +) +from sglang.multimodal_gen.runtime.utils import nvtx_pytorch_hooks +from sglang.multimodal_gen.runtime.utils.nvtx_pytorch_hooks import ( + DiffusionNvtxHooks, + _collect_input_shapes, + maybe_nvtx_range, +) + + +class TestMaybeNvtxRange(unittest.TestCase): + def test_disabled_returns_noop_context_manager(self) -> None: + ran = False + with maybe_nvtx_range("never", enabled=False): + ran = True + self.assertTrue(ran) + + def test_disabled_propagates_exception(self) -> None: + with self.assertRaises(RuntimeError): + with maybe_nvtx_range("never", enabled=False): + raise RuntimeError("boom") + + def test_disabled_does_not_call_nvtx(self) -> None: + with ( + patch.object(nvtx_pytorch_hooks.nvtx, "range_push") as push, + patch.object(nvtx_pytorch_hooks.nvtx, "range_pop") as pop, + ): + with maybe_nvtx_range("never", enabled=False): + pass + push.assert_not_called() + pop.assert_not_called() + + def test_enabled_calls_matched_push_pop(self) -> None: + with ( + patch.object(nvtx_pytorch_hooks.nvtx, "range_push") as push, + patch.object(nvtx_pytorch_hooks.nvtx, "range_pop") as pop, + ): + with maybe_nvtx_range("stage_X", enabled=True): + pass + push.assert_called_once_with("stage_X") + pop.assert_called_once_with() + + def test_enabled_pops_on_exception(self) -> None: + with ( + patch.object(nvtx_pytorch_hooks.nvtx, "range_push") as push, + patch.object(nvtx_pytorch_hooks.nvtx, "range_pop") as pop, + ): + with self.assertRaises(RuntimeError): + with maybe_nvtx_range("stage_X", enabled=True): + raise RuntimeError("boom") + push.assert_called_once_with("stage_X") + pop.assert_called_once_with() + + def test_marker_with_braces_does_not_raise(self) -> None: + """Regression: torch.cuda.nvtx.range() str-formats its argument, + which would raise on a marker containing a literal ``{``. The helper + calls range_push directly to sidestep that.""" + with ( + patch.object(nvtx_pytorch_hooks.nvtx, "range_push"), + patch.object(nvtx_pytorch_hooks.nvtx, "range_pop"), + ): + with maybe_nvtx_range("layer in={1, 2, 3}", enabled=True): + pass + + +class _TinyBlock(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear = torch.nn.Linear(4, 4) + self.norm = torch.nn.LayerNorm(4) + # Dropout is in _DEFAULT_SKIP_TYPES and must not be instrumented. + self.drop = torch.nn.Dropout(p=0.0) + + +class TestDiffusionNvtxHooks(unittest.TestCase): + def test_register_hooks_counts_non_skipped_submodules(self) -> None: + block = _TinyBlock() + hooks = DiffusionNvtxHooks() + # 4 modules total (block, linear, norm, drop); drop is skipped. + self.assertEqual(hooks.register_hooks(block, prefix="block"), 3) + # 2 hooks (pre + post) registered per instrumented module. + self.assertEqual(len(hooks._hook_handles), 6) + + def test_register_hooks_skips_duplicate_instances(self) -> None: + shared = torch.nn.Linear(4, 4) + + class TiedModel(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.a = shared + self.b = shared + + model = TiedModel() + hooks = DiffusionNvtxHooks() + # Root + 1 unique linear (the second occurrence is skipped). + self.assertEqual(hooks.register_hooks(model), 2) + + def test_remove_hooks_is_idempotent(self) -> None: + block = _TinyBlock() + hooks = DiffusionNvtxHooks() + hooks.register_hooks(block) + hooks.remove_hooks() + self.assertEqual(hooks._hook_handles, []) + self.assertEqual(hooks._module_to_name_map, {}) + # Second call is a no-op, not an error. + hooks.remove_hooks() + + def test_set_enabled_false_suppresses_nvtx_calls(self) -> None: + """When disabled, neither pre- nor post-hook should call nvtx — + guarantees no half-open push without a matching pop.""" + hooks = DiffusionNvtxHooks() + dummy = torch.nn.Linear(2, 2) + hooks._module_to_name_map[dummy] = "dummy" + hooks.set_enabled(False) + with ( + patch.object(nvtx_pytorch_hooks.nvtx, "range_push") as push, + patch.object(nvtx_pytorch_hooks.nvtx, "range_pop") as pop, + ): + hooks._forward_pre_hook(dummy, (torch.zeros(2),), {}) + hooks._forward_hook(dummy, (), None) + push.assert_not_called() + pop.assert_not_called() + + def test_set_enabled_true_emits_matched_push_pop(self) -> None: + """When enabled, a forward pre/post pair emits exactly one push + and one pop with the qualified module name as the marker.""" + hooks = DiffusionNvtxHooks() + dummy = torch.nn.Linear(2, 2) + hooks._module_to_name_map[dummy] = "dummy" + hooks.set_enabled(True) + with ( + patch.object(nvtx_pytorch_hooks.nvtx, "range_push") as push, + patch.object(nvtx_pytorch_hooks.nvtx, "range_pop") as pop, + ): + hooks._forward_pre_hook(dummy, (torch.zeros(2, 3),), {}) + hooks._forward_hook(dummy, (), None) + push.assert_called_once() + marker = push.call_args.args[0] + self.assertIn("dummy", marker) + self.assertIn("[2, 3]", marker) + pop.assert_called_once_with() + + def test_default_enabled_is_false(self) -> None: + """Default off so an unguarded forward (e.g. early warmup) cannot + emit ranges; the caller must explicitly enable via set_enabled.""" + self.assertFalse(DiffusionNvtxHooks()._enabled) + + def test_post_hook_fires_on_forward_exception(self) -> None: + """Regression: ``always_call=True`` on the registered post-hook + guarantees ``range_pop`` runs even when the wrapped ``forward`` + raises. Without it an OOM (or any other forward-time exception) + would leak a half-open NVTX range.""" + + class _RaisingModule(torch.nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + raise RuntimeError("simulated forward exception") + + model = _RaisingModule() + hooks = DiffusionNvtxHooks() + hooks.register_hooks(model, prefix="raising") + hooks.set_enabled(True) + with ( + patch.object(nvtx_pytorch_hooks.nvtx, "range_push") as push, + patch.object(nvtx_pytorch_hooks.nvtx, "range_pop") as pop, + ): + with self.assertRaises(RuntimeError): + model(torch.zeros(2)) + # One push from the pre-hook, one pop from the post-hook fired via + # always_call=True; they must match to keep the stack balanced. + self.assertEqual(push.call_count, pop.call_count) + self.assertEqual(push.call_count, 1) + + +class _NoOpResidencyStrategy: + name = "noop" + + def prepare_for_use(self, module, use, state) -> None: + pass + + def wait_for_use(self, module, use, state) -> None: + pass + + def finish_use(self, module, use, state) -> None: + pass + + def finish_request(self, module, use, state, *, preferred: bool) -> None: + pass + + def prefetch_for_use(self, module, use, state) -> bool: + return False + + def prepare_after_request(self, module, use, state) -> None: + pass + + +def _test_manager( + modules: dict[str, torch.nn.Module], + *, + enable_flag: bool = True, + is_warmup: bool = False, +) -> ComponentResidencyManager: + pipeline = SimpleNamespace( + modules=modules, + _stage_name_mapping={}, + component_residency_strategies={}, + ) + server_args = SimpleNamespace(enable_layerwise_nvtx_marker=enable_flag) + manager = ComponentResidencyManager(pipeline, server_args) + manager.state.batch_is_warmup = is_warmup + manager.strategy_for = lambda _component_name, _module: _NoOpResidencyStrategy() + return manager + + +class TestComponentResidencyNvtxHooks(unittest.TestCase): + def test_disabled_flag_is_noop(self) -> None: + module = torch.nn.Linear(2, 2) + manager = _test_manager({"linear": module}, enable_flag=False) + manager.begin_use(ComponentUse("Stage", "linear"), module=module) + self.assertEqual(manager._nvtx_hooks_by_use_key, {}) + + def test_warmup_is_noop(self) -> None: + module = torch.nn.Linear(2, 2) + manager = _test_manager({"linear": module}, is_warmup=True) + manager.begin_use(ComponentUse("Stage", "linear"), module=module) + self.assertEqual(manager._nvtx_hooks_by_use_key, {}) + + def test_begin_use_registers_and_enables_component_hooks(self) -> None: + module = torch.nn.Linear(2, 2) + manager = _test_manager({"linear": module}) + use = ComponentUse("Stage", "linear") + + manager.begin_use(use, module=module) + + _, hooks = manager._nvtx_hooks_by_use_key[("Stage", "linear", None)] + self.assertTrue(hooks._enabled) + self.assertIn(module, hooks._module_to_name_map) + self.assertTrue(hooks._module_to_name_map[module].startswith("Stage.linear")) + + def test_end_use_disables_component_hooks(self) -> None: + module = torch.nn.Linear(2, 2) + manager = _test_manager({"linear": module}) + use = ComponentUse("Stage", "linear") + + manager.begin_use(use, module=module) + _, hooks = manager._nvtx_hooks_by_use_key[("Stage", "linear", None)] + manager.end_use(use, module=module) + + self.assertFalse(hooks._enabled) + self.assertIsNone(manager._active_nvtx_key) + + def test_remove_nvtx_hooks_for_module_drops_stale_reference(self) -> None: + module = torch.nn.Linear(2, 2) + manager = _test_manager({"linear": module}) + use = ComponentUse("Stage", "linear") + + manager.begin_use(use, module=module) + _, hooks = manager._nvtx_hooks_by_use_key[("Stage", "linear", None)] + manager.remove_nvtx_hooks_for_module(module) + + self.assertEqual(manager._nvtx_hooks_by_use_key, {}) + self.assertEqual(hooks._module_to_name_map, {}) + self.assertIsNone(manager._active_nvtx_key) + + def test_re_registers_when_module_identity_changes(self) -> None: + use = ComponentUse("Stage", "linear") + first_module = torch.nn.Linear(2, 2) + manager = _test_manager({"linear": first_module}) + + manager.begin_use(use, module=first_module) + _, first_hooks = manager._nvtx_hooks_by_use_key[("Stage", "linear", None)] + manager.end_use(use, module=first_module) + + second_module = torch.nn.Linear(2, 2) + manager.pipeline.modules["linear"] = second_module + manager.begin_use(use, module=second_module) + + _, second_hooks = manager._nvtx_hooks_by_use_key[("Stage", "linear", None)] + self.assertIsNot(second_hooks, first_hooks) + self.assertEqual(first_hooks._module_to_name_map, {}) + self.assertIn(second_module, second_hooks._module_to_name_map) + + def test_same_component_in_different_stages_switches_active_prefix(self) -> None: + shared = torch.nn.Linear(2, 2) + manager = _test_manager({"vae": shared}) + first_use = ComponentUse("ImageVAEEncodingStage", "vae") + second_use = ComponentUse("DecodingStage", "vae") + + manager.begin_use(first_use, module=shared) + _, first_hooks = manager._nvtx_hooks_by_use_key[ + ("ImageVAEEncodingStage", "vae", None) + ] + manager.begin_use(second_use, module=shared) + _, second_hooks = manager._nvtx_hooks_by_use_key[("DecodingStage", "vae", None)] + + self.assertFalse(first_hooks._enabled) + self.assertTrue(second_hooks._enabled) + self.assertTrue( + second_hooks._module_to_name_map[shared].startswith("DecodingStage.vae") + ) + + def test_pipeline_stage_call_sets_explicit_range_gate_before_forward(self) -> None: + from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( + PipelineStage, + ) + + class _Spy(PipelineStage): + def __init__(self) -> None: + self.server_args = type( + "Args", + (), + { + "enable_layerwise_nvtx_marker": True, + "comfyui_mode": False, + }, + )() + self._component_residency_manager = None + self._registered_stage_name = None + self._profile_stage_name = None + self._current_use_nvtx = False + self.use_nvtx_during_forward: bool | None = None + + def forward(self, batch, server_args): + self.use_nvtx_during_forward = self.current_use_nvtx + return batch + + class _Batch: + is_warmup = False + metrics = None + perf_dump_path = None + + spy = _Spy() + spy(_Batch(), spy.server_args) + self.assertTrue(spy.use_nvtx_during_forward) + self.assertFalse(spy.current_use_nvtx) + + +class TestCollectInputShapes(unittest.TestCase): + def test_flat_positional_tensors(self) -> None: + a = torch.zeros(2, 3) + b = torch.zeros(4) + self.assertEqual(_collect_input_shapes((a, b)), [[2, 3], [4]]) + + def test_kwarg_tensors_are_captured(self) -> None: + kw = {"hidden_states": torch.zeros(1, 4)} + self.assertEqual(_collect_input_shapes((), kw), [[1, 4]]) + + def test_nested_tuple_kwarg_recurses(self) -> None: + rope = (torch.zeros(8, 16), torch.zeros(8, 16)) + kw = {"image_rotary_emb": rope} + self.assertEqual(_collect_input_shapes((), kw), [[8, 16], [8, 16]]) + + def test_non_tensor_values_are_skipped(self) -> None: + kw = {"scale": 1.0, "use_cache": True, "extras": None} + self.assertEqual(_collect_input_shapes((42, "s"), kw), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_pipeline_stage_profiling.py b/python/sglang/multimodal_gen/test/unit/test_pipeline_stage_profiling.py index edb5e141b677..a461aac32d31 100644 --- a/python/sglang/multimodal_gen/test/unit/test_pipeline_stage_profiling.py +++ b/python/sglang/multimodal_gen/test/unit/test_pipeline_stage_profiling.py @@ -7,7 +7,9 @@ class NamedNoOpStage(PipelineStage): def __init__(self): - self.server_args = SimpleNamespace(comfyui_mode=True) + self.server_args = SimpleNamespace( + comfyui_mode=True, enable_layerwise_nvtx_marker=False + ) def forward(self, batch: Req, server_args) -> Req: return batch diff --git a/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py b/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py index 0b99d3efe1f1..abcb8446232e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py +++ b/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py @@ -21,14 +21,18 @@ def __init__(self): def encode_text(self, *args, **kwargs): self.calls += 1 - embeds = torch.full((1, 1, 1), float(self.calls)) - mask = torch.ones((1, 1), dtype=torch.int64) - return [embeds], [mask], [], [mask], [[1]] + text = args[0] + batch_size = len(text) if isinstance(text, list) else 1 + embeds = torch.full((batch_size, 1, 1), float(self.calls)) + mask = torch.ones((batch_size, 1), dtype=torch.int64) + return [embeds], [mask], [], [mask], [[1] * batch_size] def make_req(**kwargs): defaults = { + "prompt": "hello", "negative_prompt": "bad quality", + "do_classifier_free_guidance": True, "prompt_template": {"template": "{}"}, "max_sequence_length": 1024, "is_warmup": False, @@ -37,12 +41,30 @@ def make_req(**kwargs): return SimpleNamespace(**defaults) +def make_server_args(**kwargs): + defaults = { + "pipeline_class_name": "LTX2TwoStagePipeline", + "model_path": "dummy-model", + "backend": "auto", + "model_id": None, + "pipeline_config": SimpleNamespace(text_encoder_configs=[]), + } + defaults.update(kwargs) + return SimpleNamespace(**defaults) + + +def get_negative_embedding_twice(stage, server_args, first_req, second_req=None): + stage.get_or_compute_negative_text_embedding(first_req, server_args, [0]) + stage.get_or_compute_negative_text_embedding( + second_req if second_req is not None else make_req(), server_args, [0] + ) + + def test_negative_text_cache_key_tracks_encode_options(): stage = DummyTextEncodingStage() - server_args = SimpleNamespace(pipeline_class_name="LTX2TwoStagePipeline") + server_args = make_server_args() - stage.get_or_compute_negative_text_embedding(make_req(), server_args, [0]) - stage.get_or_compute_negative_text_embedding(make_req(), server_args, [0]) + get_negative_embedding_twice(stage, server_args, make_req()) assert stage.calls == 1 stage.get_or_compute_negative_text_embedding( @@ -58,11 +80,23 @@ def test_negative_text_cache_key_tracks_encode_options(): def test_negative_text_cache_skips_warmup(): stage = DummyTextEncodingStage() - server_args = SimpleNamespace(pipeline_class_name="LTX2TwoStagePipeline") + server_args = make_server_args() - stage.get_or_compute_negative_text_embedding( - make_req(is_warmup=True), server_args, [0] - ) - stage.get_or_compute_negative_text_embedding(make_req(), server_args, [0]) + with patch.object( + stage, "_get_model_default_negative_prompt", return_value="default negative" + ): + get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True)) assert stage.calls == 2 + + +def test_negative_text_cache_keeps_default_warmup(): + stage = DummyTextEncodingStage() + server_args = make_server_args() + + with patch.object( + stage, "_get_model_default_negative_prompt", return_value="bad quality" + ): + get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True)) + + assert stage.calls == 1 diff --git a/python/sglang/multimodal_gen/test/unit/test_vae_loader.py b/python/sglang/multimodal_gen/test/unit/test_vae_loader.py index fd9b52f5647e..df0e9761af8c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_vae_loader.py +++ b/python/sglang/multimodal_gen/test/unit/test_vae_loader.py @@ -1,10 +1,40 @@ import unittest +from unittest.mock import patch import torch +from sglang.multimodal_gen.runtime.loader.component_loaders import vae_loader from sglang.multimodal_gen.runtime.loader.component_loaders.vae_loader import ( _backfill_ltx2_audio_vae_latent_stats, + _should_use_channels_last_3d, ) +from sglang.multimodal_gen.runtime.models.vaes.parallel import wan_common_utils + + +class _FakeServerArgs: + def __init__(self, pipeline_config, num_gpus=1): + self.pipeline_config = pipeline_config + self.num_gpus = num_gpus + + +class QwenImagePipelineConfig: + pass + + +class WanT2V480PConfig: + pass + + +class FastWan2_2_TI2V_5B_Config: + pass + + +class Wan2_2_I2V_A14B_Config: + pass + + +class LTX2PipelineConfig: + pass class TestVAELoader(unittest.TestCase): @@ -43,6 +73,143 @@ def test_backfill_ltx2_audio_vae_latent_stats_skips_non_audio_vae(self): self.assertNotIn("latents_mean", loaded) self.assertNotIn("latents_std", loaded) + def test_channels_last_3d_defaults_true_for_qwen_image_on_cuda(self): + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(QwenImagePipelineConfig()) + self.assertTrue(_should_use_channels_last_3d(server_args, "vae")) + + def test_channels_last_3d_defaults_true_for_single_gpu_wan_on_cuda(self): + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(WanT2V480PConfig(), num_gpus=1) + self.assertTrue(_should_use_channels_last_3d(server_args, "video_vae")) + + def test_channels_last_3d_defaults_true_for_single_gpu_fast_wan_on_cuda(self): + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(FastWan2_2_TI2V_5B_Config(), num_gpus=1) + self.assertTrue(_should_use_channels_last_3d(server_args, "video_vae")) + + def test_channels_last_3d_defaults_false_for_multi_gpu_wan_on_cuda(self): + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(Wan2_2_I2V_A14B_Config(), num_gpus=2) + self.assertFalse(_should_use_channels_last_3d(server_args, "video_vae")) + + def test_channels_last_3d_defaults_false_for_ltx_on_cuda(self): + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(LTX2PipelineConfig(), num_gpus=2) + self.assertFalse(_should_use_channels_last_3d(server_args, "video_vae")) + + def test_channels_last_3d_can_be_disabled_by_env(self): + with ( + patch.dict( + "os.environ", {"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": "false"} + ), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(QwenImagePipelineConfig()) + self.assertFalse(_should_use_channels_last_3d(server_args, "vae")) + + def test_channels_last_3d_can_be_enabled_by_env(self): + with ( + patch.dict("os.environ", {"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": "true"}), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(LTX2PipelineConfig(), num_gpus=2) + self.assertTrue(_should_use_channels_last_3d(server_args, "video_vae")) + + def test_channels_last_3d_auto_uses_model_policy(self): + with ( + patch.dict("os.environ", {"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": "auto"}), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + wan_args = _FakeServerArgs(WanT2V480PConfig(), num_gpus=1) + ltx_args = _FakeServerArgs(LTX2PipelineConfig(), num_gpus=2) + + self.assertTrue(_should_use_channels_last_3d(wan_args, "video_vae")) + self.assertFalse(_should_use_channels_last_3d(ltx_args, "video_vae")) + + def test_channels_last_3d_skips_non_video_vae_components(self): + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(vae_loader.current_platform, "is_cuda", return_value=True), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(QwenImagePipelineConfig()) + self.assertFalse(_should_use_channels_last_3d(server_args, "audio_vae")) + + def test_channels_last_3d_skips_unsupported_platforms(self): + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(vae_loader.current_platform, "is_cuda", return_value=False), + patch.object(vae_loader.current_platform, "is_rocm", return_value=False), + ): + server_args = _FakeServerArgs(QwenImagePipelineConfig()) + self.assertFalse(_should_use_channels_last_3d(server_args, "vae")) + + @unittest.skipUnless( + hasattr(torch, "channels_last_3d"), "channels_last_3d is unavailable" + ) + def test_match_conv3d_input_format_skips_non_cuda_platforms(self): + x = torch.randn(1, 3, 2, 4, 4) + weight = torch.randn(3, 3, 1, 1, 1).contiguous( + memory_format=torch.channels_last_3d + ) + + with ( + patch.object( + wan_common_utils.current_platform, "is_cuda", return_value=False + ), + patch.object( + wan_common_utils.current_platform, "is_rocm", return_value=False + ), + ): + out = wan_common_utils.match_conv3d_input_format(x, weight) + + self.assertIs(out, x) + + @unittest.skipUnless( + hasattr(torch, "channels_last_3d"), "channels_last_3d is unavailable" + ) + def test_match_conv3d_input_format_uses_channels_last_3d_on_cuda(self): + x = torch.randn(1, 3, 2, 4, 4) + weight = torch.randn(3, 3, 1, 1, 1).contiguous( + memory_format=torch.channels_last_3d + ) + + with ( + patch.object( + wan_common_utils.current_platform, "is_cuda", return_value=True + ), + patch.object( + wan_common_utils.current_platform, "is_rocm", return_value=False + ), + ): + out = wan_common_utils.match_conv3d_input_format(x, weight) + + self.assertTrue(out.is_contiguous(memory_format=torch.channels_last_3d)) + if __name__ == "__main__": unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_video_sparse_attention.py b/python/sglang/multimodal_gen/test/unit/test_video_sparse_attention.py new file mode 100644 index 000000000000..3e1ce351e54e --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_video_sparse_attention.py @@ -0,0 +1,39 @@ +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn import ( + VideoSparseAttentionImpl, + VideoSparseAttentionMetadataBuilder, +) + + +def test_video_sparse_attention_tile_buffer_reuse_and_untile(): + metadata = VideoSparseAttentionMetadataBuilder().build( + current_timestep=0, + raw_latent_shape=(5, 7, 9), + patch_size=(1, 1, 1), + VSA_sparsity=0.5, + device=torch.device("cpu"), + ) + + impl = object.__new__(VideoSparseAttentionImpl) + total_seq_length = metadata.total_seq_length + x = torch.arange(2 * total_seq_length * 3 * 4, dtype=torch.float32).reshape( + 2, total_seq_length, 3, 4 + ) + + tiled = impl.preprocess_qkv(x, metadata) + assert metadata.tile_buf is tiled + assert torch.equal( + metadata.untile_combined_index, + metadata.non_pad_index[metadata.reverse_tile_partition_indices], + ) + assert torch.equal(impl.postprocess_output(tiled, metadata), x) + + next_x = x + 1 + next_tiled = impl.preprocess_qkv(next_x, metadata) + assert next_tiled.data_ptr() == tiled.data_ptr() + assert torch.equal(impl.postprocess_output(next_tiled, metadata), next_x) + + pad_mask = torch.ones(next_tiled.shape[1], dtype=torch.bool) + pad_mask[metadata.non_pad_index.cpu()] = False + assert torch.all(next_tiled[:, pad_mask] == 0) diff --git a/python/sglang/srt/arg_groups/nemotron_h_hook.py b/python/sglang/srt/arg_groups/nemotron_h_hook.py index b0955a7aa0e3..87b568f3c9ac 100644 --- a/python/sglang/srt/arg_groups/nemotron_h_hook.py +++ b/python/sglang/srt/arg_groups/nemotron_h_hook.py @@ -41,9 +41,8 @@ def apply_nemotron_h_defaults(server_args: "ServerArgs", model_arch: str) -> Non server_args._handle_mamba_radix_cache( model_arch=model_arch, - support_mamba_cache=True, - support_mamba_cache_extra_buffer=False, sm100_default_attention_backend="flashinfer", + fallback_attention_backend="flashinfer", ) assert server_args.attention_backend != "triton", ( "NemotronHForCausalLM does not support triton attention backend," diff --git a/python/sglang/srt/batch_overlap/operations.py b/python/sglang/srt/batch_overlap/operations.py index 3d61ac82f500..729e00bcb75d 100644 --- a/python/sglang/srt/batch_overlap/operations.py +++ b/python/sglang/srt/batch_overlap/operations.py @@ -1,16 +1,31 @@ from __future__ import annotations import os -from contextlib import contextmanager -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Sequence, Union +from contextlib import contextmanager, nullcontext +from dataclasses import dataclass, replace +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + List, + Optional, + Sequence, + Union, +) import torch from sglang.srt.layers.dp_attention import set_dp_buffer_len +from sglang.srt.model_executor.forward_context import ( + forward_context, + get_forward_context, +) if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch + from sglang.srt.model_executor.forward_context import ForwardContext _ENABLE_PROFILE = bool(int(os.environ.get("SGLANG_OPERATIONS_ENABLE_PROFILE", "0"))) @@ -39,10 +54,15 @@ def execute_overlapped_operations( assert delta_stage_a == 0 delta_stage = delta_stage_b + # Each TBO child sub-batch dispatches against its own per-child backend + # (children[i] has metadata init'd for sub-batch i; the parent's primary + # has metadata for the full pre-split batch). + child_ctx_a, child_ctx_b = _resolve_tbo_child_contexts() + stages_a = _convert_operations_to_stages(operations_a) stages_b = _convert_operations_to_stages(operations_b) - executor_a = _StageExecutor("a", stages_a, inputs=inputs_a) - executor_b = _StageExecutor("b", stages_b, inputs=inputs_b) + executor_a = _StageExecutor("a", stages_a, inputs=inputs_a, child_ctx=child_ctx_a) + executor_b = _StageExecutor("b", stages_b, inputs=inputs_b, child_ctx=child_ctx_b) for _ in range(delta_stage): executor_a.next() @@ -58,6 +78,25 @@ def execute_overlapped_operations( return [executor_a.output, executor_b.output] +def _resolve_tbo_child_contexts(): + """Return (child_ctx_a, child_ctx_b) derived from the active TboAttnBackend, + or (None, None) if the active backend is not a TBO dispatcher (e.g. a + backend that handles TBO splitting internally like DeepSeek MHA's + _resolve_attn_backend path).""" + # Lazy import to avoid circular dependency at module load time. + from sglang.srt.layers.attention.tbo_backend import TboAttnBackend + + ctx = get_forward_context() + backend = ctx.attn_backend + if not isinstance(backend, TboAttnBackend): + return None, None + child_a, child_b = backend.children + return ( + replace(ctx, attn_backend=child_a), + replace(ctx, attn_backend=child_b), + ) + + class YieldOperation: pass @@ -73,12 +112,23 @@ class ExecutionOperation: class _StageExecutor: - def __init__(self, debug_name: str, stages: List[Stage], inputs: dict): + def __init__( + self, + debug_name: str, + stages: List[Stage], + inputs: dict, + child_ctx: Optional["ForwardContext"] = None, + ): self._debug_name = debug_name self._stages = stages self._index = 0 self._stage_state = _StateDict() self._stage_output = inputs + # When set, every next() runs inside this ForwardContext so that + # get_attn_backend() inside RadixAttention.forward resolves to the + # per-child backend (with sub-batch metadata) instead of the TBO + # parent's primary. + self._child_ctx = child_ctx # handling DP attention forward_batch: ForwardBatch = inputs["forward_batch"] @@ -102,7 +152,12 @@ def next(self): self._global_num_tokens, ) - with _annotate_region(debug_name=f"{self._debug_name}{self._index}"): + ctx_mgr = ( + forward_context(self._child_ctx) + if self._child_ctx is not None + else nullcontext() + ) + with ctx_mgr, _annotate_region(debug_name=f"{self._debug_name}{self._index}"): for op in stage: with _annotate_region(debug_name=op.debug_name): self._stage_output = op.fn( diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index f351851d54a3..de9faca4e3f4 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -14,7 +14,6 @@ ) from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.communicator import ( CommunicateContext, CommunicateSummableTensorPairFn, @@ -40,6 +39,7 @@ ForwardMode, compute_position, ) +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.server_args import get_global_server_args from sglang.srt.speculative.spec_info import SpecInput from sglang.srt.utils import BumpAllocator, empty_context, get_bool_env_var, is_hip @@ -508,8 +508,10 @@ def prepare_raw( f"forward_mode={batch.forward_mode}" ) - assert isinstance(batch.attn_backend, TboAttnBackend) - attn_backend_child_a, attn_backend_child_b = batch.attn_backend.children + # Sanity check: the global attn_backend should be a TboAttnBackend + # whose children handle the two halves. + attn_backend = get_attn_backend() + assert isinstance(attn_backend, TboAttnBackend) [out_num_token_non_padded_a, out_num_token_non_padded_b] = ( tbo_children_num_token_non_padded @@ -525,7 +527,6 @@ def prepare_raw( if is_enable_two_chunk else batch.tbo_split_seq_index ), - output_attn_backend=attn_backend_child_a, out_num_token_non_padded=out_num_token_non_padded_a, ) child_b = cls.filter_batch( @@ -534,7 +535,6 @@ def prepare_raw( end_token_index=batch.input_ids.shape[0], start_seq_index=batch.tbo_split_seq_index, end_seq_index=batch.batch_size, - output_attn_backend=attn_backend_child_b, out_num_token_non_padded=out_num_token_non_padded_b, ) @@ -620,7 +620,6 @@ def filter_batch( end_token_index: int, start_seq_index: int, end_seq_index: int, - output_attn_backend: AttentionBackend, out_num_token_non_padded: torch.Tensor, ): assert ( @@ -692,8 +691,6 @@ def filter_batch( "is_extend_in_batch", "all_extend_in_batch", "return_logprob", - "req_to_token_pool", - "token_to_kv_pool", "can_run_dp_cuda_graph", "dp_padding_mode", "global_forward_mode", @@ -743,7 +740,6 @@ def filter_batch( else None ), extend_num_tokens=extend_num_tokens, - attn_backend=output_attn_backend, num_token_non_padded=out_num_token_non_padded, # TODO: handle it when we need TBO + DeepSeek V3.2 num_token_non_padded_cpu=None, diff --git a/python/sglang/srt/compilation/piecewise_context_manager.py b/python/sglang/srt/compilation/piecewise_context_manager.py index 20a08a9972b9..bbe0d040d705 100644 --- a/python/sglang/srt/compilation/piecewise_context_manager.py +++ b/python/sglang/srt/compilation/piecewise_context_manager.py @@ -71,6 +71,7 @@ def __init__(self): self.quant_config = None self.moe_layers = None self.moe_fusions = None + self.dsa_indexers = None def set_forward_batch(self, forward_batch: ForwardBatch): self.forward_batch = forward_batch @@ -87,6 +88,9 @@ def set_moe_layers(self, layers: List[Any]): def set_moe_fusions(self, fusions: List[Any]): self.moe_fusions = fusions + def set_dsa_indexers(self, indexers: List[Any]): + self.dsa_indexers = indexers + _forward_context: Optional[ForwardContext] = None @@ -104,6 +108,7 @@ def set_forward_context( quant_config: Any, moe_layers: List[Any], moe_fusions: List[Any], + dsa_indexers: Optional[List[Any]] = None, ): global _forward_context _forward_context = ForwardContext() @@ -112,6 +117,8 @@ def set_forward_context( _forward_context.set_quant_config(quant_config) _forward_context.set_moe_layers(moe_layers) _forward_context.set_moe_fusions(moe_fusions) + if dsa_indexers is not None: + _forward_context.set_dsa_indexers(dsa_indexers) try: yield finally: diff --git a/python/sglang/srt/configs/deepseek_ocr.py b/python/sglang/srt/configs/deepseek_ocr.py index b742ff036bc0..9a404177341a 100644 --- a/python/sglang/srt/configs/deepseek_ocr.py +++ b/python/sglang/srt/configs/deepseek_ocr.py @@ -1,9 +1,11 @@ import math from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union import torch from PIL import Image, ImageOps +from torchvision.transforms import InterpolationMode +from torchvision.transforms import functional as TF from transformers import ( AutoProcessor, LlamaTokenizerFast, @@ -18,6 +20,8 @@ DeepseekOCRNoRepeatNGramLogitProcessor, ) +DeepseekOCRImage = Union[Image.Image, torch.Tensor] + BASE_SIZE = 1024 IMAGE_SIZE = 640 CROP_MODE = True @@ -50,6 +54,77 @@ def get_default_ngram_custom_params() -> Dict[str, Any]: PROMPT = "\n<|grounding|>Convert the document to markdown." +def get_image_size(img: DeepseekOCRImage) -> Tuple[int, int]: + """Return (width, height) for both PIL.Image and torch.Tensor (CHW).""" + if isinstance(img, Image.Image): + return img.size + if isinstance(img, torch.Tensor): + if img.ndim != 3: + raise TypeError(f"Expected CHW image tensor, got shape {tuple(img.shape)}") + return int(img.shape[-1]), int(img.shape[-2]) + raise TypeError(f"Unsupported image type: {type(img)}") + + +def resize_image(img: DeepseekOCRImage, size: Tuple[int, int]) -> DeepseekOCRImage: + """Resize image to (width, height) for both PIL and tensor.""" + if isinstance(img, Image.Image): + return img.resize(size, Image.BICUBIC) + return TF.resize( + img, + [size[1], size[0]], + interpolation=InterpolationMode.BICUBIC, + antialias=True, + ).contiguous() + + +def crop_image( + img: DeepseekOCRImage, box: Tuple[int, int, int, int] +) -> DeepseekOCRImage: + """Crop image with box=(left, upper, right, lower) for both PIL and tensor.""" + if isinstance(img, Image.Image): + return img.crop(box) + left, upper, right, lower = box + return img[:, upper:lower, left:right].contiguous() + + +def pad_image( + img: DeepseekOCRImage, + target_size: Tuple[int, int], + fill_color: Tuple[int, int, int], +) -> DeepseekOCRImage: + """Fit-and-center-pad image to target_size=(width, height). + + Replaces ImageOps.pad for tensor inputs. + """ + if isinstance(img, Image.Image): + return ImageOps.pad(img, target_size, color=fill_color) + # tensor path: CHW format + _, h, w = img.shape + target_w, target_h = target_size + scale = min(target_w / w, target_h / h) + new_w = int(w * scale) + new_h = int(h * scale) + resized = TF.resize( + img, + [new_h, new_w], + interpolation=InterpolationMode.BICUBIC, + antialias=True, + ) + pad_left = (target_w - new_w) // 2 + pad_top = (target_h - new_h) // 2 + if img.dtype == torch.uint8: + fill_tensor = torch.tensor( + list(fill_color), device=img.device, dtype=torch.uint8 + ).view(3, 1, 1) + else: + fill_tensor = torch.tensor( + [c / 255.0 for c in fill_color], device=img.device, dtype=img.dtype + ).view(3, 1, 1) + result = fill_tensor.expand(3, target_h, target_w).clone() + result[:, pad_top : pad_top + new_h, pad_left : pad_left + new_w] = resized + return result.contiguous() + + class DictOutput(object): def items(self): return self.__dict__.items() @@ -110,8 +185,20 @@ def __init__( self.transform = T.Compose(transform_pipelines) - def __call__(self, pil_img: Image.Image): - x = self.transform(pil_img) + def __call__(self, img): + if isinstance(img, torch.Tensor): + x = img + if x.dtype == torch.uint8: + x = x.to(torch.float32).div(255) + elif not x.is_floating_point(): + x = x.to(torch.float32) + if self.normalize: + + import torchvision.transforms as T + + x = T.Normalize(self.mean, self.std)(x) + return x + x = self.transform(img) return x @@ -134,7 +221,7 @@ def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_ def dynamic_preprocess( image, min_num=MIN_CROPS, max_num=MAX_CROPS, image_size=640, use_thumbnail=False ): - orig_width, orig_height = image.size + orig_width, orig_height = get_image_size(image) aspect_ratio = orig_width / orig_height # calculate the existing image aspect ratio @@ -158,7 +245,7 @@ def dynamic_preprocess( blocks = target_aspect_ratio[0] * target_aspect_ratio[1] # resize the image - resized_img = image.resize((target_width, target_height)) + resized_img = resize_image(image, (target_width, target_height)) processed_images = [] for i in range(blocks): box = ( @@ -168,11 +255,11 @@ def dynamic_preprocess( ((i // (target_width // image_size)) + 1) * image_size, ) # split the image - split_img = resized_img.crop(box) + split_img = crop_image(resized_img, box) processed_images.append(split_img) assert len(processed_images) == blocks if use_thumbnail and len(processed_images) != 1: - thumbnail_img = image.resize((image_size, image_size)) + thumbnail_img = resize_image(image, (image_size, image_size)) processed_images.append(thumbnail_img) return processed_images, target_aspect_ratio @@ -454,9 +541,10 @@ def tokenize_with_images( tokenized_str += tokenized_sep images_seq_mask += [False] * len(tokenized_sep) - image_shapes.append(image.size) + img_w, img_h = get_image_size(image) + image_shapes.append((img_w, img_h)) - if image.size[0] <= 640 and image.size[1] <= 640: + if img_w <= 640 and img_h <= 640: crop_ratio = [1, 1] else: if cropping: @@ -468,12 +556,12 @@ def tokenize_with_images( """process the global view""" if self.image_size <= 640 and not cropping: - image = image.resize((self.image_size, self.image_size)) + image = resize_image(image, (self.image_size, self.image_size)) - global_view = ImageOps.pad( + global_view = pad_image( image, (self.base_size, self.base_size), - color=tuple(int(x * 255) for x in self.image_transform.mean), + tuple(int(x * 255) for x in self.image_transform.mean), ) images_list.append(self.image_transform(global_view)) diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 098ad1583297..3262e0d47219 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -337,7 +337,6 @@ def __init__( self.use_ngram_embedding = getattr(self.hf_config, "use_ngram_embedding", False) self.is_piecewise_cuda_graph_disabled_model = ( is_piecewise_cuda_graph_disabled_model(self.hf_config.architectures) - or is_deepseek_dsa(self.hf_text_config) ) self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype) @@ -438,11 +437,13 @@ def _config_draft_model(self): self.hf_config.architectures[0] = "DeepseekV4ForCausalLMNextN" self.hf_config.num_nextn_predict_layers = 1 - if is_draft_model and self.hf_config.architectures[0] in [ - "Glm4MoeForCausalLM", - "Glm4MoeLiteForCausalLM", - ]: + if is_draft_model and self.hf_config.architectures[0] == "Glm4MoeForCausalLM": self.hf_config.architectures[0] = "Glm4MoeForCausalLMNextN" + if ( + is_draft_model + and self.hf_config.architectures[0] == "Glm4MoeLiteForCausalLM" + ): + self.hf_config.architectures[0] = "Glm4MoeLiteForCausalLMNextN" if is_draft_model and self.hf_config.architectures[0] in [ "GlmOcrForConditionalGeneration", @@ -627,6 +628,7 @@ def _derive_model_shapes(self): or "DeepseekV3ForCausalLM" in self.hf_config.architectures or "DeepseekV3ForCausalLMNextN" in self.hf_config.architectures or "Glm4MoeLiteForCausalLM" in self.hf_config.architectures + or "Glm4MoeLiteForCausalLMNextN" in self.hf_config.architectures or "GlmMoeDsaForCausalLM" in self.hf_config.architectures or "LongcatFlashForCausalLM" in self.hf_config.architectures or "LongcatFlashForCausalLMNextN" in self.hf_config.architectures @@ -1479,15 +1481,15 @@ def _get_and_verify_dtype( if torch_dtype != config_dtype: if torch_dtype == torch.float32: # Upcasting to float32 is allowed. - logger.info("Upcasting %s to %s.", config_dtype, torch_dtype) + logger.debug("Upcasting %s to %s.", config_dtype, torch_dtype) pass elif config_dtype == torch.float32: # Downcasting from float32 to float16 or bfloat16 is allowed. - logger.info("Downcasting %s to %s.", config_dtype, torch_dtype) + logger.debug("Downcasting %s to %s.", config_dtype, torch_dtype) pass else: # Casting between float16 and bfloat16 is allowed with a warning. - logger.warning("Casting %s to %s.", config_dtype, torch_dtype) + logger.debug("Casting %s to %s.", config_dtype, torch_dtype) return torch_dtype @@ -1584,11 +1586,9 @@ def is_generation_model(model_architectures: List[str], is_embedding: bool = Fal ] piecewise_cuda_graph_disabled_model_archs = [ - "DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM", "DeepseekV4ForCausalLMNextN", "Qwen3NextForCausalLM", - "GlmMoeDsaForCausalLM", "BailingMoeV2_5ForCausalLM", "LLaDAModelLM", ] diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 693e7fffe4c7..94aa7d569ab8 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -25,7 +25,10 @@ KVPoll, KVTransferMetric, ) -from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.disaggregation.utils import ( + DisaggregationMode, + filter_kv_indices_for_cp_rank, +) from sglang.srt.distributed import get_pp_group, get_world_group from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import ( @@ -594,6 +597,92 @@ def _mla_slice_ptrs_for_pp( return src_kv_ptrs, sliced_dst + def _start_heartbeat_checker_thread(self): + """Start the heartbeat checker thread for Decode worker.""" + + def heartbeat_checker(): + while True: + time.sleep(self.heartbeat_interval) + with self.connection_lock: + addresses = list(self.prefill_info_table.keys()) + + for bootstrap_addr in addresses: + session = None + try: + with self.session_pool_lock: + session = self.session_pool[bootstrap_addr] + response = session.get( + f"http://{bootstrap_addr}/health", + timeout=(2, 3), + headers={"Connection": "keep-alive"}, + ) + if response.status_code == 200: + self.heartbeat_failures[bootstrap_addr] = 0 + self._on_heartbeat_success(bootstrap_addr) + else: + logger.info( + f"Attempting to reconnect to {bootstrap_addr}..." + ) + self.heartbeat_failures[bootstrap_addr] = ( + self.heartbeat_failures.get(bootstrap_addr, 0) + 1 + ) + with self.session_pool_lock: + if bootstrap_addr in self.session_pool: + del self.session_pool[bootstrap_addr] + except Exception: + logger.info(f"Attempting to reconnect to {bootstrap_addr}...") + self.heartbeat_failures[bootstrap_addr] = ( + self.heartbeat_failures.get(bootstrap_addr, 0) + 1 + ) + + if ( + self.heartbeat_failures.get(bootstrap_addr, 0) + >= self.max_failures + ): + self._handle_node_failure(bootstrap_addr) + with self.session_pool_lock: + if bootstrap_addr in self.session_pool: + del self.session_pool[bootstrap_addr] + + threading.Thread(target=heartbeat_checker, daemon=True).start() + + def _on_heartbeat_success(self, bootstrap_addr: str): + """Hook called on successful heartbeat. Override for backend-specific cleanup.""" + pass + + def _handle_node_failure(self, failed_bootstrap_addr: str): + """Handle failure of a prefill node.""" + with self.connection_lock: + keys_to_remove = [ + k for k in self.connection_pool if k.startswith(failed_bootstrap_addr) + ] + for k in keys_to_remove: + del self.connection_pool[k] + self.prefill_info_table.pop(failed_bootstrap_addr, None) + + possible_affected_rooms = self.addr_to_rooms_tracker.get( + failed_bootstrap_addr, [] + ) + self.addr_to_rooms_tracker.pop(failed_bootstrap_addr, None) + + affected_rooms = [] + for room in possible_affected_rooms: + if ( + room in self.request_status + and self.check_status(room) != KVPoll.Success + ): + self.record_failure( + room, + f"Lost connection with prefill instance (bootstrap_addr: {failed_bootstrap_addr})", + ) + self.update_status(room, KVPoll.Failed) + affected_rooms.append(room) + + logger.error( + f"Lost connection with prefill instance (bootstrap_addr: {failed_bootstrap_addr}), " + f"{len(affected_rooms)} requests affected" + ) + class CommonKVSender(BaseKVSender): def __init__( @@ -614,6 +703,7 @@ def __init__( self._transfer_num_state_indices = 0 # inner state self.curr_idx = 0 + self.init_time: Optional[float] = None if self.kv_mgr.is_dummy_cp_rank: # Non-authoritative CP ranks are dummy participants. self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput) @@ -667,10 +757,10 @@ def init(self, num_kv_indices: int, aux_index: Optional[int] = None): ) def pop_decode_prefix_len(self) -> int: - return 0 + return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0) def should_send_kv_chunk(self, num_pages: int, last_chunk: bool) -> bool: - return num_pages > 0 + return num_pages > 0 or last_chunk def get_transfer_metric(self) -> KVTransferMetric: total_bytes = self._transfer_num_kv_indices * self.kv_mgr.kv_item_lens_sum @@ -691,6 +781,36 @@ def _record_transfer_indices( if component_indices is not None: self._transfer_num_state_indices += len(component_indices) + def _prepare_send_indices( + self, + kv_indices: npt.NDArray[np.int32], + state_indices: Optional[List] = None, + ) -> Tuple[npt.NDArray[np.int32], slice, bool, bool]: + """Common pre-processing for send(): index tracking and CP-rank handling. + + Returns: + (kv_indices, index_slice, is_last_chunk, should_skip) + If should_skip is True, the caller should return immediately. + """ + index_slice = slice(self.curr_idx, self.curr_idx + len(kv_indices)) + self.curr_idx += len(kv_indices) + is_last_chunk = self.curr_idx == self.num_kv_indices + + if self.kv_mgr.enable_all_cp_ranks_for_transfer: + kv_indices, index_slice = filter_kv_indices_for_cp_rank( + self.kv_mgr, + kv_indices, + index_slice, + ) + elif self.kv_mgr.is_dummy_cp_rank: + if not is_last_chunk: + return kv_indices, index_slice, is_last_chunk, True + else: + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Success) + return kv_indices, index_slice, is_last_chunk, True + + return kv_indices, index_slice, is_last_chunk, False + def send( self, kv_indices: npt.NDArray[np.int32], @@ -698,6 +818,25 @@ def send( ): pass + def _check_bootstrap_timeout(self) -> Optional[KVPoll]: + if self.init_time is None: + return None + elapsed = time.time() - self.init_time + if elapsed < self.kv_mgr.bootstrap_timeout: + return None + logger.warning_once( + "Some requests timed out when bootstrapping, " + "which means prefill instances fail to receive the KV indices from the decode instance of this request. " + "If a greater mean TTFT is acceptable, you can 'export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600' (10 minutes) to relax the timeout condition. " + ) + self.kv_mgr.record_failure( + self.bootstrap_room, + f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s " + f"in KVPoll.Bootstrapping", + ) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + return KVPoll.Failed + def poll(self) -> KVPoll: pass @@ -737,6 +876,7 @@ def __init__( self.kv_mgr = mgr self.conclude_state: Optional[KVPoll] = None self.require_staging: bool = False + self.init_time: Optional[float] = None self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room) self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) @@ -906,6 +1046,24 @@ def send_metadata( ): raise NotImplementedError + def _check_waiting_timeout(self) -> Optional[KVPoll]: + if self.init_time is None: + return None + elapsed = time.time() - self.init_time + if elapsed < self.kv_mgr.waiting_timeout: + return None + logger.warning_once( + "Some requests fail to receive KV Cache transfer done signal after bootstrapping. " + "If a greater mean TTFT is acceptable, you can 'export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600' (10 minutes) to relax the timeout condition. " + ) + self.kv_mgr.record_failure( + self.bootstrap_room, + f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s " + f"in KVPoll.WaitingForInput", + ) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + return KVPoll.Failed + def failure_exception(self): raise Exception("Fake KVReceiver Exception") diff --git a/python/sglang/srt/disaggregation/common/utils.py b/python/sglang/srt/disaggregation/common/utils.py index 4e5e96c6f205..1084b753604a 100644 --- a/python/sglang/srt/disaggregation/common/utils.py +++ b/python/sglang/srt/disaggregation/common/utils.py @@ -1,12 +1,27 @@ +import ctypes +import dataclasses import struct import threading from collections import deque -from typing import List, Tuple +from typing import List, Optional, Tuple import numpy as np import numpy.typing as npt +@dataclasses.dataclass +class TransferKVChunk: + """Work unit for KV cache transfer from prefill to decode.""" + + room: int + prefill_kv_indices: npt.NDArray[np.int32] + index_slice: slice + is_last_chunk: bool + prefill_aux_index: Optional[int] + state_indices: Optional[List] + chunk_id: Optional[int] = None + + def pack_list_of_buffers(buffers: List[bytes]) -> bytes: if not buffers: return b"" @@ -59,6 +74,26 @@ def get(self): return self._buf.popleft() +class AuxDataCodec: + """Handles serialization and deserialization of auxiliary data buffers.""" + + @staticmethod + def serialize_data_from_buffer(src_addr, data_length): + """Serialize data from memory buffer to bytes.""" + buffer = (ctypes.c_byte * data_length).from_address(src_addr) + return bytes(buffer) + + @staticmethod + def deserialize_data_to_buffer(kv_args, buffer_index, aux_index, data): + """Deserialize bytes into target memory buffer.""" + dst_aux_ptr = kv_args.aux_data_ptrs[buffer_index] + item_len = kv_args.aux_item_lens[buffer_index] + dst_addr = dst_aux_ptr + item_len * aux_index + buffer = (ctypes.c_byte * len(data)).from_address(dst_addr) + buffer[:] = data + return + + def group_concurrent_contiguous( src_indices: npt.NDArray[np.int32], dst_indices: npt.NDArray[np.int32] ) -> Tuple[List[npt.NDArray[np.int32]], List[npt.NDArray[np.int32]]]: diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 65f2e4cb8c9c..93208639fb44 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -37,12 +37,12 @@ from sglang.srt.disaggregation.base.conn import StateType from sglang.srt.disaggregation.common.conn import CommonKVManager, CommonKVReceiver from sglang.srt.disaggregation.utils import ( - FAKE_BOOTSTRAP_HOST, DisaggregationMode, KVClassType, MetadataBuffers, ReqToMetadataIdxAllocator, TransferBackend, + _is_fake_transfer, get_kv_class, is_mla_backend, poll_and_all_reduce, @@ -82,18 +82,10 @@ if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req from sglang.srt.managers.scheduler import Scheduler - from sglang.srt.server_args import ServerArgs CLIP_MAX_NEW_TOKEN = envs.SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION.get() -def _is_fake_transfer(req: Req, server_args: ServerArgs) -> bool: - return req.bootstrap_host == FAKE_BOOTSTRAP_HOST or ( - req.bootstrap_host is None - and server_args.disaggregation_transfer_backend == "fake" - ) - - def _bootstrap_addr(req: Req) -> str: # FIXME: make a property of a req return NetworkAddress(req.bootstrap_host, req.bootstrap_port).to_host_port_str() @@ -1378,12 +1370,7 @@ def extend(self, decode_reqs: List[DecodeRequest]) -> None: ): self.staging_handler.register_decode_req(dr.req.bootstrap_room, dr) - def _commit_transfer_to_req(self, decode_req: DecodeRequest) -> bool: - """ - Returns: - True if the request should be removed from the queue (success or corruption) - False if metadata not ready yet (keep in queue for next poll) - """ + def _commit_transfer_to_req(self, decode_req: DecodeRequest): idx = decode_req.metadata_buffer_index ( output_id, @@ -1409,11 +1396,25 @@ def _commit_transfer_to_req(self, decode_req: DecodeRequest) -> bool: if _is_fake_transfer(decode_req.req, self.scheduler.server_args): pass elif actual_room == 0: - # Case 1: Metadata not ready yet (actual_room == 0) - # Keep request in queue and wait for next poll - return False + # Should never happen: _poll_with_metadata_gate already confirmed + # readiness on all TP ranks. Abort deterministically to avoid + # cross-rank queue divergence. + logger.error( + f"Metadata unexpectedly not ready after readiness gate: " + f"request {decode_req.req.rid}, bootstrap_room={expected_room}, " + f"metadata_buffer_index={idx}" + ) + prepare_abort( + decode_req.req, + "Metadata unexpectedly not ready after readiness gate " + "(bootstrap_room=0)", + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + ) + decode_req.kv_receiver.clear() + decode_req.kv_receiver = None + return elif actual_room != expected_room: - # Case 2: Real corruption detected (mismatch) + # Real corruption detected (mismatch) # Abort the request and remove from the queue error_msg = ( f"Context corruption detected: Request {decode_req.req.rid} " @@ -1430,9 +1431,9 @@ def _commit_transfer_to_req(self, decode_req: DecodeRequest) -> bool: ) decode_req.kv_receiver.clear() decode_req.kv_receiver = None - return True + return - # Case 3: Success - commit the transfer + # Success - commit the transfer decode_req.req.output_ids.append(output_id[0].item()) decode_req.req.cached_tokens = cached_tokens[0].item() decode_req.req.cached_tokens_device = cached_tokens[1].item() @@ -1464,11 +1465,24 @@ def _commit_transfer_to_req(self, decode_req: DecodeRequest) -> bool: decode_req.kv_receiver.clear() decode_req.kv_receiver = None decode_req.req.time_stats.set_wait_queue_entry_time() - return True + return + + def _poll_with_metadata_gate(self) -> List[int]: + return poll_and_all_reduce( + [dr.kv_receiver for dr in self.queue], + self.gloo_group, + decode_reqs=self.queue, + metadata_buffers=self.metadata_buffers, + server_args=self.scheduler.server_args, + ) def _poll_with_staging(self) -> list: return poll_and_all_reduce_with_staging( - self.queue, self.staging_handler, self.gloo_group + self.queue, + self.staging_handler, + self.gloo_group, + metadata_buffers=self.metadata_buffers, + server_args=self.scheduler.server_args, ) def _init_staging_handler(self, kv_manager): @@ -1489,9 +1503,7 @@ def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req if self.enable_staging: polls = self._poll_with_staging() else: - polls = poll_and_all_reduce( - [dr.kv_receiver for dr in self.queue], self.gloo_group - ) + polls = self._poll_with_metadata_gate() transferred_reqs = [] indices_to_remove = set() @@ -1524,26 +1536,23 @@ def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req self.scheduler.metrics_collector.increment_transfer_failed_reqs() continue elif poll == KVPoll.Success: - should_remove = self._commit_transfer_to_req(decode_req) - if should_remove: - indices_to_remove.add(i) - # Check if request was aborted due to corruption - if isinstance(decode_req.req.finished_reason, FINISH_ABORT): - self.scheduler.output_streamer.stream_output( - [decode_req.req], - decode_req.req.return_logprob, - ) - if self.scheduler.enable_hisparse: - self.scheduler.hisparse_coordinator.request_finished( - decode_req.req - ) - release_kv_cache( - decode_req.req, self.tree_cache, is_insert=False + self._commit_transfer_to_req(decode_req) + indices_to_remove.add(i) + # Check if request was aborted due to corruption + if isinstance(decode_req.req.finished_reason, FINISH_ABORT): + self.scheduler.output_streamer.stream_output( + [decode_req.req], + decode_req.req.return_logprob, + ) + if self.scheduler.enable_hisparse: + self.scheduler.hisparse_coordinator.request_finished( + decode_req.req ) - if self.scheduler.metrics_reporter.enable_metrics: - self.scheduler.metrics_collector.increment_transfer_failed_reqs() - else: - transferred_reqs.append(decode_req.req) + release_kv_cache(decode_req.req, self.tree_cache, is_insert=False) + if self.scheduler.metrics_reporter.enable_metrics: + self.scheduler.metrics_collector.increment_transfer_failed_reqs() + else: + transferred_reqs.append(decode_req.req) elif poll in [ KVPoll.Bootstrapping, KVPoll.WaitingForInput, diff --git a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py index 76cc76e9354a..6f52307b2e3a 100644 --- a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py +++ b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from array import array from http import HTTPStatus from typing import TYPE_CHECKING, List @@ -71,7 +72,7 @@ def prepare_for_prebuilt(self: ScheduleBatch): # Set fields self.input_ids = torch.tensor( - sum(input_ids, []), dtype=torch.int32, device=self.device + sum(input_ids, array("q")), dtype=torch.int32, device=self.device ) self.req_pool_indices = torch.tensor( req_pool_indices, dtype=torch.int64, device=self.device @@ -173,16 +174,15 @@ def process_prebuilt( topk_index=topk_index, hidden_states=hidden_states, bonus_tokens=last_tokens_tensor, - new_seq_lens=self.seq_lens, ) spec_info.capture_hidden_mode = CaptureHiddenMode.LAST if self.enable_overlap: - from sglang.srt.managers.overlap_utils import FutureIndices - - spec_info.future_indices = FutureIndices(indices=self.req_pool_indices) - future_map.publish(spec_info.future_indices, spec_info.new_seq_lens) + spec_info.future_indices = self.req_pool_indices + future_map.publish(spec_info.future_indices, self.seq_lens) future_map.stash(spec_info.future_indices, spec_info) self.spec_info = spec_info else: - # Non-spec: input_ids feeds the next decode forward directly. + # Non-spec: positive last token feeds decode directly. No FutureMap + # bootstrap needed (SB self-maintains seq_lens; resolve_future is + # a no-op on positive input_ids). self.input_ids = last_tokens_tensor diff --git a/python/sglang/srt/disaggregation/encode_receiver.py b/python/sglang/srt/disaggregation/encode_receiver.py index f5a5b5724a35..72f3982cfa48 100644 --- a/python/sglang/srt/disaggregation/encode_receiver.py +++ b/python/sglang/srt/disaggregation/encode_receiver.py @@ -7,6 +7,7 @@ import time import uuid from abc import ABC, abstractmethod +from array import array from collections import OrderedDict, defaultdict from enum import IntEnum from http import HTTPStatus @@ -440,6 +441,7 @@ def __init__( recv_req: TokenizedGenerateReqInput, mm_processor, encoder_urls, + model_type, host_name, receive_count, ): @@ -450,6 +452,7 @@ def __init__( self.thread = None self.mm_processor = mm_processor self.encoder_urls = encoder_urls + self.model_type = model_type self.host_name = host_name self.receive_count = receive_count self.num_items_assigned = recv_req.num_items_assigned @@ -588,7 +591,7 @@ def _try_recv_mm_data(self): **self.recv_embedding_data.get_mm_extra_meta(), ) self.recv_req.mm_inputs = mm_inputs - self.recv_req.input_ids = mm_inputs.input_ids + self.recv_req.input_ids = array("q", mm_inputs.input_ids) self.status = WaitingImageRequestStatus.SUCCESS self.recv_socket.close() @@ -925,6 +928,7 @@ def _process_waiting_requests(self, recv_reqs, waiting_cls): recv_req=recv_req, mm_processor=self.mm_processor, encoder_urls=self.encode_urls, + model_type=self.model_type, host_name=self.hostname, receive_count=self.tp_size, ) diff --git a/python/sglang/srt/disaggregation/encode_server.py b/python/sglang/srt/disaggregation/encode_server.py index ff13cbb1d802..3bd925a9c5ae 100644 --- a/python/sglang/srt/disaggregation/encode_server.py +++ b/python/sglang/srt/disaggregation/encode_server.py @@ -1,5 +1,6 @@ import asyncio import concurrent.futures +import contextlib import ctypes import logging import multiprocessing as mp @@ -7,6 +8,7 @@ import pickle import time import traceback +from collections import defaultdict from http import HTTPStatus from typing import Dict, List, Optional, Set, Tuple, Union @@ -78,9 +80,12 @@ cond_dict_lock = asyncio.Lock() rid_to_cond: Dict[str, asyncio.Condition] = {} -use_image_processor_gpu = ( - int(os.getenv("SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU", "0")) == 1 -) +use_image_processor_gpu = envs.SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU.get() + +ENCODER_MAX_BATCH_SIZE = envs.SGLANG_ENCODER_MAX_BATCH_SIZE.get() +# Watchdog: max time to wait for a batched /encode result. Bounds HTTP latency +# if the batch worker stalls (NCCL hang, dead worker proc, etc.). +ENCODER_REQ_TIMEOUT = envs.SGLANG_ENCODER_REQ_TIMEOUT.get() class MMError(Exception): @@ -223,6 +228,8 @@ def __init__( use_image_processor_gpu and not server_args.disable_fast_image_processor ) self._build_vision_config(server_args.mm_process_config) + self.model_audio_sr = self._resolve_audio_sr() + logger.info(f"Resolved model audio sample rate: {self.model_audio_sr} Hz") init_distributed_environment( backend=get_default_distributed_backend(self.device), @@ -341,6 +348,44 @@ def _infer_embedding_dims(self) -> dict: logger.info(f"Global cache embedding dims: {dims}") return dims + def _resolve_audio_sr(self) -> int: + # Must match MiMoProcessor.from_hf_config — on drift, mimo tags the + # ndarray with its own audio_sampling_rate and skips resample, so the + # waveform is interpreted at the wrong rate and warped. + def _read(obj, attr): + if obj is None: + return None + if isinstance(obj, dict): + return obj.get(attr) + return getattr(obj, attr, None) + + audio_cfg = self.vision_config.get("audio", {}) + sr = audio_cfg.get("audio_sampling_rate") + if sr: + return int(sr) + + hf_cfg = self.model_config.hf_config + thinker_cfg = _read(hf_cfg, "thinker_config") + pc = _read(thinker_cfg, "processor_config") or _read(hf_cfg, "processor_config") + sr = _read(pc, "audio_sampling_rate") + if sr: + return int(sr) + ac = _read(thinker_cfg, "audio_config") or _read(hf_cfg, "audio_config") + for attr in ("sampling_rate", "sample_rate"): + sr = _read(ac, attr) + if sr: + return int(sr) + + sr = audio_cfg.get("sampling_rate") + if sr: + return int(sr) + logger.warning( + "No audio sampling rate found in mm_config or hf_config; " + "falling back to 16000 Hz. If the model expects a different SR " + "(e.g. MiMo-V2 defaults to 24000), audio will be warped." + ) + return 16000 + def _build_vision_config(self, mm_process_config): """ Validate vision config, used for image/video/audio. @@ -440,7 +485,6 @@ def _load_single_item( data, modality: Modality, frame_count_limit=None, - audio_sample_rate: Optional[int] = None, discard_alpha_channel=True, ): """ @@ -463,7 +507,7 @@ def _load_single_item( elif modality == Modality.VIDEO: return load_video(data, frame_count_limit) elif modality == Modality.AUDIO: - return load_audio(data, audio_sample_rate) + return load_audio(data, self.model_audio_sr) except Exception as e: raise RuntimeError(f"Error while loading data {data}: {e}") @@ -500,6 +544,11 @@ def _get_feat_extract_output_lengths(self, feature_lens): ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (feature_lens // 100) * 13 ) return output_lengths + elif self.model_type == "mimo_v2": + # MiMo-V2's preprocess_audio returns audio_token_len (already + # post-encoder/avg-pooler/group-size). Stored in audio_feature_lens_raw, + # so no further reduction here. + return feature_lens else: # fallback to original HF audio sample logic for other models logger.warning( @@ -631,10 +680,18 @@ def slice_embedding( return slices def _calculate_hashes_from_features( - self, mm_feature: torch.Tensor, grid_thw: List, modality: Modality + self, mm_feature, grid_thw: List, modality: Modality ) -> List[str]: """CPU Task: Compute hashes based on processed feature patches.""" - hashes, offset = [], 0 + hashes = [] + if modality == Modality.AUDIO and isinstance(mm_feature, list): + for feature in mm_feature: + tmp_item = MultimodalDataItem(modality=modality, feature=feature) + tmp_item.set_pad_value() + hashes.append(tmp_item.hash) + return hashes + + offset = 0 logger.info(f"{mm_feature.shape=} with {modality=}") for grid in grid_thw: num_patches = self.get_num_patches(grid, modality) @@ -647,7 +704,7 @@ def _calculate_hashes_from_features( async def _encode_missing( self, - mm_feature: torch.Tensor, + mm_feature, mm_inputs: dict, indices: List[int], modality: Modality = Modality.IMAGE, @@ -658,23 +715,34 @@ async def _encode_missing( """ grid_thw = _get_mm_grid_dim(mm_inputs, modality, self.model_type) - # 1. Slice mm_feature to get only the patches for missing mm items - sub_feature_list = [] - offsets = [0] - curr = 0 - for g in grid_thw: - curr += self.get_num_patches(g, modality) - offsets.append(curr) - - for idx in indices: - sub_feature_list.append(mm_feature[offsets[idx] : offsets[idx + 1]]) - - sub_feature = torch.cat(sub_feature_list, dim=0) + # Audio features are per-item (list of mels for mimo_v2, or batched + # N x n_mels x T_max for qwen2_audio); slice by item index and keep + # per-item shape. Image/video features are concatenated along the + # patch dim; slice by cumulative patch offsets and cat. + if modality == Modality.AUDIO: + if isinstance(mm_feature, list): + sub_feature = [mm_feature[i] for i in indices] + else: + sub_feature = mm_feature[list(indices)] + else: + sub_feature_list = [] + offsets = [0] + curr = 0 + for g in grid_thw: + curr += self.get_num_patches(g, modality) + offsets.append(curr) + for idx in indices: + sub_feature_list.append(mm_feature[offsets[idx] : offsets[idx + 1]]) + sub_feature = torch.cat(sub_feature_list, dim=0) mm_item = MultimodalDataItem.from_dict( { "modality": modality, - "feature": _convert(sub_feature), + "feature": ( + sub_feature + if isinstance(sub_feature, list) + else _convert(sub_feature) + ), } ) @@ -710,6 +778,15 @@ async def encode_with_global_cache( mm_feature = _convert(_get_mm_feature(mm_inputs, modality)) num_items = len(grid_thw) + # Hashes must be grid-space; a leaf-space list would size-mismatch + # rank>0's mask (zeros(num_items)) and deadlock TP. + if hashes is not None and len(hashes) != num_items: + raise BadRequestError( + f"User-supplied hashes length {len(hashes)} != grid count " + f"{num_items} for {self.model_type}/{modality.name}; hashes " + f"must be in grid space (1 per encoder grid entry)." + ) + # Step 1: Rank 0 checks global cache and broadcasts hit/miss mask to all ranks. if self.rank == 0: if hashes is None: @@ -852,7 +929,8 @@ async def _background_insert(): async def _flatten_and_load_audios(self, mm_items): """ - Flatten mm_items structure, load audios concurrently, and restore original structure. + Flatten mm_items, load audios concurrently as np.ndarray at + self.model_audio_sr, restore original structure. """ return await self._flatten_and_load_data_by_modality(mm_items, Modality.AUDIO) @@ -893,6 +971,28 @@ def _flatten_nested_items(items): flat.append(item) return flat + def _grid_count_per_leaf(self, leaves: List, modality: Modality) -> List[int]: + """Number of grid entries each leaf produces under the model's processor. + + Most processors map 1 leaf → 1 grid. Kimi-VL/K25 image processors expand + a leaf shaped {"type": "image", "image": [pil1, pil2, ...]} into N grids + (see _normalize_kimi_encoder_images). Cross-request batching needs these + counts to keep per-request boundaries aligned with grid_dim. + """ + if self.model_type not in ("kimi_k25", "kimi_vl") or modality != Modality.IMAGE: + return [1] * len(leaves) + + def count(leaf): + if ( + isinstance(leaf, dict) + and leaf.get("type") == "image" + and isinstance(leaf.get("image"), (list, tuple)) + ): + return len(self._flatten_nested_items(leaf["image"])) + return 1 + + return [count(leaf) for leaf in leaves] + def _normalize_kimi_encoder_images(self, images): """Normalize Kimi image inputs for the image processor call.""" from PIL import Image as PILImage @@ -1039,18 +1139,21 @@ async def _process_video_items(self, mm_items, model_preprocessor): return processor_input async def _process_audio_items(self, mm_items, model_preprocessor): + # Await off the event loop so EncoderScheduler can accumulate + # cross-request batches during download. + audios = await self._flatten_and_load_audios(mm_items) + if model_preprocessor: - return model_preprocessor(mm_items, Modality.AUDIO, self.vision_config) + return model_preprocessor(audios, Modality.AUDIO, self.vision_config) + if not self.audio_processor: raise ValueError("No audio processor available") - audios = await self._flatten_and_load_audios(mm_items) audio_config = self.vision_config.get("audio", {}) processor_input = self.audio_processor.feature_extractor(audios, **audio_config) processor_input["feature_attention_mask"] = processor_input.pop( "attention_mask" ) - # convert to same format as image/video input_lengths = torch.tensor( processor_input["feature_attention_mask"].sum(-1), dtype=torch.long ) @@ -1225,6 +1328,122 @@ async def encode(self, mm_items, modality: Modality, req_id, num_parts, part_idx logger.debug(f"Created error EmbeddingData: {mm_data}") return 0, 0, 0, error_msg, error_code + async def encode_request(self, req: dict, modality: Modality): + """Single-request encode dispatcher: picks cache vs no-cache path.""" + if self.mm_global_cache is not None: + return await self.encode_with_global_cache( + mm_items=req["mm_items"], + modality=modality, + req_id=req["req_id"], + num_parts=req["num_parts"], + part_idx=req["part_idx"], + hashes=req.get("hashes"), + ) + return await self.encode( + mm_items=req["mm_items"], + modality=modality, + req_id=req["req_id"], + num_parts=req["num_parts"], + part_idx=req["part_idx"], + ) + + async def batch_encode( + self, requests: List[dict], modality: Modality + ) -> List[Tuple[int, int, int, Optional[str], Optional[int]]]: + """Cross-request encoder fusion (image/audio). No cache path.""" + # items_per_req counts grid entries (post-expansion) so per-request + # slicing of grid_dim/final_slices stays aligned for processors that + # expand one leaf into multiple grids (e.g. Kimi-VL/K25 dict-of-images). + flat_items, items_per_req = [], [] + for req in requests: + leaves = MMEncoder._flatten_nested_items(req["mm_items"]) + flat_items.extend(leaves) + items_per_req.append(sum(self._grid_count_per_leaf(leaves, modality))) + total = sum(items_per_req) + + try: + mm_inputs, get_feat = await self._process_mm_items(flat_items, modality) + except NotImplementedError as e: + return self._batch_set_error( + requests, modality, InternalError(f"Not implemented error: {e}") + ) + except Exception as e: + return self._batch_set_error( + requests, modality, BadRequestError(f"Failed to process mm items: {e}") + ) + + try: + mm_feature = _convert(_get_mm_feature(mm_inputs, modality)) + grid_dim = _get_mm_grid_dim(mm_inputs, modality, self.model_type) + if len(grid_dim) != total: + return self._batch_set_error( + requests, + modality, + InternalError( + f"Grid count mismatch for {self.model_type}/" + f"{modality.name}: {len(flat_items)} leaves across " + f"{len(requests)} requests → expected {total} grids " + f"(per-req {items_per_req}), but processor produced " + f"{len(grid_dim)}. Add tile-expansion handling in " + f"_grid_count_per_leaf." + ), + ) + + final_slices = await self._encode_missing( + mm_feature, + mm_inputs, + list(range(total)), + modality, + get_feat, + ) + + if self.profiler is not None: + for _ in requests: + self.profiler.step() + # No aux_data here: batch_encode only handles IMAGE/AUDIO + # (_BATCHABLE_MODALITIES), and _build_mm_aux_data only extracts + # video-meta fields — which never appear in image/audio mm_inputs. + results = [] + offset = 0 + for req, n in zip(requests, items_per_req): + slices = final_slices[offset : offset + n] + emb = slices[0] if n == 1 else torch.cat(slices, dim=0) + if self.rank == 0: + self.embedding_to_send[req["req_id"]] = EmbeddingData( + req["req_id"], + req["num_parts"], + req["part_idx"], + grid_dim[offset : offset + n], + modality, + emb, + ) + results.append((emb.nbytes, emb.shape[0], emb.shape[1], None, None)) + offset += n + return results + except Exception as e: + return self._batch_set_error( + requests, modality, InternalError(f"Internal encoding error: {e}") + ) + + def _batch_set_error( + self, requests: List[dict], modality: Modality, exc: Exception + ) -> List[Tuple[int, int, int, str, int]]: + code = getattr(exc, "code", HTTPStatus.INTERNAL_SERVER_ERROR) + msg = str(exc) + logger.error(f"Rank {self.rank} batch_encode failed: {msg} {code = }") + if self.rank == 0: + for req in requests: + self.embedding_to_send[req["req_id"]] = EmbeddingData( + req["req_id"], + req["num_parts"], + req["part_idx"], + None, + modality, + error_msg=msg, + error_code=code, + ) + return [(0, 0, 0, msg, code)] * len(requests) + # For zmq_to_tokenizer zmq_to_scheduler and mooncake async def send( self, req_id, prefill_host, embedding_port, session_id=None, buffer_address=None @@ -1396,9 +1615,239 @@ def stop(self): return True, None -app = FastAPI() +class PendingRequest: + __slots__ = ("request", "future", "submit_time") + + def __init__(self, request: dict, loop: asyncio.AbstractEventLoop): + self.request = request + self.future: asyncio.Future = loop.create_future() + self.submit_time = time.time() + + +# VIDEO excluded: per-video preprocess kwargs (do_sample_frames, video_metadata) +# vary per request and can't merge into one HF processor call. +_BATCHABLE_MODALITIES = {Modality.IMAGE, Modality.AUDIO} + + +class EncoderScheduler: + """Aggregate concurrent /encode requests into bounded image/audio batches.""" + + def __init__( + self, + encoder: "MMEncoder", + send_sockets: List[zmq.Socket], + max_batch_size: int, + request_timeout: float = ENCODER_REQ_TIMEOUT, + ): + self.encoder = encoder + self.send_sockets = send_sockets + self.max_batch_size = max(1, int(max_batch_size)) + self.request_timeout = max(1.0, float(request_timeout)) + self.pending_queue: "asyncio.Queue[PendingRequest]" = asyncio.Queue() + self._worker_task: Optional[asyncio.Task] = None + + def start(self) -> None: + if self._worker_task is None: + self._worker_task = asyncio.create_task(self._batch_worker()) + logger.info( + f"EncoderScheduler started with max_batch_size={self.max_batch_size}" + ) + + async def stop(self) -> None: + if self._worker_task is not None: + self._worker_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._worker_task + self._worker_task = None + # Reject any requests still queued so their HTTP handlers don't hang. + while True: + try: + pending = self.pending_queue.get_nowait() + except asyncio.QueueEmpty: + break + if not pending.future.done(): + pending.future.set_exception(RuntimeError("EncoderScheduler stopped")) + + async def submit(self, request: dict) -> Tuple: + pending = PendingRequest(request, asyncio.get_running_loop()) + await self.pending_queue.put(pending) + try: + return await asyncio.wait_for(pending.future, timeout=self.request_timeout) + except asyncio.TimeoutError: + if not pending.future.done(): + pending.future.cancel() + req_id = request.get("req_id") + logger.error( + f"EncoderScheduler.submit timed out after {self.request_timeout}s " + f"for req_id={req_id}" + ) + raise + + async def _collect_batch(self) -> List[PendingRequest]: + batch = [await self.pending_queue.get()] + while len(batch) < self.max_batch_size: + try: + batch.append(self.pending_queue.get_nowait()) + except asyncio.QueueEmpty: + break + return batch + + async def _batch_worker(self) -> None: + while True: + batch: List[PendingRequest] = [] + try: + batch = await self._collect_batch() + groups: Dict[Modality, List[PendingRequest]] = defaultdict(list) + for p in batch: + groups[ + Modality.from_str(p.request.get("modality", "image")) + ].append(p) + for modality, group in groups.items(): + await self._dispatch_group(group, modality) + except asyncio.CancelledError: + for p in batch: + if not p.future.done(): + p.future.set_exception(RuntimeError("EncoderScheduler stopped")) + raise + except Exception as e: + logger.error( + f"Error in EncoderScheduler batch worker: {e}", exc_info=True + ) + for p in batch: + if not p.future.done(): + p.future.set_exception(e) + + @staticmethod + def _validate_request_shape(req: dict) -> Optional[str]: + # Cheap pre-broadcast checks: shape errors that don't require running + # the HF processor. Once a request reaches TP workers they enter + # batch_encode and expect to join its collectives — a malformed batch + # that makes rank-0 bail mid-flight would deadlock the workers. + if not isinstance(req, dict): + return f"request is not a dict: {type(req).__name__}" + if not req.get("req_id"): + return "missing req_id" + if not req.get("mm_items"): + return "missing or empty mm_items" + if "num_parts" not in req or "part_idx" not in req: + return "missing num_parts / part_idx" + h = req.get("hashes") + if h is not None and not isinstance(h, (list, tuple, str, int, bytes)): + return f"hashes must be list/scalar, got {type(h).__name__}" + return None + + async def _dispatch_group( + self, group: List[PendingRequest], modality: Modality + ) -> None: + # Video can't fuse (per-video preprocess kwargs vary). + if modality not in _BATCHABLE_MODALITIES: + await self._dispatch_per_request(group, modality) + return + + # Drop structurally-bad requests before broadcasting; otherwise TP + # workers would join batch_encode collectives that rank-0 has already + # abandoned. + valid: List[PendingRequest] = [] + for p in group: + err = self._validate_request_shape(p.request) + if err is None: + valid.append(p) + continue + logger.error(f"Dropping req_id={p.request.get('req_id')} from batch: {err}") + if not p.future.done(): + p.future.set_exception(BadRequestError(err)) + if not valid: + return + group = valid + + requests = [p.request for p in group] + start = time.time() + for sock in self.send_sockets: + sock.send_pyobj( + { + "type": "batch_encode", + "modality": modality.name, + "requests": requests, + "enter_time": start, + } + ) + + logger.info(f"Dispatching batch of {len(group)} {modality.name} requests") + + try: + results = await self.encoder.batch_encode(requests, modality) + if len(group) > 1: + logger.info( + f"Batch of {len(group)} {modality.name} requests completed in " + f"{(time.time() - start) * 1000:.1f}ms" + ) + except Exception as e: + # batch_encode normally catches and returns errors via _batch_set_error. + # If it raised, rank-0 may have skipped a collective broadcast, leaving + # TP workers stuck. Don't try to recover — fail every pending future + # and let the client retry. Re-broadcasting would risk a deadlock. + logger.error(f"batch_encode raised: {e}", exc_info=True) + for p in group: + if not p.future.done(): + p.future.set_exception(e) + return + + if len(results) != len(group): + err = RuntimeError( + f"batch_encode returned {len(results)} results for {len(group)} requests" + ) + logger.error(str(err)) + for p in group: + if not p.future.done(): + p.future.set_exception(err) + return + + for p, result in zip(group, results): + if not p.future.done(): + p.future.set_result(result) + + async def _dispatch_per_request( + self, + group: List[PendingRequest], + modality: Modality, + ) -> None: + for p in group: + req = p.request + try: + for sock in self.send_sockets: + sock.send_pyobj(req) + result = await self.encoder.encode_request(req, modality) + if not p.future.done(): + p.future.set_result(result) + except Exception as e: + logger.error( + f"Per-request encode failed for req_id={req.get('req_id')}: {e}" + ) + if not p.future.done(): + p.future.set_exception(e) + + encoder: Optional[MMEncoder] = None send_sockets: List[zmq.Socket] = [] +encoder_scheduler: Optional[EncoderScheduler] = None + + +@contextlib.asynccontextmanager +async def _lifespan(app: FastAPI): + global encoder_scheduler + if encoder is not None: + encoder_scheduler = EncoderScheduler( + encoder, send_sockets, max_batch_size=ENCODER_MAX_BATCH_SIZE + ) + encoder_scheduler.start() + try: + yield + finally: + if encoder_scheduler is not None: + await encoder_scheduler.stop() + + +app = FastAPI(lifespan=_lifespan) async def run_encoder( @@ -1414,24 +1863,15 @@ async def run_encoder( encoder.profiler.start(request) else: encoder.profiler.stop() + elif isinstance(request, dict) and request.get("type") == "batch_encode": + await encoder.batch_encode( + request["requests"], + Modality.from_str(request["modality"]), + ) else: - if encoder.mm_global_cache is not None: - await encoder.encode_with_global_cache( - mm_items=request["mm_items"], - modality=Modality.from_str(request["modality"]), - req_id=request["req_id"], - num_parts=request["num_parts"], - part_idx=request["part_idx"], - hashes=request.get("hashes", None), - ) - else: - await encoder.encode( - mm_items=request["mm_items"], - modality=Modality.from_str(request["modality"]), - req_id=request["req_id"], - num_parts=request["num_parts"], - part_idx=request["part_idx"], - ) + await encoder.encode_request( + request, Modality.from_str(request["modality"]) + ) def launch_encoder(server_args, schedule_path, dist_init_method, rank): @@ -1489,30 +1929,27 @@ def start_background_send(req_id): encoder.background_tasks.add(task) task.add_done_callback(encoder.background_tasks.discard) - # broadcast request request.update({"enter_time": time.time()}) - for socket in send_sockets: - socket.send_pyobj(request) - if encoder.mm_global_cache is not None: - nbytes, embedding_len, embedding_dim, error_msg, error_code = ( - await encoder.encode_with_global_cache( - mm_items=request["mm_items"], - modality=Modality.from_str(request["modality"]), - req_id=request["req_id"], - num_parts=request["num_parts"], - part_idx=request["part_idx"], - hashes=request.get("hashes", None), + modality = Modality.from_str(request["modality"]) + if encoder_scheduler is not None and modality in _BATCHABLE_MODALITIES: + try: + nbytes, embedding_len, embedding_dim, error_msg, error_code = ( + await encoder_scheduler.submit(request) + ) + except asyncio.TimeoutError: + return ORJSONResponse( + status_code=HTTPStatus.GATEWAY_TIMEOUT, + content={ + "status": "error", + "message": "encoder batch timed out", + "req_id": req_id, + }, ) - ) else: + for socket in send_sockets: + socket.send_pyobj(request) nbytes, embedding_len, embedding_dim, error_msg, error_code = ( - await encoder.encode( - mm_items=request["mm_items"], - modality=Modality.from_str(request["modality"]), - req_id=request["req_id"], - num_parts=request["num_parts"], - part_idx=request["part_idx"], - ) + await encoder.encode_request(request, modality) ) if error_msg: diff --git a/python/sglang/srt/disaggregation/kv_events.py b/python/sglang/srt/disaggregation/kv_events.py index ef7dda684ea8..91d8b5ee9443 100644 --- a/python/sglang/srt/disaggregation/kv_events.py +++ b/python/sglang/srt/disaggregation/kv_events.py @@ -256,10 +256,15 @@ def _socket_setup(self) -> None: self._pub = self._ctx.socket(zmq.PUB) self._pub.set_hwm(self._hwm) # Heuristic: bind if wildcard / * present, else connect. - # bind stable, connect volatile convention + # bind stable, connect volatile convention. + # ``0.0.0.0`` is the IPv4 bind-all wildcard alongside ``*`` + # and ``::``; ``/server_info`` advertises it as a wildcard, + # so the publisher must bind it for the advertised endpoint + # to actually be listening. if ( "*" in self._endpoint or "::" in self._endpoint + or "0.0.0.0" in self._endpoint or self._endpoint.startswith("ipc://") or self._endpoint.startswith("inproc://") ): diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index ef7405624e83..e3951a77767d 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -1,7 +1,6 @@ from __future__ import annotations import concurrent.futures -import ctypes import dataclasses import logging import os @@ -29,7 +28,9 @@ StagingTransferInfo, ) from sglang.srt.disaggregation.common.utils import ( + AuxDataCodec, FastQueue, + TransferKVChunk, group_concurrent_contiguous, pack_int_lists, unpack_int_lists, @@ -37,10 +38,7 @@ from sglang.srt.disaggregation.mooncake.utils import ( check_mooncake_custom_mem_pool_enabled, ) -from sglang.srt.disaggregation.utils import ( - DisaggregationMode, - filter_kv_indices_for_cp_rank, -) +from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.distributed.parallel_state import get_mooncake_transfer_engine from sglang.srt.environ import envs from sglang.srt.server_args import ServerArgs @@ -64,17 +62,6 @@ def __str__(self): return f"KVTransferError(bootstrap_room={self.bootstrap_room}): {self.failure_reason}" -# prefill -@dataclasses.dataclass -class TransferKVChunk: - room: int - prefill_kv_indices: npt.NDArray[np.int32] - index_slice: slice - is_last_chunk: bool - prefill_aux_index: Optional[int] - state_indices: Optional[List] - - # decode @dataclasses.dataclass class TransferInfo: @@ -162,26 +149,6 @@ def from_zmq(cls, msg: List[bytes]): ) -class AuxDataCodec: - """Handles serialization and deserialization of auxiliary data buffers""" - - @staticmethod - def serialize_data_from_buffer(src_addr, data_length): - """Serialize data from memory buffer to bytes""" - buffer = (ctypes.c_byte * data_length).from_address(src_addr) - return bytes(buffer) - - @staticmethod - def deserialize_data_to_buffer(kv_args, buffer_index, aux_index, data): - """Deserialize bytes into target memory buffer""" - dst_aux_ptr = kv_args.aux_data_ptrs[buffer_index] - item_len = kv_args.aux_item_lens[buffer_index] - dst_addr = dst_aux_ptr + item_len * aux_index - buffer = (ctypes.c_byte * len(data)).from_address(dst_addr) - buffer[:] = data - return - - class MooncakeKVManager(CommonKVManager): AUX_DATA_HEADER = b"AUX_DATA" @@ -1478,62 +1445,8 @@ def decode_thread(): ) self.update_status(bootstrap_room, status) - def heartbeat_checker(): - while True: - time.sleep(self.heartbeat_interval) - with self.connection_lock: - addresses = list(self.prefill_info_table.keys()) - - for bootstrap_addr in addresses: - session = None - try: - with self.session_pool_lock: - session = self.session_pool[bootstrap_addr] - response = session.get( - f"http://{bootstrap_addr}/health", - timeout=(2, 3), - headers={"Connection": "keep-alive"}, - ) - if response.status_code == 200: - self.heartbeat_failures[bootstrap_addr] = 0 - - current_rooms = self.addr_to_rooms_tracker[ - bootstrap_addr - ].copy() - - for bootstrap_room in current_rooms: - # Remove KVPoll.Success requests from the tracker - if bootstrap_room not in self.request_status: - self.addr_to_rooms_tracker[bootstrap_addr].discard( - bootstrap_room - ) - else: - logger.info( - f"Attempting to reconnect to {bootstrap_addr}..." - ) - self.heartbeat_failures[bootstrap_addr] = ( - self.heartbeat_failures.get(bootstrap_addr, 0) + 1 - ) - with self.session_pool_lock: - if bootstrap_addr in self.session_pool: - del self.session_pool[bootstrap_addr] - except Exception: - logger.info(f"Attempting to reconnect to {bootstrap_addr}...") - self.heartbeat_failures[bootstrap_addr] = ( - self.heartbeat_failures.get(bootstrap_addr, 0) + 1 - ) - - if ( - self.heartbeat_failures.get(bootstrap_addr, 0) - >= self.max_failures - ): - self._handle_node_failure(bootstrap_addr) - with self.session_pool_lock: - if bootstrap_addr in self.session_pool: - del self.session_pool[bootstrap_addr] - threading.Thread(target=decode_thread).start() - threading.Thread(target=heartbeat_checker).start() + self._start_heartbeat_checker_thread() def add_transfer_request( self, @@ -1583,6 +1496,13 @@ def add_transfer_request( def get_session_id(self): return self.engine.get_session_id() + def _on_heartbeat_success(self, bootstrap_addr: str): + current_rooms = self.addr_to_rooms_tracker[bootstrap_addr].copy() + for bootstrap_room in current_rooms: + # Remove KVPoll.Success requests from the tracker + if bootstrap_room not in self.request_status: + self.addr_to_rooms_tracker[bootstrap_addr].discard(bootstrap_room) + def _run_one_probe_pass(self) -> None: with self.session_lock: snapshot = list(self.failed_sessions) @@ -1666,34 +1586,16 @@ def __init__( self.conclude_state = None self.init_time = time.time() - def pop_decode_prefix_len(self) -> int: - return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0) - - def should_send_kv_chunk(self, num_pages: int, last_chunk: bool) -> bool: - return num_pages > 0 or last_chunk - def send( self, kv_indices: npt.NDArray[np.int32], state_indices: Optional[List] = None, ): - index_slice = slice(self.curr_idx, self.curr_idx + len(kv_indices)) - self.curr_idx += len(kv_indices) - is_last_chunk = self.curr_idx == self.num_kv_indices - - # Special handling for cp - if self.kv_mgr.enable_all_cp_ranks_for_transfer: - kv_indices, index_slice = filter_kv_indices_for_cp_rank( - self.kv_mgr, - kv_indices, - index_slice, - ) - elif self.kv_mgr.is_dummy_cp_rank: - if not is_last_chunk: - return - else: - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Success) - return + kv_indices, index_slice, is_last_chunk, should_skip = ( + self._prepare_send_indices(kv_indices, state_indices) + ) + if should_skip: + return if not is_last_chunk: self.kv_mgr.add_transfer_request( @@ -1719,21 +1621,9 @@ def poll(self) -> KVPoll: if status in (KVPoll.Success, KVPoll.Failed): self.conclude_state = status elif status == KVPoll.Bootstrapping: - if self.init_time is not None: - now = time.time() - elapsed = now - self.init_time - if elapsed >= self.kv_mgr.bootstrap_timeout: - logger.warning_once( - "Some requests timed out when bootstrapping, " - "which means prefill instances fail to receive the KV indices from the decode instance of this request. " - "If a greater mean TTFT is acceptable, you can 'export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600' (10 minutes) to relax the timeout condition. " - ) - self.kv_mgr.record_failure( - self.bootstrap_room, - f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s in KVPoll.Bootstrapping", - ) - self.conclude_state = KVPoll.Failed - return KVPoll.Failed + timeout_result = self._check_bootstrap_timeout() + if timeout_result is not None: + return timeout_result return status else: @@ -1819,12 +1709,6 @@ def _register_kv_args(self): ] ) - def init( - self, - prefill_dp_rank: int, - ): - super().init(prefill_dp_rank) - def send_metadata( self, kv_indices: npt.NDArray[np.int32], @@ -1874,33 +1758,20 @@ def send_metadata( self.init_time = time.time() def poll(self) -> KVPoll: - if self.conclude_state is None: - status = self.kv_mgr.check_status(self.bootstrap_room) - if status in (KVPoll.Success, KVPoll.Failed): - self.conclude_state = status - elif status == KVPoll.WaitingForInput: - if self.init_time is not None: - now = time.time() - elapsed = now - self.init_time - if elapsed >= self.kv_mgr.waiting_timeout: - logger.warning_once( - "Some requests fail to receive KV Cache transfer done signal after bootstrapping. " - "If a greater mean TTFT is acceptable, you can 'export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600' (10 minutes) to relax the timeout condition. " - ) - self.kv_mgr.record_failure( - self.bootstrap_room, - f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s in KVPoll.WaitingForInput", - ) - self.conclude_state = KVPoll.Failed - return KVPoll.Failed + if self.conclude_state is not None: + return self.conclude_state - return status + status = self.kv_mgr.check_status(self.bootstrap_room) + if status in (KVPoll.Success, KVPoll.Failed): + self.conclude_state = status + elif status == KVPoll.WaitingForInput: + timeout_result = self._check_waiting_timeout() + if timeout_result is not None: + return timeout_result - else: - return self.conclude_state + return status def failure_exception(self): - # Explicitly set the status to failure since this request has failed in another rank if self.conclude_state is None: self.conclude_state = KVPoll.Failed diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index 45bdb6b501b8..408d0a980d10 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -1,6 +1,5 @@ from __future__ import annotations -import ctypes import dataclasses import logging import os @@ -33,11 +32,11 @@ CommonKVReceiver, CommonKVSender, ) -from sglang.srt.disaggregation.common.utils import group_concurrent_contiguous -from sglang.srt.disaggregation.utils import ( - DisaggregationMode, - filter_kv_indices_for_cp_rank, +from sglang.srt.disaggregation.common.utils import ( + AuxDataCodec, + group_concurrent_contiguous, ) +from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.server_args import ServerArgs from sglang.srt.utils.common import get_int_env_var from sglang.srt.utils.network import NetworkAddress, get_local_ip_auto @@ -176,22 +175,6 @@ def from_zmq(cls, payload: List[bytes]) -> KVArgsRegisterInfo: ) -class AuxDataCodec: - @staticmethod - def serialize_data_from_buffer(src_addr, data_length): - buffer = (ctypes.c_byte * data_length).from_address(src_addr) - return bytes(buffer) - - @staticmethod - def deserialize_data_to_buffer(kv_args, buffer_index, aux_index, data): - dst_aux_ptr = kv_args.aux_data_ptrs[buffer_index] - item_len = kv_args.aux_item_lens[buffer_index] - dst_addr = dst_aux_ptr + item_len * aux_index - buffer = (ctypes.c_byte * len(data)).from_address(dst_addr) - buffer[:] = data - return - - @dataclasses.dataclass class TPSliceConfig: page_size: int @@ -1132,7 +1115,7 @@ def add_transfer_request( bootstrap_room: int, kv_indices: npt.NDArray[np.int32], index_slice: slice, - is_last: bool, + is_last_chunk: bool, aux_index: Optional[int] = None, state_indices: Optional[npt.NDArray[np.int32]] = None, ) -> Tuple[List[TransferStatus], Optional[List[TransferInfo]]]: @@ -1163,7 +1146,7 @@ def add_transfer_request( self.update_status(bootstrap_room, KVPoll.Failed) return [], list(transfer_infos.values()) targets.append(TransferTarget(info=info, peer_info=peer_info)) - if is_last: + if is_last_chunk: target_infos_snapshot = list(transfer_infos.values()) result_statuses: List[TransferStatus] = [] @@ -1179,7 +1162,7 @@ def add_transfer_request( ) if ( - is_last + is_last_chunk and state_indices is not None and not info.is_dummy and self.state_mem_descs @@ -1191,7 +1174,7 @@ def add_transfer_request( ) if ( - is_last + is_last_chunk and aux_index is not None and info.dst_aux_index >= 0 and self.pp_group.is_last_rank @@ -1212,7 +1195,7 @@ def add_transfer_request( ) return result_statuses, target_infos_snapshot - if is_last: + if is_last_chunk: with self.transfer_lock: # Keep transfer_infos alive until sender.clear() so abort/failure # paths can still recover notification targets after posting. @@ -1243,38 +1226,28 @@ def send( kv_indices: npt.NDArray[np.int32], state_indices: Optional[List] = None, ): - index_slice = slice(self.curr_idx, self.curr_idx + len(kv_indices)) - self.curr_idx += len(kv_indices) - is_last = self.curr_idx == self.num_kv_indices - - # Special handling for cp - if self.kv_mgr.enable_all_cp_ranks_for_transfer: - kv_indices, index_slice = filter_kv_indices_for_cp_rank( - self.kv_mgr, - kv_indices, - index_slice, - ) - elif self.kv_mgr.is_dummy_cp_rank: - if not is_last: - return - else: - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Success) - return + kv_indices, index_slice, is_last_chunk, should_skip = ( + self._prepare_send_indices(kv_indices, state_indices) + ) + if should_skip: + return - normalized_state = _normalize_state_indices(state_indices) if is_last else None + normalized_state = ( + _normalize_state_indices(state_indices) if is_last_chunk else None + ) statuses, infos = self.kv_mgr.add_transfer_request( self.bootstrap_room, kv_indices, index_slice, - is_last, - aux_index=self.aux_index if is_last else None, + is_last_chunk, + aux_index=self.aux_index if is_last_chunk else None, state_indices=normalized_state, ) self.transfer_statuses.extend(statuses) self._record_transfer_indices(kv_indices, None) if infos is not None: self.pending_infos = infos - if is_last: + if is_last_chunk: self.sent_last_chunk = True self._maybe_finalize_if_room_failed() @@ -1295,15 +1268,9 @@ def poll(self) -> KVPoll: status = self.kv_mgr.check_status(self.bootstrap_room) if status == KVPoll.Bootstrapping: - elapsed = time.time() - self.init_time - if elapsed >= self.kv_mgr.bootstrap_timeout: - reason = ( - f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s " - "waiting for decode handshake" - ) - self.kv_mgr.record_failure(self.bootstrap_room, reason) - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - self._finalize_failure(reason) + timeout_result = self._check_bootstrap_timeout() + if timeout_result is not None: + self._finalize_failure() return KVPoll.Failed return status @@ -1499,14 +1466,10 @@ def poll(self) -> KVPoll: self.conclude_state = status return status - if status == KVPoll.WaitingForInput and self.init_time is not None: - elapsed = time.time() - self.init_time - if elapsed >= self.kv_mgr.waiting_timeout: - reason = f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s waiting for KV transfer" - self.kv_mgr.record_failure(self.bootstrap_room, reason) - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) - self.conclude_state = KVPoll.Failed - return KVPoll.Failed + if status == KVPoll.WaitingForInput: + timeout_result = self._check_waiting_timeout() + if timeout_result is not None: + return timeout_result return status diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 6dc92555d3d0..19f2dd627c08 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -26,14 +26,12 @@ from sglang.srt.disaggregation.common.staging_handler import StagingRegisterInfo from sglang.srt.disaggregation.common.utils import ( FastQueue, + TransferKVChunk, group_concurrent_contiguous, pack_int_lists, unpack_int_lists, ) -from sglang.srt.disaggregation.utils import ( - DisaggregationMode, - filter_kv_indices_for_cp_rank, -) +from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.environ import envs from sglang.srt.server_args import ServerArgs @@ -104,17 +102,6 @@ def from_zmq(cls, msg: List[bytes]): ) -@dataclasses.dataclass -class TransferKVChunk: - room: int - prefill_kv_indices: npt.NDArray[np.int32] - index_slice: slice - is_last: bool - chunk_id: int - prefill_aux_index: Optional[int] - state_indices: Optional[List] - - @dataclasses.dataclass class KVArgsRegisterInfo: """Contains base pointers and other info which only needs to be sent once by KVReceiver. Received by prefill bootstrap thread.""" @@ -176,7 +163,7 @@ class TransferStatus: received_kvs_per_pp: Dict[int, Set[int]] = dataclasses.field( default_factory=lambda: defaultdict(set) ) - # Expected chunk count per pp_rank (set when is_last=True): {pp_rank: expected_count} + # Expected chunk count per pp_rank (set when is_last_chunk=True): {pp_rank: expected_count} expected_kvs_per_pp: Dict[int, int] = dataclasses.field(default_factory=dict) # Number of PP ranks expected to send data. num_pp_ranks_expected: Optional[int] = None @@ -186,12 +173,8 @@ class TransferStatus: received_state_per_pp: Set[int] = dataclasses.field(default_factory=set) # Whether state data is expected (set based on state_type). expects_state: bool = False - # Mark as failed - is_failure: bool = False def is_done(self): - if self.is_failure: - return True if self.num_pp_ranks_expected is None or not self.received_aux: return False # If state data is expected, check all PP ranks have sent it @@ -209,9 +192,6 @@ def is_done(self): return False return True - def is_failed(self): - return self.is_failure - class NixlKVManager(CommonKVManager): def __init__( @@ -471,92 +451,6 @@ def _prefetch_staging_reqs(self, room: int): ) self._staging_ctx.prefetched_rooms.add(room) - def _start_heartbeat_checker_thread(self): - """ - Start the heartbeat checker thread for Decode worker. - TODO (smor): unite nixl heartbeat checker with mooncake's. - """ - - def heartbeat_checker(): - while True: - time.sleep(self.heartbeat_interval) - with self.connection_lock: - addresses = list(self.prefill_info_table.keys()) - - for bootstrap_addr in addresses: - session = None - try: - with self.session_pool_lock: - session = self.session_pool[bootstrap_addr] - response = session.get( - f"http://{bootstrap_addr}/health", - timeout=(2, 3), - headers={"Connection": "keep-alive"}, - ) - if response.status_code == 200: - self.heartbeat_failures[bootstrap_addr] = 0 - - else: - logger.info( - f"Attempting to reconnect to {bootstrap_addr}..." - ) - self.heartbeat_failures[bootstrap_addr] = ( - self.heartbeat_failures.get(bootstrap_addr, 0) + 1 - ) - with self.session_pool_lock: - if bootstrap_addr in self.session_pool: - del self.session_pool[bootstrap_addr] - except Exception: - logger.info(f"Attempting to reconnect to {bootstrap_addr}...") - self.heartbeat_failures[bootstrap_addr] = ( - self.heartbeat_failures.get(bootstrap_addr, 0) + 1 - ) - - if ( - self.heartbeat_failures.get(bootstrap_addr, 0) - >= self.max_failures - ): - self._handle_node_failure(bootstrap_addr) - with self.session_pool_lock: - if bootstrap_addr in self.session_pool: - del self.session_pool[bootstrap_addr] - - threading.Thread(target=heartbeat_checker, daemon=True).start() - - def _handle_node_failure(self, failed_bootstrap_addr): - """Handle failure of a prefill node.""" - with self.connection_lock: - keys_to_remove = [ - k for k in self.connection_pool if k.startswith(failed_bootstrap_addr) - ] - for k in keys_to_remove: - del self.connection_pool[k] - self.prefill_info_table.pop(failed_bootstrap_addr, None) - - possible_affected_rooms = self.addr_to_rooms_tracker.get( - failed_bootstrap_addr, [] - ) - self.addr_to_rooms_tracker.pop(failed_bootstrap_addr, None) - - # Mark all pending transfers associated with the failed node as failed - affected_rooms = [] - for room in possible_affected_rooms: - if ( - room in self.transfer_statuses - and not self.transfer_statuses[room].is_done() - ): - # Mark the transfer as failed - self.transfer_statuses[room].is_failure = True - affected_rooms.append(room) - - logger.error( - f"Lost connection with prefill instance (bootstrap_addr: {failed_bootstrap_addr}), " - f"{len(affected_rooms)} transfers affected" - ) - for room in possible_affected_rooms: - logger.error(f"Let room {room} be failed due to prefill down") - self.update_status(room, KVPoll.Failed) - def check_status(self, bootstrap_room: int): return self.request_status.get(bootstrap_room, KVPoll.WaitingForInput) @@ -606,7 +500,7 @@ def transfer_worker(self, queue: FastQueue, staging_buffer=None): # Skip KV RDMA transfer when there are no pages to send # (e.g., decode-side radix cache matched the entire prefix). - # Aux data is still sent below when is_last=True. + # Aux data is still sent below when is_last_chunk=True. if len(kv_chunk.prefill_kv_indices) > 0: chunked_dst_kv_indice = req.dst_kv_indices[kv_chunk.index_slice] @@ -659,7 +553,7 @@ def transfer_worker(self, queue: FastQueue, staging_buffer=None): if kv_xfer_handle is None: notif = ( f"{req.room}_kv_{kv_chunk.chunk_id}" - f"_{int(kv_chunk.is_last)}_{self.kv_args.engine_rank}" + f"_{int(kv_chunk.is_last_chunk)}_{self.kv_args.engine_rank}" ) if self.is_mla_backend or ( decode_tp_size == self.attn_tp_size @@ -688,7 +582,7 @@ def transfer_worker(self, queue: FastQueue, staging_buffer=None): handles.append(kv_xfer_handle) - if kv_chunk.is_last: + if kv_chunk.is_last_chunk: dst_info = self.decode_kv_args_table[req.agent_name] if kv_chunk.state_indices: state_xfer_handles = self.maybe_send_extra( @@ -739,7 +633,7 @@ def transfer_worker(self, queue: FastQueue, staging_buffer=None): break time.sleep(0) - if kv_chunk.is_last: + if kv_chunk.is_last_chunk: self.update_status(room, KVPoll.Success) # Drop per-room state on Success (parity with mooncake # transfer_worker; staging prefetch sets are NIXL-only). @@ -1265,7 +1159,7 @@ def _do_staging_transfer( return (None, True) notif_tag = ( - f"{req.room}_stg_{kv_chunk.chunk_id}_{int(kv_chunk.is_last)}" + f"{req.room}_stg_{kv_chunk.chunk_id}_{int(kv_chunk.is_last_chunk)}" f"_{self.kv_args.engine_rank}_{chunk_idx}" f"_{page_start}_{num_pages}_{req.agent_name}" ) @@ -1574,13 +1468,13 @@ def add_transfer_request( bootstrap_room: int, kv_indices: npt.NDArray[np.int32], index_slice: slice, - is_last: bool, + is_last_chunk: bool, chunk_id: int, aux_index: Optional[int] = None, state_indices: Optional[List] = None, ): assert self.disaggregation_mode == DisaggregationMode.PREFILL - assert not is_last or (is_last and aux_index is not None) + assert not is_last_chunk or (is_last_chunk and aux_index is not None) # Prefetch STAGING_REQ to decode before enqueueing so decode has # already allocated staging by the time the worker picks up the @@ -1601,7 +1495,7 @@ def add_transfer_request( room=bootstrap_room, prefill_kv_indices=kv_indices, index_slice=index_slice, - is_last=is_last, + is_last_chunk=is_last_chunk, chunk_id=chunk_id, prefill_aux_index=aux_index, state_indices=state_indices, @@ -1628,9 +1522,9 @@ def update_transfer_status(self): tag = components[1] if tag == "kv": chunk_id = int(components[2]) - is_last = bool(int(components[3])) + is_last_chunk = bool(int(components[3])) pp_rank = int(components[4]) if len(components) > 4 else 0 - self._track_kv_arrival(room, chunk_id, is_last, pp_rank) + self._track_kv_arrival(room, chunk_id, is_last_chunk, pp_rank) elif tag == "stg": self._handle_stg_notification(components, room) elif tag == "aux": @@ -1647,13 +1541,13 @@ def _handle_stg_notification(self, components, room: int): Format: {room}_stg_{chunk_id}_{is_last}_{pp_rank}_{chunk_idx}_{page_start}_{num_pages}_{agent_name} """ chunk_id = int(components[2]) - is_last = bool(int(components[3])) + is_last_chunk = bool(int(components[3])) pp_rank = int(components[4]) chunk_idx = int(components[5]) page_start = int(components[6]) num_pages = int(components[7]) agent_name = components[8] if len(components) > 8 else "" - self._track_kv_arrival(room, chunk_id, is_last, pp_rank) + self._track_kv_arrival(room, chunk_id, is_last_chunk, pp_rank) self._handle_staging_chunk_arrived( room, chunk_idx, page_start, num_pages, agent_name ) @@ -1683,10 +1577,12 @@ def _handle_aux_notification(self, room: int, components: List[str]): ): self._maybe_submit_last_scatter(room) - def _track_kv_arrival(self, room: int, chunk_id: int, is_last: bool, pp_rank: int): + def _track_kv_arrival( + self, room: int, chunk_id: int, is_last_chunk: bool, pp_rank: int + ): """Update transfer status tracking for a kv chunk arrival.""" self.transfer_statuses[room].received_kvs_per_pp[pp_rank].add(chunk_id) - if is_last: + if is_last_chunk: self.transfer_statuses[room].expected_kvs_per_pp[pp_rank] = chunk_id + 1 if self.transfer_statuses[room].num_pp_ranks_expected is None: self.transfer_statuses[room].num_pp_ranks_expected = ( @@ -1827,12 +1723,6 @@ def __init__( self._send_error: Optional[Exception] = None self._transfer_start_time: Optional[float] = None - def pop_decode_prefix_len(self) -> int: - return self.kv_mgr.req_to_decode_prefix_len.pop(self.bootstrap_room, 0) - - def should_send_kv_chunk(self, num_pages: int, last_chunk: bool) -> bool: - return num_pages > 0 or last_chunk - def send( self, kv_indices: npt.NDArray[np.int32], @@ -1841,23 +1731,11 @@ def send( if self._send_failed: return - index_slice = slice(self.curr_idx, self.curr_idx + len(kv_indices)) - self.curr_idx += len(kv_indices) - is_last = self.curr_idx == self.num_kv_indices - - # Special handling for cp - if self.kv_mgr.enable_all_cp_ranks_for_transfer: - kv_indices, index_slice = filter_kv_indices_for_cp_rank( - self.kv_mgr, - kv_indices, - index_slice, - ) - elif self.kv_mgr.is_dummy_cp_rank: - if not is_last: - return - else: - self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Success) - return + kv_indices, index_slice, is_last_chunk, should_skip = ( + self._prepare_send_indices(kv_indices, state_indices) + ) + if should_skip: + return if self._transfer_start_time is None and ( len(kv_indices) > 0 or state_indices is not None @@ -1868,14 +1746,14 @@ def send( self.bootstrap_room, kv_indices, index_slice, - is_last, + is_last_chunk, self.chunk_id, self.aux_index, state_indices, ) self._record_transfer_indices(kv_indices, state_indices) self.chunk_id += 1 - if is_last: + if is_last_chunk: self.has_sent = True def poll(self) -> KVPoll: @@ -1892,9 +1770,6 @@ def poll(self) -> KVPoll: ) return status - def clear(self): - super().clear() - def failure_exception(self): if self._send_error is not None: raise self._send_error @@ -1915,12 +1790,6 @@ def __init__( super().__init__(mgr, bootstrap_addr, bootstrap_room) self.init_time = None - def init( - self, - prefill_dp_rank: int, - ): - super().init(prefill_dp_rank) - def send_metadata( self, kv_indices: npt.NDArray[np.int32], @@ -1997,31 +1866,16 @@ def poll(self) -> KVPoll: if not self.started_transfer: return status - now = time.time() - elapsed = now - self.init_time - - if elapsed >= self.kv_mgr.waiting_timeout: - logger.error(f"Request {self.bootstrap_room} waiting_timeout") - self.kv_mgr.record_failure( - self.bootstrap_room, - f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s in KVPoll.WaitingForInput", - ) - self.conclude_state = KVPoll.Failed - return KVPoll.Failed + timeout_result = self._check_waiting_timeout() + if timeout_result is not None: + return timeout_result self.kv_mgr.update_transfer_status() if self.kv_mgr.check_transfer_done(self.bootstrap_room): # type: ignore self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].discard( self.bootstrap_room ) - # Check if the transfer failed - if self.kv_mgr.transfer_statuses[self.bootstrap_room].is_failed(): - self.conclude_state = KVPoll.Failed - logger.error( - f"Transfer for room {self.bootstrap_room} failed due to node failure" - ) - else: - self.conclude_state = KVPoll.Success + self.conclude_state = KVPoll.Success del self.kv_mgr.transfer_statuses[self.bootstrap_room] return self.conclude_state # type: ignore return KVPoll.WaitingForInput # type: ignore diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index bb07f4012a5c..93dee0cc2070 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -510,6 +510,17 @@ def process_batch_result_disagg_prefill( logits_output.input_token_logprobs = tuple( logits_output.input_token_logprobs.tolist() ) + if logits_output.next_token_top_logprobs_val: + logits_output.next_token_top_logprobs_val = [ + v.tolist() for v in logits_output.next_token_top_logprobs_val + ] + logits_output.next_token_top_logprobs_idx = [ + x.tolist() for x in logits_output.next_token_top_logprobs_idx + ] + if logits_output.next_token_token_ids_logprobs_val: + logits_output.next_token_token_ids_logprobs_val = [ + v.tolist() for v in logits_output.next_token_token_ids_logprobs_val + ] for i, (req, next_token_id) in enumerate( zip(batch.reqs, next_token_ids, strict=True) diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 030b0cb7a9c2..d64fd0298a07 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -11,6 +11,7 @@ import torch import torch.distributed as dist +from sglang.srt.disaggregation.base import KVPoll from sglang.srt.environ import envs from sglang.srt.utils import is_npu @@ -23,6 +24,7 @@ CommonKVSender, ) from sglang.srt.managers.schedule_batch import Req + from sglang.srt.server_args import ServerArgs ######################### # Constants & Enums @@ -52,17 +54,54 @@ def to_engine_type(mode: str) -> str: FAILURE_PROB = float(os.getenv("DISAGGREGATION_TEST_FAILURE_PROB", 0)) -def poll_and_all_reduce(pollers, gloo_group: dist.ProcessGroup): +def _is_fake_transfer(req: Req, server_args: ServerArgs) -> bool: + return req.bootstrap_host == FAKE_BOOTSTRAP_HOST or ( + req.bootstrap_host is None + and server_args.disaggregation_transfer_backend == "fake" + ) + + +def _apply_metadata_gate(polls, decode_reqs, metadata_buffers, server_args) -> None: + """Downgrade Success → Transferring for requests whose metadata hasn't landed. + + Mutates `polls` in-place. Called before all-reduce so that MIN across TP + ranks naturally prevents any rank from committing before all ranks are ready. + """ + for i, poll_val in enumerate(polls): + if poll_val == int(KVPoll.Success): + decode_req = decode_reqs[i] + if _is_fake_transfer(decode_req.req, server_args): + continue + actual_room = metadata_buffers.bootstrap_room[ + decode_req.metadata_buffer_index, 0 + ].item() + if actual_room == 0: + polls[i] = int(KVPoll.Transferring) + + +def poll_and_all_reduce( + pollers, + gloo_group: dist.ProcessGroup, + decode_reqs=None, + metadata_buffers: Optional[MetadataBuffers] = None, + server_args: Optional[ServerArgs] = None, +): # at a certain prob, the poll is failed to simulate failure if FAILURE_PROB > 0: - from sglang.srt.disaggregation.base import KVPoll - polls = [ int(KVPoll.Failed) if random.random() < FAILURE_PROB else int(poller.poll()) for poller in pollers ] else: polls = [int(poller.poll()) for poller in pollers] + + # Apply metadata gate on the decode requests to downgrade Success → Transferring for requests whose metadata hasn't landed. + if ( + decode_reqs is not None + and metadata_buffers is not None + and server_args is not None + ): + _apply_metadata_gate(polls, decode_reqs, metadata_buffers, server_args) tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu") dist.all_reduce(tensor_to_reduce, op=dist.ReduceOp.MIN, group=gloo_group) return tensor_to_reduce.tolist() @@ -89,11 +128,13 @@ def poll_and_all_reduce_attn_cp_tp_group( def poll_and_all_reduce_with_staging( - decode_reqs, staging_handler, gloo_group: dist.ProcessGroup + decode_reqs, + staging_handler, + gloo_group: dist.ProcessGroup, + metadata_buffers: Optional[MetadataBuffers] = None, + server_args: Optional[ServerArgs] = None, ): """Staging-aware polling: advance scatter, demote incomplete transfers, all_reduce.""" - from sglang.srt.disaggregation.base import KVPoll - for decode_req in decode_reqs: if decode_req.kv_receiver.require_staging and not staging_handler.is_done( decode_req @@ -107,6 +148,9 @@ def poll_and_all_reduce_with_staging( decode_req ): raw_polls[i] = int(KVPoll.Transferring) + # Apply metadata gate on the decode requests to downgrade Success → Transferring for requests whose metadata hasn't landed. + if metadata_buffers is not None and server_args is not None: + _apply_metadata_gate(raw_polls, decode_reqs, metadata_buffers, server_args) poll_tensor = torch.tensor(raw_polls, dtype=torch.uint8, device="cpu") dist.all_reduce(poll_tensor, op=dist.ReduceOp.MIN, group=gloo_group) return poll_tensor.tolist() diff --git a/python/sglang/srt/dllm/mixin/req.py b/python/sglang/srt/dllm/mixin/req.py index 720b9d1db162..80b624f12fbe 100644 --- a/python/sglang/srt/dllm/mixin/req.py +++ b/python/sglang/srt/dllm/mixin/req.py @@ -1,6 +1,7 @@ from __future__ import annotations import enum +from array import array from typing import TYPE_CHECKING, Optional from sglang.srt.dllm.config import DllmConfig @@ -62,7 +63,7 @@ def _init_fill_ids_for_dllm(self: Req): self.fill_ids = ( self.origin_input_ids + self.output_ids - + [self.dllm_config.mask_id] * self.dllm_config.block_size + + array("q", [self.dllm_config.mask_id] * self.dllm_config.block_size) ) def _update_block_offset_for_dllm(self): diff --git a/python/sglang/srt/dllm/mixin/scheduler.py b/python/sglang/srt/dllm/mixin/scheduler.py index 834fec06af3e..3fbff753118a 100644 --- a/python/sglang/srt/dllm/mixin/scheduler.py +++ b/python/sglang/srt/dllm/mixin/scheduler.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from array import array from typing import TYPE_CHECKING, List, Optional, Set, Union from sglang.srt.dllm.config import DllmConfig @@ -79,7 +80,7 @@ def process_batch_result_dllm( if new_tokens == 0: continue - req.fill_ids[-new_tokens:] = next_token_ids[:] + req.fill_ids[-new_tokens:] = array("q", next_token_ids) self.metrics_reporter.num_generated_tokens += new_tokens req.output_ids.extend(next_token_ids) diff --git a/python/sglang/srt/elastic_ep/elastic_ep.py b/python/sglang/srt/elastic_ep/elastic_ep.py index 0cf0ebd0c66d..16a8546183c1 100644 --- a/python/sglang/srt/elastic_ep/elastic_ep.py +++ b/python/sglang/srt/elastic_ep/elastic_ep.py @@ -100,7 +100,7 @@ def healthy_rank_state( def _get_process_group_backend(process_group, device: str): - return process_group._get_backend(torch.device(device)) + return process_group def _iter_live_parallel_groups() -> Iterator[parallel_state.GroupCoordinator]: diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 3a3274c18c2f..0c585e4609e9 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -679,12 +679,18 @@ async def server_info(): await _global_state.tokenizer_manager.get_internal_state() ) + server_args = _global_state.tokenizer_manager.server_args + # server_args.model_config is not serializable but should be excluded by asdict. return { - **dataclasses.asdict(_global_state.tokenizer_manager.server_args), + **dataclasses.asdict(server_args), **_global_state.scheduler_info, "internal_states": internal_states, "version": __version__, + # Structured KV-event publisher descriptor for KV-aware routers. + # `None` when publishing is disabled or misconfigured; see + # `ServerArgs.describe_kv_events_publisher` for the precise contract. + "kv_events": server_args.describe_kv_events_publisher(), } diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 3f9c64914948..d7f186b05fc9 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -254,6 +254,18 @@ def __init__( # Per-request response parser for custom decoding (set by _encode_messages) self._response_parser: Optional[ResponseParserProtocol] = None + # Probe whether ``encode("")`` returns specials. If it does, we must + # keep ``add_special_tokens=False`` at the chat-template encode site + # to avoid double BOS; otherwise the kwarg is a no-op and dropping it + # lets slow tokenizers (e.g. Kimi's TikTokenTokenizer) stay on the + # fast internal path. + try: + self._tokenizer_auto_adds_specials = ( + len(self.tokenizer_manager.tokenizer.encode("")) > 0 + ) + except Exception: + self._tokenizer_auto_adds_specials = True + def _handle_last_assistant_message( self, messages: List[Dict[str, Any]], @@ -805,15 +817,28 @@ def _apply_jinja_template( if request.chat_template_kwargs: extra_template_kwargs.update(request.chat_template_kwargs) + # Split apply_chat_template(tokenize=True) into render + encode so we + # can skip add_special_tokens=False on tokenizers that don't auto-add + # specials (Kimi-like, OpenAI-chat analogue of #25265). Chat + # templates already include role/special tokens, so the encode must + # avoid double BOS on tokenizers that would add it. + encode_kwargs = ( + {"add_special_tokens": False} + if self._tokenizer_auto_adds_specials + else {} + ) try: - prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( + rendered_prompt = self.tokenizer_manager.tokenizer.apply_chat_template( openai_compatible_messages, - tokenize=True, + tokenize=False, add_generation_prompt=True, tools=tools, return_dict=False, **extra_template_kwargs, ) + prompt_ids = self.tokenizer_manager.tokenizer.encode( + rendered_prompt, **encode_kwargs + ) except Exception as e: # If the first attempt fails, try with flat function-only format. # Some templates (e.g. Mistral) expect tools without the OpenAI wrapper. @@ -823,16 +848,22 @@ def _apply_jinja_template( else None ) try: - prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( - openai_compatible_messages, - tokenize=True, - add_generation_prompt=True, - tools=tools, - return_dict=False, - **extra_template_kwargs, + rendered_prompt = ( + self.tokenizer_manager.tokenizer.apply_chat_template( + openai_compatible_messages, + tokenize=False, + add_generation_prompt=True, + tools=tools, + return_dict=False, + **extra_template_kwargs, + ) + ) + prompt_ids = self.tokenizer_manager.tokenizer.encode( + rendered_prompt, **encode_kwargs ) - except jinja2.TemplateError as template_error: + except (jinja2.TemplateError, TypeError) as template_error: # Template errors (e.g., from raise_exception in Jinja templates) + # and TypeError (e.g., tojson filter on Jinja2 Undefined variables) # should be treated as client errors (400 BadRequest) raise ValueError(str(template_error)) from template_error diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 2791aeec9a8e..a91ab5b37fbb 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -288,12 +288,16 @@ class Envs: # Scheduler: others: SGLANG_EMPTY_CACHE_INTERVAL = EnvFloat(-1) # in seconds. Set if you observe high memory accumulation over a long serving period. SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP = EnvBool(False) + # PP: skip output send/recv when the entire batch consists of non-final chunked prefill requests, + # since process_batch_result_prefill discards next_token_ids for those anyway. + SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM = EnvBool(False) SGLANG_SCHEDULER_MAX_RECV_PER_POLL = EnvInt(-1) SGLANG_EXPERIMENTAL_CPP_RADIX_TREE = EnvBool(False) SGLANG_RADIX_FORCE_MISS = EnvBool(False) SGLANG_DYNAMIC_CHUNKING_SMOOTH_FACTOR = EnvFloat(0.75) SGLANG_SCHEDULER_SKIP_ALL_GATHER = EnvBool(False) SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE = EnvBool(False) + SGLANG_KILLPG_ON_SCHEDULER_EXCEPTION = EnvBool(False) SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(None) SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None) SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1) @@ -465,6 +469,8 @@ class Envs: # DSA Backend (canonical names; fall back to SGLANG_NSA_* with deprecation warning) SGLANG_DSA_FUSE_TOPK = EnvBoolWithAlias(True, deprecated_name="SGLANG_NSA_FUSE_TOPK") + SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTIC = EnvBool(False) + SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK = EnvStr(None) SGLANG_DSA_ENABLE_MTP_PRECOMPUTE_METADATA = EnvBoolWithAlias( True, deprecated_name="SGLANG_NSA_ENABLE_MTP_PRECOMPUTE_METADATA" ) @@ -520,8 +526,10 @@ class Envs: # Spec Config SGLANG_SPEC_ENABLE_STRICT_FILTER_CHECK = EnvBool(True) - SGLANG_SPEC_NAN_DETECTION = EnvBool(False) - SGLANG_SPEC_OOB_DETECTION = EnvBool(False) + # Master switch for all async-asserted invariant probes (NaN, Inf, OOB, + # page alignment). Off in prod; tests turn it on to fail-fast on + # numerical / index violations instead of getting silent NaN cascades. + SGLANG_ENABLE_ASYNC_ASSERT = EnvBool(False) # VLM SGLANG_VLM_CACHE_SIZE_MB = EnvInt(100) @@ -624,7 +632,11 @@ class Envs: SGLANG_OPT_USE_TRITON_SWA_PREPARE = EnvBool(True) SGLANG_OPT_USE_AITER_MHC_PRE = EnvBool(True) SGLANG_OPT_USE_AITER_MHC_POST = EnvBool(True) + SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False) SGLANG_OPT_USE_FUSED_COMPRESS = EnvBool(False) + SGLANG_OPT_USE_FUSED_COMPRESS_TRITON = EnvBool(False) + SGLANG_OPT_USE_FUSED_QK_NORM_ROPE = EnvBool(True) + SGLANG_OPT_USE_FUSED_CLAMP_ACT_MUL = EnvBool(True) SGLANG_FIX_MTP_HC_HIDDEN = EnvBool(False) # ==================================================================== @@ -639,6 +651,7 @@ class Envs: SGLANG_OPT_USE_TILELANG_MHC_PRE = EnvBool(True) SGLANG_OPT_USE_TILELANG_MHC_POST = EnvBool(True) SGLANG_OPT_USE_TILELANG_INDEXER = EnvBool(False) + SGLANG_OPT_USE_AITER_INDEXER = EnvBool(False) SGLANG_OPT_USE_JIT_INDEXER_METADATA = EnvBool(True) SGLANG_OPT_USE_ONLINE_COMPRESS = EnvBool(False) SGLANG_OPT_USE_COMPRESSOR_V2 = EnvBool(True) @@ -683,6 +696,7 @@ class Envs: # Cache / overlap SGLANG_OPT_USE_FUSED_STORE_CACHE = EnvBool(True) + SGLANG_OPT_USE_JIT_NORM = EnvBool(True) SGLANG_OPT_USE_MULTI_STREAM_OVERLAP = EnvBool(True) # CUDA graph @@ -703,6 +717,9 @@ class Envs: SGLANG_ENCODER_RECV_TIMEOUT = EnvFloat(180.0) SGLANG_ENCODER_SEND_TIMEOUT = EnvFloat(180.0) SGLANG_ENCODER_DISPATCH_MIN_ITEMS = EnvInt(2) + SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU = EnvBool(False) + SGLANG_ENCODER_MAX_BATCH_SIZE = EnvInt(8) + SGLANG_ENCODER_REQ_TIMEOUT = EnvFloat(180.0) # Elastic EP Backup Port SGLANG_BACKUP_PORT_BASE = EnvInt(10000) diff --git a/python/sglang/srt/eplb/expert_distribution.py b/python/sglang/srt/eplb/expert_distribution.py index 30a1f302c81a..17d20fbbd08e 100644 --- a/python/sglang/srt/eplb/expert_distribution.py +++ b/python/sglang/srt/eplb/expert_distribution.py @@ -30,7 +30,11 @@ from sglang.srt.environ import envs from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.observability.metrics_collector import ExpertDispatchCollector +from sglang.srt.observability.metrics_collector import ( + STAT_LOGGER_ROLE_EXPERT_DISPATCH, + ExpertDispatchCollector, + resolve_collector_class, +) from sglang.srt.server_args import ServerArgs from sglang.srt.utils import Withable, get_device, get_int_env_var @@ -672,7 +676,12 @@ def __init__(self, *args, **kwargs): self.window_sizes = [10, 100, 1000] self._history = _DequeCollection(maxlens=self.window_sizes) self._rank = torch.distributed.get_rank() - self._expert_dispatch_collector = ExpertDispatchCollector( + expert_dispatch_cls = resolve_collector_class( + self._server_args, + STAT_LOGGER_ROLE_EXPERT_DISPATCH, + ExpertDispatchCollector, + ) + self._expert_dispatch_collector = expert_dispatch_cls( self._expert_location_metadata.ep_size ) self._metric_heatmap_collection_counter = 0 diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index 602e93fdd67c..432929d305fb 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -28,6 +28,7 @@ from sglang.srt.function_call.lfm2_detector import Lfm2Detector from sglang.srt.function_call.llama32_detector import Llama32Detector from sglang.srt.function_call.mimo_detector import MiMoDetector +from sglang.srt.function_call.minicpm5_detector import MiniCPM5Detector from sglang.srt.function_call.minimax_m2 import MinimaxM2Detector from sglang.srt.function_call.mistral_detector import MistralDetector from sglang.srt.function_call.poolside_v1_detector import PoolsideV1Detector @@ -66,6 +67,7 @@ class FunctionCallParser: "lfm2": Lfm2Detector, "llama3": Llama32Detector, "mimo": MiMoDetector, + "minicpm5": MiniCPM5Detector, "mistral": MistralDetector, "poolside_v1": PoolsideV1Detector, "pythonic": PythonicDetector, diff --git a/python/sglang/srt/function_call/minicpm5_detector.py b/python/sglang/srt/function_call/minicpm5_detector.py new file mode 100644 index 000000000000..ec475a051d1f --- /dev/null +++ b/python/sglang/srt/function_call/minicpm5_detector.py @@ -0,0 +1,317 @@ +import ast +import json +import logging +import re +from typing import Dict, List, Optional + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + _GetInfoFunc, +) + +logger = logging.getLogger(__name__) + +try: + from lxml import etree as ET # type: ignore + + _HAS_LXML = True +except Exception: # pragma: no cover - environment may not have lxml + import xml.etree.ElementTree as ET # type: ignore + + _HAS_LXML = False + +_FUNC_NAME_V1_REGEX = re.compile(r"]*>") +_PARAM_WITH_NAME_REGEX = re.compile( + r"([\s\S]*?)", re.DOTALL +) +_PARAM_MISSING_NAME_REGEX = re.compile(r"]*\bname=)[^>]*>", re.DOTALL) + + +def get_argument_type( + func_name: str, arg_key: str, name_to_tool: Dict[str, Tool] +) -> Optional[str]: + tool = name_to_tool.get(func_name) + if not tool: + return None + params = tool.function.parameters or {} + if not isinstance(params, dict): + return None + return params.get("properties", {}).get(arg_key, {}).get("type") + + +def parse_arguments(json_value): + try: + try: + parsed_value = json.loads(json_value) + except (json.JSONDecodeError, TypeError): + parsed_value = ast.literal_eval(json_value) + return parsed_value, True + except (ValueError, SyntaxError, TypeError): + return json_value, False + + +class MiniCPM5Detector(BaseFormatDetector): + """ + Detector for MiniCPM-4 models (V3 schema) adapted to the new chat template. + + Expected format example (multiple calls allowed): + 北京2024-06-27 + + """ + + def __init__(self): + super().__init__() + self.bot_token = "" + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a MiniCPM-4 V3 XML-styled tool call.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + idx = text.find(self.bot_token) + if idx == -1: + return StreamingParseResult(normal_text=text, calls=[]) + + normal_parts = [] + calls = [] + name_to_tool = {t.function.name: t for t in tools if t.function.name} + tool_names = set(name_to_tool.keys()) + name_to_allowed_props = {} + name_to_required = {} + for name, t in name_to_tool.items(): + params = t.function.parameters or {} + props = ( + (params.get("properties", {}) or {}) if isinstance(params, dict) else {} + ) + name_to_allowed_props[name] = set(props.keys()) + req = params.get("required", []) if isinstance(params, dict) else [] + try: + name_to_required[name] = set(req) + except Exception: + name_to_required[name] = set() + + try: + last_end = 0 + for m in re.finditer(self.func_call_regex, text, re.DOTALL): + if m.start() > last_end: + normal_parts.append(text[last_end : m.start()]) + + block = m.group(0) + func_name = None + arguments = {} + parsed_ok = False + param_invalid = False + + # Primary path: XML parsing (lxml preferred, stdlib fallback) + try: + if _HAS_LXML: + try: + parser = ET.XMLParser(**{"strip_cdata": False}) # type: ignore[call-arg] + except TypeError: + parser = ET.XMLParser() + root = ET.fromstring(block, parser=parser) + else: + root = ET.fromstring(block) + + if root.tag == "function": + func_node = root + else: + func_node = ( + root.find("function") if hasattr(root, "find") else None + ) + + if func_node is not None: + func_name = (func_node.attrib.get("name") or "").strip() + + args_node = ( + func_node.find("arguments") if func_node is not None else None + ) + param_nodes = [] + if func_node is not None: + param_nodes = list(func_node.findall("param")) + if args_node is not None and not param_nodes: + param_nodes = list(args_node.findall("param")) + + if func_node is not None: + seen_keys = set() + allowed_props = set() + if func_name in tool_names: + allowed_props = name_to_allowed_props.get(func_name, set()) + has_invalid_param = False + for param in param_nodes: + key = param.attrib.get("name") + if not key: + has_invalid_param = True + break + if allowed_props and key not in allowed_props: + has_invalid_param = True + break + if key in seen_keys: + has_invalid_param = True + break + seen_keys.add(key) + val_text = param.text or "" + val_text = val_text.strip() + arg_type = get_argument_type( + func_name or "", key, name_to_tool + ) + if arg_type != "string": + parsed_val, _ = parse_arguments(val_text) + arguments[key] = parsed_val + else: + arguments[key] = val_text + if has_invalid_param: + arguments.clear() + param_invalid = True + parsed_ok = bool(func_name) + except Exception: + parsed_ok = False + + if not parsed_ok: + # Fallback path: regex extraction + try: + m_fn = _FUNC_NAME_V1_REGEX.search(block) + if m_fn: + func_name = (m_fn.group(1) or "").strip() + has_invalid_param = ( + _PARAM_MISSING_NAME_REGEX.search(block) is not None + ) + seen_keys = set() + allowed_props = set() + if func_name in tool_names: + allowed_props = name_to_allowed_props.get(func_name, set()) + for pm in _PARAM_WITH_NAME_REGEX.finditer(block): + key = pm.group(1).strip() + if allowed_props and key not in allowed_props: + has_invalid_param = True + break + if key in seen_keys: + has_invalid_param = True + break + seen_keys.add(key) + val_text = pm.group(2) or "" + if val_text.startswith("" + ): + val_text = val_text[len("")] + val_text = val_text.strip() + arg_type = get_argument_type( + func_name or "", key, name_to_tool + ) + if arg_type != "string": + parsed_val, _ = parse_arguments(val_text) + arguments[key] = parsed_val + else: + arguments[key] = val_text + if has_invalid_param: + arguments.clear() + param_invalid = True + parsed_ok = bool(func_name) + except Exception: + parsed_ok = False + + if not func_name or func_name not in tool_names or param_invalid: + parsed_ok = False + else: + req_props = name_to_required.get(func_name, set()) + if req_props and not req_props.issubset(arguments.keys()): + parsed_ok = False + + if parsed_ok: + tool_call_obj = {"name": func_name, "parameters": arguments} + calls.extend(self.parse_base_json(tool_call_obj, tools)) + else: + normal_parts.append(block) + + last_end = m.end() + + if last_end < len(text): + normal_parts.append(text[last_end:]) + + return StreamingParseResult(normal_text="".join(normal_parts), calls=calls) + except Exception as e: + logger.error(f"Error in detect_and_parse: {e}") + return StreamingParseResult(normal_text=text) + + def _append_tool_call(self, call, all_calls: List) -> None: + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + self.prev_tool_call_arr[self.current_tool_id] = { + "name": call.name, + "arguments": json.loads(call.parameters), + } + self.streamed_args_for_tool[self.current_tool_id] = call.parameters + call.tool_index = self.current_tool_id + self.current_tool_id += 1 + all_calls.append(call) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + self._buffer += new_text + normal_parts = [] + all_calls = [] + + while True: + current_text = self._buffer + start = current_text.find(self.bot_token) + if start == -1: + partial_len = self._ends_with_partial_token( + current_text, self.bot_token + ) + if partial_len > 0: + self._buffer = current_text[-partial_len:] + emit = current_text[:-partial_len] + else: + self._buffer = "" + emit = "" if self.current_tool_id > 0 else current_text + if emit: + normal_parts.append(emit) + break + + if start > 0: + normal_parts.append(current_text[:start]) + current_text = current_text[start:] + + end = current_text.find(self.eot_token) + if end == -1: + self._buffer = current_text + break + + block = current_text[: end + len(self.eot_token)] + self._buffer = current_text[end + len(self.eot_token) :] + + result = self.detect_and_parse(block, tools=tools) + for call in result.calls: + self._append_tool_call(call, all_calls) + + if self.bot_token not in self._buffer: + partial_len = self._ends_with_partial_token( + self._buffer, self.bot_token + ) + if partial_len == 0: + emit = "" if self.current_tool_id > 0 else self._buffer + if emit: + normal_parts.append(emit) + self._buffer = "" + break + + return StreamingParseResult(normal_text="".join(normal_parts), calls=all_calls) + + def supports_structural_tag(self) -> bool: + return False + + def structure_info(self) -> _GetInfoFunc: + raise NotImplementedError() diff --git a/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py b/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py index 17fb35ae6ccb..6044e0a82469 100644 --- a/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py +++ b/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py @@ -264,11 +264,11 @@ def forward_extend( else forward_batch.encoder_out_cache_loc ) if not self.use_mla: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) else: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, @@ -357,9 +357,7 @@ def forward_extend( can_run_tbo=forward_batch.can_run_tbo, ) if not self.use_mla: - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim @@ -555,9 +553,9 @@ def _fa_cp_attn( return output, lse return output else: - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer( - layer.layer_id - ).to(q.dtype) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to( + q.dtype + ) k_rope = kv_cache[:, :, layer.v_head_dim :] c_kv = kv_cache[:, :, : layer.v_head_dim] k_rope_cache = k_rope.view( @@ -657,11 +655,11 @@ def forward_decode( else forward_batch.encoder_out_cache_loc ) if not self.use_mla: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) else: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, @@ -710,9 +708,7 @@ def forward_decode( can_run_tbo=forward_batch.can_run_tbo, ) if not self.use_mla: - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim ) @@ -831,9 +827,7 @@ def forward_decode( else: o = result else: - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( - q.dtype - ) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype) k_rope = kv_cache[:, :, layer.v_head_dim :] c_kv = kv_cache[:, :, : layer.v_head_dim] k_rope_cache = k_rope.view( diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py index 1035a96eb7c6..03811968824d 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py @@ -208,7 +208,9 @@ def get_splitfuse_attn_mask( return attn_mask -def _cp_allgather_and_save_kv_npu(forward_batch, layer, k, v, cp_size): +def _cp_allgather_and_save_kv_npu( + forward_batch, layer, k, v, cp_size, token_to_kv_pool +): """NPU-compatible CP KV all-gather with merged K/V communication. Merges K and V along the feature dimension so only one all-gather is @@ -243,7 +245,7 @@ def _cp_allgather_and_save_kv_npu(forward_batch, layer, k, v, cp_size): key_cache_full = kv_full[..., :k_feat_size].reshape(-1, *k_tail) value_cache_full = kv_full[..., k_feat_size:].reshape(-1, *v_tail) - forward_batch.token_to_kv_pool.set_kv_buffer( + token_to_kv_pool.set_kv_buffer( layer, cache_loc, key_cache_full, @@ -287,6 +289,10 @@ def __init__(self, model_runner: ModelRunner, speculative_step_id: int = 0): self.native_attn = AscendTorchNativeAttnBackend() self.graph_metadata = {} self.max_context_len = model_runner.model_config.context_len + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.graph_mode = False self.use_fia = get_bool_env_var("ASCEND_USE_FIA", "False") @@ -357,7 +363,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ): seq_lens_max += self.speculative_step_id + 1 self.forward_metadata.block_tables = ( - forward_batch.req_to_token_pool.req_to_token[ + self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, :seq_lens_max ][:, :: self.page_size] // self.page_size @@ -366,7 +372,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): self.forward_metadata.block_tables_swa = ( ( self.full_to_swa_index_mapping[ - forward_batch.req_to_token_pool.req_to_token[ + self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, :seq_lens_max ] ][:, :: self.page_size] @@ -421,7 +427,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): for req_idx, seq_len in zip( forward_batch.req_pool_indices.tolist(), seq_prefix_lens ): - req_indices = forward_batch.req_to_token_pool.req_to_token[req_idx] + req_indices = self.req_to_token_pool.req_to_token[req_idx] req_prefix_block_tables = ( req_indices[:seq_len][:: self.page_size] // self.page_size ) @@ -883,11 +889,11 @@ def forward_sparse( if save_kv_cache: k = k.view(-1, layer.tp_k_head_num, self.kv_lora_rank) k_rope = k_rope.view(-1, layer.tp_k_head_num, self.qk_rope_head_dim) - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, k_rope ) q_nope, q_pe = q, q_rope - k_nope, k_pe = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + k_nope, k_pe = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) if is_prefill: if self.forward_metadata.actual_seq_lengths_q is not None: @@ -1041,7 +1047,12 @@ def forward_extend( if is_cp_mode: # All-gather K/V from all CP ranks and write full sequence to KV pool _cp_allgather_and_save_kv_npu( - forward_batch, layer, k, v, self.attn_cp_size + forward_batch, + layer, + k, + v, + self.attn_cp_size, + self.token_to_kv_pool, ) else: # support cross attention @@ -1050,10 +1061,10 @@ def forward_extend( if not layer.is_cross_attention else forward_batch.encoder_out_cache_loc ) - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id) if sinks is not None: # Use SWA block tables if hybrid SWA is enabled for this layer @@ -1200,7 +1211,7 @@ def forward_extend( o_, k_cache.view(-1, layer.tp_k_head_num, layer.qk_head_dim), v_cache.view(-1, layer.tp_v_head_num, layer.v_head_dim), - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, forward_batch.extend_prefix_lens, @@ -1223,10 +1234,8 @@ def forward_extend( if layer.qk_head_dim == layer.v_head_dim: q = q.reshape(-1, layer.tp_q_head_num, layer.qk_head_dim) - k_buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_buffer = forward_batch.token_to_kv_pool.get_value_buffer( - layer.layer_id - ) + k_buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_buffer = self.token_to_kv_pool.get_value_buffer(layer.layer_id) kv_cached = torch.index_select( k_buffer, 0, self.forward_metadata.flatten_prefix_block_tables ) @@ -1335,10 +1344,8 @@ def forward_extend( ) # 2nd, load history kvcache(kv_a and k_pe) and calculate k_nope - k_buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_buffer = forward_batch.token_to_kv_pool.get_value_buffer( - layer.layer_id - ) + k_buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_buffer = self.token_to_kv_pool.get_value_buffer(layer.layer_id) kv_cached = torch.index_select( k_buffer, 0, self.forward_metadata.flatten_prefix_block_tables ) @@ -1427,7 +1434,7 @@ def forward_extend( kv_lora_rank = k.shape[-1] - self.qk_rope_head_dim kv_c, k_rope = k.split([kv_lora_rank, self.qk_rope_head_dim], dim=-1) if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, kv_c, k_rope ) attn_output = q.new_empty( @@ -1435,17 +1442,15 @@ def forward_extend( ) use_gqa = layer.tp_q_head_num != layer.tp_k_head_num - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer( - layer.layer_id - ) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id) kv_cache = torch.cat([k_cache, v_cache], dim=-1) attn_output = self.native_attn.run_sdpa_forward_extend( q, attn_output, kv_cache.view(-1, layer.tp_k_head_num, layer.qk_head_dim), k_cache.view(-1, layer.tp_v_head_num, layer.v_head_dim), - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, forward_batch.extend_prefix_lens, @@ -1514,12 +1519,12 @@ def forward_dllm( topk_indices: Optional[torch.Tensor] = None, ): if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id) query = q.reshape(-1, layer.tp_q_head_num, layer.qk_head_dim) if self.forward_metadata.seq_lens_cpu_int is None: @@ -1574,21 +1579,21 @@ def forward_mtp( if self.use_mla: k = k.view(-1, layer.tp_k_head_num, self.kv_lora_rank) k_rope = k_rope.view(-1, layer.tp_k_head_num, self.qk_rope_head_dim) - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, k_rope ) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) if not self.use_mla: - k_cache = forward_batch.token_to_kv_pool.get_key_buffer( - layer.layer_id - ).view(-1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer( - layer.layer_id - ).view(-1, self.page_size, layer.tp_v_head_num * layer.v_head_dim) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).view( + -1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim + ) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id).view( + -1, self.page_size, layer.tp_v_head_num * layer.v_head_dim + ) query = q.reshape(-1, layer.tp_q_head_num, layer.qk_head_dim).contiguous() if not self.graph_mode: num_token_padding = query.shape[0] @@ -1642,7 +1647,7 @@ def forward_mtp( ) return attn_output else: - c_kv, k_rope = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + c_kv, k_rope = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) if is_fia_nz(): k_rope_cache = _reshape_kv_for_fia_nz( k_rope, layer.tp_k_head_num, self.qk_rope_head_dim, self.page_size @@ -1756,17 +1761,17 @@ def forward_decode_graph( if self.use_mla: k = k.view(-1, layer.tp_k_head_num, self.kv_lora_rank) k_rope = k_rope.view(-1, layer.tp_k_head_num, self.qk_rope_head_dim) - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, k_rope ) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) if sinks is not None: - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id) # Use SWA block tables if hybrid SWA is enabled for this layer if self.is_hybrid_swa and layer.sliding_window_size != -1: @@ -1788,12 +1793,12 @@ def forward_decode_graph( return attn_out if not self.use_mla: - k_cache = forward_batch.token_to_kv_pool.get_key_buffer( - layer.layer_id - ).view(-1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer( - layer.layer_id - ).view(-1, self.page_size, layer.tp_v_head_num * layer.v_head_dim) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).view( + -1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim + ) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id).view( + -1, self.page_size, layer.tp_v_head_num * layer.v_head_dim + ) query = q.reshape(-1, 1, layer.tp_q_head_num * layer.qk_head_dim) if self.forward_metadata.seq_lens_cpu_int is None: actual_seq_len_kv = self.forward_metadata.seq_lens_cpu_list @@ -1836,7 +1841,7 @@ def forward_decode_graph( ) return output.view(num_tokens, layer.tp_q_head_num * layer.v_head_dim) else: - c_kv, k_rope = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + c_kv, k_rope = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) if is_fia_nz(): k_rope_cache = _reshape_kv_for_fia_nz( k_rope, layer.tp_k_head_num, self.qk_rope_head_dim, self.page_size @@ -1976,10 +1981,10 @@ def forward_decode( if not layer.is_cross_attention else forward_batch.encoder_out_cache_loc ) - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) num_tokens = q.shape[0] - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id) if sinks is not None: # Use SWA block tables if hybrid SWA is enabled for this layer @@ -2098,7 +2103,7 @@ def forward_decode( o_, k_cache.view(-1, layer.tp_k_head_num, layer.qk_head_dim), v_cache.view(-1, layer.tp_v_head_num, layer.v_head_dim), - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, forward_batch.encoder_lens, @@ -2112,12 +2117,12 @@ def forward_decode( return attn_output.view(num_tokens, layer.tp_q_head_num * layer.v_head_dim) else: if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, k_rope ) num_tokens = q.shape[0] - kv_c = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - k_pe = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + kv_c = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_pe = self.token_to_kv_pool.get_value_buffer(layer.layer_id) if self.use_fia and (layer.tp_q_head_num // layer.tp_k_head_num) >= 8: """layer.tp_q_head_num // layer.tp_k_head_num < 8 will support in the later version of CANN""" @@ -2218,11 +2223,11 @@ def forward_mixed( "3. When the environment variable ASCEND_USE_FIA is set to 0 and qk_head_dim exceeds 128 on Ascend NPU devices." ) if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - v_cache = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = self.token_to_kv_pool.get_value_buffer(layer.layer_id) num_block, block_size, _, _ = k_cache.shape key = k_cache.view(num_block, block_size, -1) value = v_cache.view(num_block, block_size, -1) diff --git a/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py b/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py index 51cf7421e9ca..1107f11b2a33 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py +++ b/python/sglang/srt/hardware_backend/npu/attention/mla_preprocess.py @@ -6,6 +6,10 @@ import torch.nn.functional as F from sglang.srt.hardware_backend.npu.utils import npu_format_cast +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_token_to_kv_pool, +) from sglang.srt.utils import get_bool_env_var if TYPE_CHECKING: @@ -253,7 +257,7 @@ def get_sin_cos(self, positions): return cos, sin def get_kv_cache_and_cache_idx(self, forward_batch): - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer(self.layer_id) + k_cache, v_cache = get_token_to_kv_pool().get_kv_buffer(self.layer_id) slot_mapping = forward_batch.out_cache_loc.to(dtype=torch.int32) return k_cache, v_cache, slot_mapping @@ -314,15 +318,15 @@ def forward_absorb_prepare_npu_rms_norm_cache( cache_mode = "PA_NZ" if is_fia_nz() else "PA_BNSD" self.kvCache = self.kvCache.view( -1, - forward_batch.attn_backend.page_size, + get_attn_backend().page_size, 1, - forward_batch.attn_backend.kv_lora_rank, + get_attn_backend().kv_lora_rank, ) self.kvCacheRope = self.kvCacheRope.view( -1, - forward_batch.attn_backend.page_size, + get_attn_backend().page_size, 1, - forward_batch.attn_backend.qk_rope_head_dim, + get_attn_backend().qk_rope_head_dim, ) k_rope, k_nope, _, _ = torch.ops.npu.npu_kv_rmsnorm_rope_cache( latent_cache, diff --git a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py index 68f23f1ac081..79f0bb86ab9b 100644 --- a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py +++ b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py @@ -16,6 +16,7 @@ dsa_use_prefill_cp, ) from sglang.srt.layers.communicator import ScatterMode, get_attn_tp_context +from sglang.srt.model_executor.forward_context import get_token_to_kv_pool if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -88,9 +89,7 @@ def forward_mha_prepare_npu( ) q_pe = q_pe.reshape(B, -1, m.qk_rope_head_dim) - ckv_cache, k_rope_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - m.layer_id - ) + ckv_cache, k_rope_cache = get_token_to_kv_pool().get_kv_buffer(m.layer_id) _, _, k_pe, kv_a = torch_npu.npu_kv_rmsnorm_rope_cache( latent_cache.view(-1, 1, 1, m.kv_lora_rank + m.qk_rope_head_dim), # bnsd m.kv_a_layernorm.weight, @@ -115,7 +114,7 @@ def forward_mha_prepare_npu( if m.rotary_emb is not None: q_pe, k_pe = m.rotary_emb(positions, q_pe, k_pe) # this is for model kimi-vl-a3B-instruct - forward_batch.token_to_kv_pool.set_kv_buffer( + get_token_to_kv_pool().set_kv_buffer( m, forward_batch.out_cache_loc, kv_a.unsqueeze(1), k_pe ) diff --git a/python/sglang/srt/hardware_backend/npu/moe/fuseep.py b/python/sglang/srt/hardware_backend/npu/moe/fuseep.py new file mode 100644 index 000000000000..deabbf623bb4 --- /dev/null +++ b/python/sglang/srt/hardware_backend/npu/moe/fuseep.py @@ -0,0 +1,171 @@ +"""Ascend FuseEP fused dispatch+GEMM+combine forward path. + +Follows the mega_moe shape: a free-function bypass invoked from +``FusedMoE.forward`` when ``--moe-a2a-backend ascend_fuseep`` is set, plus a +weight-postprocess helper that NPU quant_methods call from their +``process_weights_after_loading`` when the same backend is selected. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.srt.distributed import get_tp_group +from sglang.srt.environ import envs +from sglang.srt.hardware_backend.npu.utils import FusedMoEMode, npu_format_cast +from sglang.srt.layers.moe.token_dispatcher.deepep import DeepEPBuffer +from sglang.srt.layers.moe.utils import DeepEPMode + +if TYPE_CHECKING: + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + from sglang.srt.layers.moe.topk import TopKOutput + + +_PARAMS_BYTES = 2 # bf16 — Ascend's Dispatch & Combine does not support fp16 + + +def _get_fuseep_buffer(layer: "FusedMoE"): + DeepEPBuffer.set_dispatch_mode_as_low_latency() + return DeepEPBuffer.get_deepep_buffer( + get_tp_group().device_group, + layer.hidden_size, + _PARAMS_BYTES, + DeepEPMode.LOW_LATENCY, + envs.SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get(), + layer.num_experts, + ) + + +def forward_fuseep( + layer: "FusedMoE", + hidden_states: torch.Tensor, + topk_output: "TopKOutput", +) -> torch.Tensor: + buf = _get_fuseep_buffer(layer) + hidden_states, _ = buf.fused_deep_moe( + hidden_states, + topk_idx=topk_output.topk_ids, + topk_weights=topk_output.topk_weights, + gmm1_permuted_weight=layer.w13_weight, + gmm1_permuted_weight_scale=layer.w13_weight_scale, + gmm2_weight=layer.w2_weight, + gmm2_weight_scale=layer.w2_weight_scale, + num_max_dispatch_tokens_per_rank=( + envs.SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() + ), + num_experts=layer.num_experts, + fuse_mode=envs.SGLANG_NPU_FUSED_MOE_MODE.get(), + ) + return hidden_states + + +def _permute_w13_weight_scale(w: torch.Tensor, tile_n: int) -> torch.Tensor: + if tile_n % 2 != 0: + raise ValueError(f"tile_n must be even, got {tile_n}") + + *dims, n = w.shape + if n % tile_n != 0: + raise ValueError(f"Last dimension {n} must be divisible by tile_n {tile_n}") + + w_reshaped = w.reshape(*dims, 2, n // tile_n, tile_n // 2) + perm_order = list(range(len(dims))) + [-2, -3, -1] + return w_reshaped.permute(perm_order).reshape(*dims, n) + + +def _reshape_w13_weight( + weight: torch.Tensor, dim: int, chunk_size: int = 64 +) -> torch.Tensor: + # Achieving greater computing power through reshape on Ascend. + original_shape = weight.shape + if dim < 0: + dim += len(original_shape) + + if original_shape[dim] % (2 * chunk_size) != 0: + raise ValueError( + f"Dimension {dim} size {original_shape[dim]} must be divisible by " + f"{2 * chunk_size}" + ) + + new_shape = ( + *original_shape[:dim], + 2, + original_shape[dim] // (2 * chunk_size), + chunk_size, + *original_shape[dim + 1 :], + ) + + weight = weight.view(new_shape) + weight = weight.transpose(dim, dim + 1).contiguous() + return weight.view(*original_shape[:dim], -1, *original_shape[dim + 1 :]) + + +def _release_weight_cache(weight: torch.Tensor) -> torch.Tensor: + # .contiguous() introduces additional memory overhead; release with resize_(0) + origin_weight = weight.data.transpose(1, 2) + new_weight = origin_weight.contiguous() + origin_weight.untyped_storage().resize_(0) + return new_weight + + +def _scale_from_float_to_int64(scale: torch.Tensor) -> torch.nn.Parameter: + import numpy as np + + converted = torch.from_numpy( + np.frombuffer( + scale.cpu().to(torch.float32).numpy().tobytes(), dtype=np.int32 + ).astype(np.int64) + ).to(scale.device) + return torch.nn.Parameter(converted, requires_grad=False) + + +def process_fuseep_weights(layer: torch.nn.Module) -> None: + """Apply the Ascend FuseEP-specific weight layout. + + Replaces NPU quant_method weight layouts with the form required by the + fused_deep_moe op. Invoked from NPU ``process_weights_after_loading`` + when ``--moe-a2a-backend ascend_fuseep`` is set. + """ + if envs.SGLANG_NPU_FUSED_MOE_MODE.get() == FusedMoEMode.DISPATCH_FFN_COMBINE.value: + w13_weight = _release_weight_cache(layer.w13_weight) + layer.w13_weight.data = npu_format_cast(w13_weight) + w2_weight = _release_weight_cache(layer.w2_weight) + layer.w2_weight.data = npu_format_cast(w2_weight) + + layer.w13_weight_scale.data = layer.w13_weight_scale.data.view( + layer.w13_weight_scale.data.shape[0], -1 + ) + w2_scale = layer.w2_weight_scale.data.squeeze(-1).contiguous() + layer.w2_weight_scale = torch.nn.Parameter( + w2_scale.to(torch.float32), requires_grad=False + ) + + layer.w13_weight_scale = _scale_from_float_to_int64(layer.w13_weight_scale.data) + layer.w2_weight_scale = _scale_from_float_to_int64(layer.w2_weight_scale.data) + else: + cpu_w13 = layer.w13_weight.data.transpose(1, 2).cpu() + layer.w13_weight.data = _reshape_w13_weight(cpu_w13, -1).npu() + w13_scale = layer.w13_weight_scale.data.squeeze(-1).contiguous() + w13_scale = _permute_w13_weight_scale(w13_scale, 128) + layer.w13_weight_scale = torch.nn.Parameter( + w13_scale.to(torch.float32), requires_grad=False + ) + layer.w13_weight.data = npu_format_cast(layer.w13_weight.data) + layer.w2_weight.data = npu_format_cast(layer.w2_weight.data) + + w2_scale = layer.w2_weight_scale.data.squeeze(-1).contiguous() + layer.w2_weight_scale = torch.nn.Parameter( + w2_scale.to(torch.float32), requires_grad=False + ) + + if hasattr(layer, "w13_weight_offset"): + layer.w13_weight_offset = torch.nn.Parameter( + layer.w13_weight_offset.data.squeeze(-1).contiguous(), + requires_grad=False, + ) + if hasattr(layer, "w2_weight_offset"): + layer.w2_weight_offset = torch.nn.Parameter( + layer.w2_weight_offset.data.squeeze(-1).contiguous(), + requires_grad=False, + ) diff --git a/python/sglang/srt/hardware_backend/npu/quantization/fused_moe_method_npu.py b/python/sglang/srt/hardware_backend/npu/quantization/fused_moe_method_npu.py index 31aeb25cf505..910e56fda79a 100644 --- a/python/sglang/srt/hardware_backend/npu/quantization/fused_moe_method_npu.py +++ b/python/sglang/srt/hardware_backend/npu/quantization/fused_moe_method_npu.py @@ -9,7 +9,9 @@ if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher import ( CombineInput, - StandardDispatchOutput, + DeepEPLLDispatchOutput, + DeepEPNormalDispatchOutput, + DispatchOutput, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -384,6 +386,93 @@ def fused_moe_npu( return final_hidden_states +def maybe_apply_deepep_npu( + quant_method, + layer: torch.nn.Module, + dispatch_output: "DispatchOutput", +) -> Optional["CombineInput"]: + """Route DeepEP dispatch outputs through the NPU compute path. + + Replaces the deprecated DeepEPMoE.forward_npu wrapper: detects DeepEP + normal/LL formats, calls ``quant_method.apply_without_routing_weights``, + and wraps the result in the matching CombineInput. Returns None for + non-DeepEP formats so the caller falls through to its standard path. + """ + from sglang.srt.layers.moe.token_dispatcher import ( + DeepEPLLCombineInput, + DeepEPNormalCombineInput, + ) + from sglang.srt.layers.moe.token_dispatcher.base import DispatchOutputChecker + + if not dispatch_output.format.is_deepep(): + return None + + # NOTE: Ascend's Dispatch & Combine does not support FP16 + output_dtype = torch.bfloat16 + group_list_type = 1 + + if DispatchOutputChecker.format_is_deepep_normal(dispatch_output): + if TYPE_CHECKING: + assert isinstance(dispatch_output, DeepEPNormalDispatchOutput) + ( + hidden_states, + hidden_states_scale, + _, + _, + num_recv_tokens_per_expert, + ) = dispatch_output + group_list = torch.tensor( + num_recv_tokens_per_expert, + dtype=torch.int64, + device=hidden_states.device, + ) + combine_cls = DeepEPNormalCombineInput + else: + if TYPE_CHECKING: + assert isinstance(dispatch_output, DeepEPLLDispatchOutput) + ( + hidden_states, + hidden_states_scale, + _, + _, + group_list, + _, + ) = dispatch_output + group_list = group_list.to(torch.int64) + combine_cls = DeepEPLLCombineInput + + hidden_states = quant_method.apply_without_routing_weights( + layer, + hidden_states, + hidden_states_scale, + group_list_type, + group_list, + output_dtype, + ) + + return combine_cls( + hidden_states=hidden_states, + topk_ids=dispatch_output.topk_ids, + topk_weights=dispatch_output.topk_weights, + ) + + +def maybe_apply_fuseep_weights(layer: torch.nn.Module) -> bool: + """Apply the FuseEP weight layout if --moe-a2a-backend is ascend_fuseep. + + Returns True when the FuseEP layout was applied and the caller should + skip its own ``process_weights_after_loading`` body. + """ + from sglang.srt.layers.moe import get_moe_a2a_backend + + if not get_moe_a2a_backend().is_ascend_fuseep(): + return False + from sglang.srt.hardware_backend.npu.moe.fuseep import process_fuseep_weights + + process_fuseep_weights(layer) + return True + + class _NPUFusedMoEMethodBase(FusedMoEMethodBase): def __init__( @@ -392,6 +481,17 @@ def __init__( ): self.quant_config = quant_config + def _maybe_apply_deepep( + self, + layer: torch.nn.Module, + dispatch_output: "DispatchOutput", + ) -> Optional["CombineInput"]: + return maybe_apply_deepep_npu(self, layer, dispatch_output) + + @staticmethod + def _maybe_apply_fuseep_weights(layer: torch.nn.Module) -> bool: + return maybe_apply_fuseep_weights(layer) + class NPUW4A4Int4DynamicMoEMethod(_NPUFusedMoEMethodBase): @@ -444,10 +544,14 @@ def _pack_to_int32(self, weight: torch.Tensor): def apply( self, layer, - dispatch_output: "StandardDispatchOutput", + dispatch_output: "DispatchOutput", ) -> "CombineInput": from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + combine_input = self._maybe_apply_deepep(layer, dispatch_output) + if combine_input is not None: + return combine_input + x = dispatch_output.hidden_states topk_output = dispatch_output.topk_output @@ -512,6 +616,8 @@ def apply_without_routing_weights( class NPUW8A8Int8DynamicMoEMethod(_NPUFusedMoEMethodBase): def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if self._maybe_apply_fuseep_weights(layer): + return layer.w13_weight.data = npu_format_cast(layer.w13_weight.data.transpose(1, 2)) layer.w2_weight.data = npu_format_cast(layer.w2_weight.data.transpose(1, 2)) layer.w13_weight_scale = torch.nn.Parameter( @@ -544,10 +650,14 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: def apply( self, layer, - dispatch_output: "StandardDispatchOutput", + dispatch_output: "DispatchOutput", ) -> "CombineInput": from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + combine_input = self._maybe_apply_deepep(layer, dispatch_output) + if combine_input is not None: + return combine_input + # release fp32 scale to save memory layer.w13_weight_scale = None layer.w2_weight_scale = None @@ -761,10 +871,14 @@ def _process_weights_with_clip(self, layer: torch.nn.Module) -> None: def apply( self, layer, - dispatch_output: "StandardDispatchOutput", + dispatch_output: "DispatchOutput", ) -> "CombineInput": from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + combine_input = self._maybe_apply_deepep(layer, dispatch_output) + if combine_input is not None: + return combine_input + hidden_states = dispatch_output.hidden_states topk_output = dispatch_output.topk_output @@ -1020,10 +1134,14 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: def apply( self, layer, - dispatch_output: "StandardDispatchOutput", + dispatch_output: "DispatchOutput", ) -> "CombineInput": from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + combine_input = self._maybe_apply_deepep(layer, dispatch_output) + if combine_input is not None: + return combine_input + x = dispatch_output.hidden_states topk_output = dispatch_output.topk_output diff --git a/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py b/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py index f1d5bdabd960..bdb52b1c3ab0 100644 --- a/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py +++ b/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py @@ -16,7 +16,10 @@ CHUNK_SIZE = 64 -# This kernel handles K blocks in a for loop to minimize register spills +# This kernel handles K blocks in a for loop to minimize register spills. +# Time is the OUTER loop; K blocks are processed in two inner phases per step: +# Phase 1: store h to output, accumulate v_correction = sum_k(w_k @ h_k^T) +# Phase 2: update h = gate * h + k^T @ v_gated, save to scratch (initial_state) @triton.autotune( configs=[triton.Config({"BV": 64}, num_warps=8, num_stages=2)], key=["H", "K", "V", "BT", "USE_GK", "USE_INITIAL_STATE", "NT_BUCKET"], @@ -110,67 +113,104 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop( if INPLACE_UPDATE: ht = ht + i_h * V * K - # Explicit K loop here to reduce register pressure - for k_start in range(0, K, 64): - # [BV, BK] - b_h1 = tl.zeros([BV, 64], dtype=tl.float32) - - # load initial state - if USE_INITIAL_STATE: - p_h0_1 = tl.make_block_ptr( - h0, (V, K), (K, 1), (i_v * BV, k_start), (BV, 64), (1, 0) - ) - b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32) + # main recurrence — time is the outer loop + for i_t in range(NT): + ######################################################################## + # Phase 1: store h to output, compute v_new = u - sum_k(w_k @ h_k^T) + ######################################################################## + b_v_corr = tl.zeros([BT, BV], dtype=tl.float32) + for k_blk in range(0, K, 64): + # Load h: from initial_state (i_t==0) or scratch (i_t>0) + if i_t == 0: + if USE_INITIAL_STATE: + p_hs = tl.make_block_ptr( + h0, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0) + ) + b_h = tl.load(p_hs, boundary_check=(0, 1)).to(tl.float32) + else: + b_h = tl.zeros([BV, 64], dtype=tl.float32) + else: + p_hs = tl.make_block_ptr( + ht, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0) + ) + b_h = tl.load(p_hs, boundary_check=(0, 1)).to(tl.float32) - # main recurrence - for i_t in range(NT): - p_h1 = tl.make_block_ptr( + # Store pre-update h to output + p_ho = tl.make_block_ptr( h + i_t * stride_h, (V, K), (K, 1), - (i_v * BV, k_start), + (i_v * BV, k_blk), (BV, 64), (1, 0), ) - tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1)) + tl.store(p_ho, b_h.to(p_ho.dtype.element_ty), boundary_check=(0, 1)) - b_w = w_desc.load([i_t * BT, k_start]) - b_v = tl.dot(b_w, tl.trans(b_h1).to(b_w.dtype)) - b_v = v_desc.load([i_t * BT, i_v * BV]) - b_v + # Accumulate correction: w_k @ h_k^T + b_w = w_desc.load([i_t * BT, k_blk]) + b_v_corr += tl.dot(b_w, tl.trans(b_h).to(b_w.dtype)) - if SAVE_NEW_VALUE: - v_new_desc.store([i_t * BT, i_v * BV], b_v.to(v_new.dtype.element_ty)) + # v_new = u - correction + b_v = v_desc.load([i_t * BT, i_v * BV]) - b_v_corr - last_idx = min((i_t + 1) * BT, T) - 1 - if USE_G: - b_g_last = tl.load(g + bos * H + last_idx * H + i_h) - p_g = tl.make_block_ptr( - g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) + if SAVE_NEW_VALUE: + v_new_desc.store([i_t * BT, i_v * BV], b_v.to(v_new.dtype.element_ty)) + + # Apply gate to v + last_idx = min((i_t + 1) * BT, T) - 1 + if USE_G: + b_g_last = tl.load(g + bos * H + last_idx * H + i_h) + p_g = tl.make_block_ptr( + g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) + ) + b_g = tl.load(p_g, boundary_check=(0,)) + b_v = b_v * tl.expand_dims(safe_exp(b_g_last - b_g), 1) + b_g_last = exp(b_g_last) + + b_v = b_v.to(k.dtype.element_ty) + + ######################################################################## + # Phase 2: reload h, apply gate, update h += k^T @ v, save to scratch + ######################################################################## + for k_blk in range(0, K, 64): + # Reload h (same source as Phase 1) + if i_t == 0: + if USE_INITIAL_STATE: + p_hs = tl.make_block_ptr( + h0, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0) + ) + b_h = tl.load(p_hs, boundary_check=(0, 1)).to(tl.float32) + else: + b_h = tl.zeros([BV, 64], dtype=tl.float32) + else: + p_hs = tl.make_block_ptr( + ht, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0) ) - b_g = tl.load(p_g, boundary_check=(0,)) - b_v = b_v * safe_exp(b_g_last - b_g)[:, None] - b_g_last = exp(b_g_last) - b_h1 = b_h1 * b_g_last + b_h = tl.load(p_hs, boundary_check=(0, 1)).to(tl.float32) + + # Gate decay on h + if USE_G: + b_h = b_h * b_g_last if USE_GK: - o_k1 = tl.arange(0, 64) + k_start + o_k1 = tl.arange(0, 64) + k_blk b_gk_last1 = tl.load( gk + (bos + last_idx) * H * K + i_h * K + o_k1, mask=(o_k1 < K), other=0.0, ) - b_h1 *= exp(b_gk_last1)[None, :] - b_v = b_v.to(k.dtype.element_ty) + b_h *= tl.expand_dims(exp(b_gk_last1), 0) - b_k = tl.trans(k_desc.load([i_t * BT, k_start])) - b_h1 += tl.trans(tl.dot(b_k, b_v)) + # Delta update: h += k^T @ v + b_k = tl.trans(k_desc.load([i_t * BT, k_blk])) + b_h += tl.trans(tl.dot(b_k, b_v)) - # epilogue - if INPLACE_UPDATE: - p_ht = tl.make_block_ptr( - ht, (V, K), (K, 1), (i_v * BV, k_start), (BV, 64), (1, 0) - ) - tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + # Save updated h to scratch (initial_state) for next time step + if INPLACE_UPDATE: + p_hs = tl.make_block_ptr( + ht, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0) + ) + tl.store(p_hs, b_h.to(p_hs.dtype.element_ty), boundary_check=(0, 1)) def chunk_gated_delta_rule_fwd_h( diff --git a/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_fwd.py b/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_fwd.py index 3e54712dd9d7..c2299a2f00c6 100644 --- a/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_fwd.py +++ b/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_fwd.py @@ -233,6 +233,16 @@ def chunk_gated_delta_rule_fwd_kkt_solve_kernel_low_reg( ) tl.store(p_Ai_ij, b_Ai_ij.to(A.dtype.element_ty), boundary_check=(0, 1)) + # Clean up scratch slots: Pass 2 stored raw A_ij blocks in the upper-triangular + # part of row i_tc0 (cols BC..3*BC). These must be zeroed because + # recompute_w_u_fwd reads the full BT×BT block. + b_zero = tl.zeros([BC, BC], dtype=tl.float32) + for sc in tl.static_range(1, BT // BC): + p_scratch = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_tc0, sc * BC), (BC, BC), (1, 0) + ) + tl.store(p_scratch, b_zero.to(A.dtype.element_ty), boundary_check=(0, 1)) + def chunk_gated_delta_rule_fwd_intra( k: torch.Tensor, diff --git a/python/sglang/srt/layers/activation.py b/python/sglang/srt/layers/activation.py index 216e37a234ae..a76c454fab75 100644 --- a/python/sglang/srt/layers/activation.py +++ b/python/sglang/srt/layers/activation.py @@ -33,6 +33,7 @@ from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( cpu_has_amx_support, + get_bool_env_var, is_cpu, is_cuda, is_hip, @@ -50,6 +51,7 @@ _is_cpu = is_cpu() _is_hip = is_hip() _is_xpu = is_xpu() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip if _is_cuda: from sglang.jit_kernel.activation import ( @@ -71,6 +73,9 @@ def _(x): return torch.empty(output_shape, dtype=x.dtype, device=x.device) +if _use_aiter: + from aiter import silu_and_mul as _aiter_silu_and_mul + if is_npu(): import torch_npu @@ -82,6 +87,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if get_global_server_args().rl_on_policy_target is not None: self._forward_method = self.forward_native + elif _use_aiter and envs.SGLANG_OPT_USE_AITER_SILU_MUL.get(): + self._forward_method = self.forward_aiter def forward_native(self, x: torch.Tensor) -> torch.Tensor: d = x.shape[-1] // 2 @@ -94,6 +101,13 @@ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: silu_and_mul(x, out) return out + def forward_aiter(self, x: torch.Tensor, limit: float = 0.0) -> torch.Tensor: + d = x.shape[-1] // 2 + output_shape = x.shape[:-1] + (d,) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + _aiter_silu_and_mul(out, x, limit) + return out + def forward_cpu(self, x: torch.Tensor) -> torch.Tensor: if _is_cpu_amx_available: out = torch.ops.sgl_kernel.silu_and_mul_cpu(x) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 897740536321..05ec7b1cd504 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -204,6 +204,11 @@ def __init__( model_runner, self ) + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool + # sliding window attention self.use_sliding_window_kv_pool = ( isinstance(model_runner.token_to_kv_pool, SWAKVPool) @@ -211,7 +216,6 @@ def __init__( ) if self.use_sliding_window_kv_pool: - self.token_to_kv_pool = model_runner.token_to_kv_pool self.use_triton_unified_attention = True else: self.use_triton_unified_attention = get_bool_env_var( @@ -1402,14 +1406,40 @@ def init_cuda_graph_state( max_num_blocks_per_seq = ( self.max_context_len + self.page_size - 1 ) // self.page_size + # Non-unified AITER CUDA graph paths fill this buffer with flat + # token-level kv_indices via create_flashinfer_kv_indices_triton + # (kv_indptr = cumsum(seq_lens)). Even when the allocator is + # page-based, these writes are per-token, so page-sized allocation + # would under-allocate by page_size when page_size > 1. + # TODO(aiter, page_size>1): root fix is to make page_size>1 + # actually engage the attention kernel (`forward_decode` still + # calls paged_attention_ragged with view(-1, 1, ...) and + # block_size=1). That requires a per-page indices kernel + all + # metadata sites + paged_attention_ragged call site + FP8 KV + # coordination, after which this allocation can revert to + # per-page (gated on use_mla). + buffer_numel = max_bs * max_num_blocks_per_seq * self.page_size self.cuda_graph_kv_indices = torch.zeros( - (max_bs * max_num_blocks_per_seq), + (buffer_numel,), dtype=torch.int32, device=self.device, ) else: self.cuda_graph_kv_indices = kv_indices_buf + if self.use_triton_unified_attention: + # Keep a distinct page-table buffer for unified attention. Sharing + # cuda_graph_kv_indices with non-unified token indices makes + # page-table width ambiguous after the token buffer is expanded. + max_num_blocks_per_seq = ( + self.max_context_len + self.page_size - 1 + ) // self.page_size + self.cuda_graph_page_table = torch.zeros( + (max_bs, max_num_blocks_per_seq), + dtype=torch.int32, + device=self.device, + ) + if not self.skip_prefill: self.cuda_graph_custom_mask = torch.zeros( (max_num_tokens * self.max_context_len), @@ -1506,9 +1536,7 @@ def init_forward_metadata_capture_cuda_graph( ) else: max_q_len = 1 - kv_indices = self.cuda_graph_kv_indices.view( - -1, max_num_blocks_per_seq - ) + kv_indices = self.cuda_graph_page_table if self.use_sliding_window_kv_pool: swa_page_table = self.cuda_graph_swa_page_table @@ -1683,9 +1711,7 @@ def init_forward_metadata_capture_cuda_graph( max_num_blocks_per_seq = ( self.max_context_len + self.page_size - 1 ) // self.page_size - page_table = self.cuda_graph_kv_indices.view( - -1, max_num_blocks_per_seq - )[:bs] + page_table = self.cuda_graph_page_table[:bs] swa_page_table = None @@ -1938,9 +1964,7 @@ def init_forward_metadata_replay_cuda_graph( ) else: max_q_len = 1 - kv_indices = self.cuda_graph_kv_indices.view( - -1, max_num_blocks_per_seq - ) + kv_indices = self.cuda_graph_page_table if self.use_sliding_window_kv_pool: swa_page_table = self.cuda_graph_swa_page_table @@ -2117,9 +2141,7 @@ def init_forward_metadata_replay_cuda_graph( max_num_blocks_per_seq = ( self.max_context_len + self.page_size - 1 ) // self.page_size - page_table = self.cuda_graph_kv_indices.view( - -1, max_num_blocks_per_seq - )[:bs] + page_table = self.cuda_graph_page_table[:bs] swa_page_table = None @@ -2355,8 +2377,8 @@ def forward_extend( self.use_triton_unified_attention and self.use_sliding_window_kv_pool ): - token_to_kv_pool = forward_batch.token_to_kv_pool - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer( + token_to_kv_pool = self.token_to_kv_pool + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer( layer.layer_id ) slot_mapping_swa = token_to_kv_pool.full_to_swa_index_mapping @@ -2380,9 +2402,9 @@ def forward_extend( v_scale=v_descale, ) elif self.use_mla: - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, k_descale, v_descale ) @@ -2392,8 +2414,8 @@ def forward_extend( kv_indptr = self.forward_metadata.kv_indptr kv_indices = self.forward_metadata.kv_indices qo_indptr = self.forward_metadata.qo_indptr - K_Buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) - V_Buffer = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + K_Buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id) + V_Buffer = self.token_to_kv_pool.get_value_buffer(layer.layer_id) kv_lora_rank = V_Buffer.shape[-1] qk_rope_head_dim = K_Buffer.shape[-1] - kv_lora_rank qk_nope_head_dim = k.shape[-1] - qk_rope_head_dim @@ -2646,7 +2668,7 @@ def forward_extend( self._use_unified_verify and forward_batch.forward_mode.is_target_verify() ): - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer( + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer( layer.layer_id ) page_table = self.forward_metadata.kv_indices @@ -2705,8 +2727,8 @@ def forward_extend( k.contiguous(), v.contiguous(), o.view(-1, layer.tp_q_head_num, layer.v_head_dim), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), self.forward_metadata.qo_indptr, self.forward_metadata.kv_indptr, self.forward_metadata.kv_indices, @@ -2721,9 +2743,7 @@ def forward_extend( ) return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) bs0 = forward_batch.batch_size + 1 @@ -2798,10 +2818,8 @@ def forward_decode( # use standard set_kv_buffer, as they lack SWA-specific attributes # like full_to_swa_index_mapping. if self.use_triton_unified_attention and self.use_sliding_window_kv_pool: - token_to_kv_pool = forward_batch.token_to_kv_pool - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + token_to_kv_pool = self.token_to_kv_pool + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) slot_mapping_swa = token_to_kv_pool.full_to_swa_index_mapping launch_reshape_and_cache_flash( @@ -2822,7 +2840,7 @@ def forward_decode( # [PATCH] FP8 non-SWA: use launch_reshape_and_cache_flash to # fuse bf16→fp8 cast + paged write in one Triton kernel, # eliminating separate float8_copy + store_kvcache overhead. - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = self.token_to_kv_pool k_cache, v_cache = token_to_kv_pool.get_kv_buffer(layer.layer_id) launch_reshape_and_cache_flash( k.view(-1, layer.tp_k_head_num, layer.qk_head_dim), @@ -2836,12 +2854,12 @@ def forward_decode( forward_batch.out_cache_loc, ) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) if self.use_mla: - k_buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id) work_metadata = self.forward_metadata.work_metadata work_indptr = self.forward_metadata.work_indptr @@ -2878,9 +2896,7 @@ def forward_decode( else: self.logits_soft_cap = layer.logit_cap - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) if layer.qk_head_dim != layer.v_head_dim: o = q.new_empty( @@ -3185,6 +3201,7 @@ def __init__( ) self.device = model_runner.device # Cached variables for generate_draft_decode_kv_indices + self.req_to_token_pool = model_runner.req_to_token_pool self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] self.page_size = model_runner.server_args.page_size @@ -3199,7 +3216,7 @@ def common_template( (self.speculative_num_steps, num_seqs, self.topk) ]( forward_batch.req_pool_indices, - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, diff --git a/python/sglang/srt/layers/attention/cute_utils/__init__.py b/python/sglang/srt/layers/attention/cute_utils/__init__.py new file mode 100644 index 000000000000..c2ad3187e844 --- /dev/null +++ b/python/sglang/srt/layers/attention/cute_utils/__init__.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/cute_utils/__init__.py +from cutlass import BFloat16, Float32, Int64, Uint32, cute +from cutlass._mlir import ir +from cutlass._mlir.dialects import llvm, vector +from cutlass.cute.nvgpu import cpasync +from cutlass.cutlass_dsl import T, dsl_user_op + +# https://github.com/NVIDIA/cutlass/blob/v4.3.2/include/cute/arch/copy_sm90_desc.hpp#L193-L197 +EVICT_NORMAL = Int64(0x1000000000000000) +EVICT_FIRST = Int64(0x12F0000000000000) +EVICT_LAST = Int64(0x14F0000000000000) + + +@dsl_user_op +def recast_val(x, dtype, *, loc=None, ip=None): + return dtype(llvm.bitcast(dtype.mlir_type, x.ir_value(loc=loc, ip=ip))) + + +def simple_tma_copy(atom, src, dst, mbar=None, cache_policy=None): + """A simple helper that wraps group_modes() and tma_partition() + NOTE: this should be called WITHOUT cute.elect_one() + """ + if isinstance(atom.op, cpasync.CopyBulkTensorTileG2SOp): + gmem = src + smem = dst + elif isinstance(atom.op, cpasync.CopyBulkTensorTileS2GOp): + smem = src + gmem = dst + else: + raise ValueError + + s_part, g_part = cpasync.tma_partition( + atom, + 0, + cute.make_layout(1), + cute.group_modes(smem, 0), + cute.group_modes(gmem, 0), + ) + + if isinstance(atom.op, cpasync.CopyBulkTensorTileG2SOp): + cute.copy(atom, g_part, s_part, tma_bar_ptr=mbar, cache_policy=cache_policy) + elif isinstance(atom.op, cpasync.CopyBulkTensorTileS2GOp): + cute.copy(atom, s_part, g_part, cache_policy=cache_policy) + else: + raise ValueError + + +# can't find the equivalent in nvvm +@dsl_user_op +def fence_before_tma_store(*, loc=None, ip=None): + llvm.inline_asm( + T.i32(), + [], + "mov.u32 $0, 0;\n\t" + "fence.proxy.async::generic.release.sync_restrict::shared::cta.cluster;", + "=r", + has_side_effects=True, + is_align_stack=False, + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def mma_bf16( + a: cute.TensorSSA, b: cute.TensorSSA, c: cute.TensorSSA, *, loc=None, ip=None +): + if a.element_type == BFloat16: + a = cute.recast_tensor(a, Uint32) + if b.element_type == BFloat16: + b = cute.recast_tensor(b, Uint32) + + mlir_ty = Float32.mlir_type + out = llvm.inline_asm( + llvm.StructType.get_literal([mlir_ty] * 4), + [a[i].ir_value(loc=loc, ip=ip) for i in range(4)] + + [b[i].ir_value(loc=loc, ip=ip) for i in range(2)] + + [c[i].ir_value(loc=loc, ip=ip) for i in range(4)], + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, " + "{$10, $11, $12, $13};", + "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + has_side_effects=False, + is_align_stack=False, + loc=loc, + ip=ip, + ) + vec = vector.from_elements( + ir.VectorType.get([4], mlir_ty, loc=loc), + [llvm.extractvalue(mlir_ty, out, [i], loc=loc, ip=ip) for i in range(4)], + loc=loc, + ip=ip, + ) + return cute.TensorSSA(vec, 4, Float32) + + +@dsl_user_op +def _bf16x2_abs(a: Uint32, *, loc=None, ip=None) -> Uint32: + out = llvm.inline_asm( + T.i32(), + [a.ir_value(loc=loc, ip=ip)], + "abs.bf16x2 $0, $1;", + "=r,r", + has_side_effects=False, + is_align_stack=False, + ) + return Uint32(out) + + +@dsl_user_op +def _bf16x2_max(a: Uint32, b: Uint32, *, loc=None, ip=None) -> Uint32: + out = llvm.inline_asm( + T.i32(), + [a.ir_value(loc=loc, ip=ip), b.ir_value(loc=loc, ip=ip)], + "max.bf16x2 $0, $1, $2;", + "=r,r,r", + has_side_effects=False, + is_align_stack=False, + ) + return Uint32(out) + + +@dsl_user_op +def _bf16x2_mul(a: Uint32, b: Uint32, *, loc=None, ip=None) -> Uint32: + out = llvm.inline_asm( + T.i32(), + [a.ir_value(loc=loc, ip=ip), b.ir_value(loc=loc, ip=ip)], + "mul.rn.bf16x2 $0, $1, $2;", + "=r,r,r", + has_side_effects=False, + is_align_stack=False, + ) + return Uint32(out) diff --git a/python/sglang/srt/layers/attention/cute_utils/_tcgen05.py b/python/sglang/srt/layers/attention/cute_utils/_tcgen05.py new file mode 100644 index 000000000000..8c0bdd3bc1d7 --- /dev/null +++ b/python/sglang/srt/layers/attention/cute_utils/_tcgen05.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/cute_utils/_tcgen05.py +# this module is named _tcgen05 to avoid name collision with cute.nvgpu.tcgen05 + +import cutlass +from cutlass import Boolean, Float32, Int32, Uint32, Uint64, cute +from cutlass._mlir import ir +from cutlass._mlir.dialects import llvm, nvvm, vector +from cutlass.cutlass_dsl import dsl_user_op + +NVVM_CTA_GROUP_MAP = [ + None, + nvvm.Tcgen05GroupKind.CTA_1, + nvvm.Tcgen05GroupKind.CTA_2, +] +LDST_MAP = { + "32x32b": (nvvm.Tcgen05LdStShape.SHAPE_32X32B, 1), + "16x128b": (nvvm.Tcgen05LdStShape.SHAPE_16X128B, 2), + "16x256b": (nvvm.Tcgen05LdStShape.SHAPE_16X256B, 4), +} + + +def _make_tmem_llvm_ptr(addr, *, loc=None, ip=None): + ptr_ty = llvm.PointerType.get(cute.AddressSpace.tmem.value) + val = Int32(addr).ir_value(loc=loc, ip=ip) + return llvm.inttoptr(ptr_ty, val, loc=loc, ip=ip) + + +@dsl_user_op +def alloc( + taddr: cute.Pointer, + cta_group: int = 1, + *, + loc=None, + ip=None, +) -> None: + nvvm.tcgen05_alloc( + taddr.to_llvm_ptr(loc=loc, ip=ip), + Uint32(512).ir_value(loc=loc, ip=ip), + group=NVVM_CTA_GROUP_MAP[cta_group], + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def dealloc(cta_group: int = 1, *, loc=None, ip=None) -> None: + nvvm.tcgen05_dealloc( + _make_tmem_llvm_ptr(0, loc=loc, ip=ip), + Int32(512).ir_value(loc=loc, ip=ip), + group=NVVM_CTA_GROUP_MAP[cta_group], + loc=loc, + ip=ip, + ) + + +def make_bf16_idesc( + MMA_M: int, + MMA_N: int, + *, + negate_A: bool = False, + negate_B: bool = False, + transpose_A: bool = False, + transpose_B: bool = False, +): + idesc = Uint32( + (1 << 4) | (1 << 7) | (1 << 10) | ((MMA_N >> 3) << 17) | ((MMA_M >> 4) << 24) + ) + idesc |= Uint32(negate_A) << 13 + idesc |= Uint32(negate_B) << 14 + idesc |= Uint32(transpose_A) << 15 + idesc |= Uint32(transpose_B) << 16 + return idesc + + +def make_sdesc_128B_swizzle(LBO: int): + SBO = 8 * 128 + return Uint64((LBO >> 4 << 16) | (SBO >> 4 << 32) | (1 << 46) | (2 << 61)) + + +@dsl_user_op +def mma_f16( + d_tmem, + a_desc, + b_desc, + idesc, + enable_input_d, + cta_group: int = 1, + *, + loc=None, + ip=None, +) -> None: + nvvm.tcgen05_mma( + nvvm.Tcgen05MMAKind.F16, + NVVM_CTA_GROUP_MAP[cta_group], + _make_tmem_llvm_ptr(d_tmem, loc=loc, ip=ip), + Uint64(a_desc).ir_value(loc=loc, ip=ip), + Uint64(b_desc).ir_value(loc=loc, ip=ip), + Int32(idesc).ir_value(loc=loc, ip=ip), + Boolean(enable_input_d).ir_value(loc=loc, ip=ip), + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def mma_ts_f16( + d_tmem, + a_tmem, + b_desc, + idesc, + enable_input_d, + cta_group: int = 1, + *, + loc=None, + ip=None, +) -> None: + nvvm.tcgen05_mma( + nvvm.Tcgen05MMAKind.F16, + NVVM_CTA_GROUP_MAP[cta_group], + _make_tmem_llvm_ptr(d_tmem, loc=loc, ip=ip), + _make_tmem_llvm_ptr(a_tmem, loc=loc, ip=ip), + Uint64(b_desc).ir_value(loc=loc, ip=ip), + Int32(idesc).ir_value(loc=loc, ip=ip), + Boolean(enable_input_d).ir_value(loc=loc, ip=ip), + loc=loc, + ip=ip, + ) + + +@dsl_user_op +def commit(mbar, cta_mask=None, cta_group: int = 1, *, loc=None, ip=None): + mbar_llvm = mbar.to_llvm_ptr(loc=loc, ip=ip) + group = NVVM_CTA_GROUP_MAP[cta_group] + if cutlass.const_expr(cta_mask is not None): + nvvm.tcgen05_commit_arrive( + mbar_llvm, + multicast_mask=cta_mask.ir_value(loc=loc, ip=ip), + group=group, + loc=loc, + ip=ip, + ) + else: + nvvm.tcgen05_commit_arrive(mbar_llvm, group=group, loc=loc, ip=ip) + + +@dsl_user_op +def ld(row, col, shape: str, num: int, *, loc=None, ip=None): + nvvm_shape, regs_per_num = LDST_MAP[shape] + num_regs = regs_per_num * num + tmem = (Int32(row) << Int32(16)) | Int32(col) + tmem_ptr = _make_tmem_llvm_ptr(tmem, loc=loc, ip=ip) + + if num_regs == 1: + reg = nvvm.tcgen05_ld(Int32.mlir_type, nvvm_shape, tmem_ptr, loc=loc, ip=ip) + reg_f32 = llvm.bitcast(Float32.mlir_type, reg, loc=loc, ip=ip) + return Float32(reg_f32) + + else: + vec_i32_ty = ir.VectorType.get([num_regs], Int32.mlir_type, loc=loc) + vec_f32_ty = ir.VectorType.get([num_regs], Float32.mlir_type, loc=loc) + regs = nvvm.tcgen05_ld(vec_i32_ty, nvvm_shape, tmem_ptr, loc=loc, ip=ip) + regs_f32 = llvm.bitcast(vec_f32_ty, regs, loc=loc, ip=ip) + return cute.TensorSSA(regs_f32, (num_regs,), Float32) + + +@dsl_user_op +def st(row, col, shape: str, num: int, vals, *, loc=None, ip=None) -> None: + # if input is TensorSSA, convert to Tensor so we can bitcast + if isinstance(vals, cute.TensorSSA): + vals_ = cute.make_rmem_tensor_like(vals) + vals_.store(vals) + vals = vals_ + + # bitcast to Int32 + vals = cute.recast_tensor(vals, Int32) + + nvvm_shape, regs_per_num = LDST_MAP[shape] + num_regs = regs_per_num * num + tmem = (Int32(row) << Int32(16)) | Int32(col) + tmem_ptr = _make_tmem_llvm_ptr(tmem, loc=loc, ip=ip) + + if num_regs == 1: + nvvm.tcgen05_st( + nvvm_shape, + tmem_ptr, + vals[0].ir_value(loc=loc, ip=ip), + loc=loc, + ip=ip, + ) + else: + vec_i32_ty = ir.VectorType.get([num_regs], Int32.mlir_type, loc=loc) + val_vec = vector.from_elements( + vec_i32_ty, + [vals[i].ir_value(loc=loc, ip=ip) for i in range(num_regs)], + loc=loc, + ip=ip, + ) + nvvm.tcgen05_st(nvvm_shape, tmem_ptr, val_vec, loc=loc, ip=ip) + + +@dsl_user_op +def fence_after_thread_sync(*, loc=None, ip=None): + nvvm.tcgen05_fence(nvvm.Tcgen05FenceKind.AFTER_THREAD_SYNC, loc=loc, ip=ip) + + +@dsl_user_op +def fence_before_thread_sync(*, loc=None, ip=None): + nvvm.tcgen05_fence(nvvm.Tcgen05FenceKind.BEFORE_THREAD_SYNC, loc=loc, ip=ip) + + +@dsl_user_op +def wait_ld(*, loc=None, ip=None): + nvvm.tcgen05_wait(nvvm.Tcgen05WaitKind.LOAD, loc=loc, ip=ip) + + +@dsl_user_op +def wait_st(*, loc=None, ip=None): + nvvm.tcgen05_wait(nvvm.Tcgen05WaitKind.STORE, loc=loc, ip=ip) diff --git a/python/sglang/srt/layers/attention/cute_utils/cvt.py b/python/sglang/srt/layers/attention/cute_utils/cvt.py new file mode 100644 index 000000000000..1eccb09a1187 --- /dev/null +++ b/python/sglang/srt/layers/attention/cute_utils/cvt.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/cute_utils/cvt.py +from cutlass import Constexpr, Float32, Uint32, cute +from cutlass._mlir import ir +from cutlass._mlir.dialects import llvm, vector +from cutlass.cutlass_dsl import T, dsl_user_op + + +@dsl_user_op +def fp32x2_to_bf16x2(a: Float32, b: Float32, *, loc=None, ip=None) -> Uint32: + out = llvm.inline_asm( + T.i32(), + [a.ir_value(loc=loc, ip=ip), b.ir_value(loc=loc, ip=ip)], + "cvt.rn.bf16x2.f32 $0, $2, $1;", + "=r,f,f", + has_side_effects=False, + is_align_stack=False, + ) + return Uint32(out) + + +@dsl_user_op +def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]: + if isinstance(data, Uint32): + out = llvm.inline_asm( + llvm.StructType.get_literal([T.f32(), T.f32()]), + [data.ir_value(loc=loc, ip=ip)], + "shl.b32 $0, $2, 16;\n\tand.b32 $1, $2, 0xFFFF0000;", + "=f,=f,r", + has_side_effects=False, + is_align_stack=False, + loc=loc, + ip=ip, + ) + return ( + Float32(llvm.extractvalue(T.f32(), out, [0], loc=loc, ip=ip)), + Float32(llvm.extractvalue(T.f32(), out, [1], loc=loc, ip=ip)), + ) + + elif isinstance(data, (cute.Tensor, cute.TensorSSA)): + # NOTE: the output is always 1D + size = cute.size(data.shape) + out = cute.make_rmem_tensor(size * 2, Float32) + for i in range(size): + out[i * 2], out[i * 2 + 1] = bf16x2_to_fp32x2(data[i]) + return out + + else: + raise ValueError(f"Unsupported type {type(data)}") + + +@dsl_user_op +def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA: + # there is only fp8->fp16 conversion, hence we need to go + # round trip through fp16. + out = llvm.inline_asm( + llvm.StructType.get_literal([T.i32()] * 2), + [x.ir_value(loc=loc, ip=ip)], + "{\n\t" + ".reg .b16 x0, x1;\n\t" + ".reg .b16 t00, t01, t10, t11;\n\t" + "mov.b32 {x0, x1}, $2;\n\t" + "cvt.rn.f16x2.e4m3x2 $0, x0;\n\t" + "cvt.rn.f16x2.e4m3x2 $1, x1;\n\t" + "mov.b32 {t00, t01}, $0;\n\t" + "mov.b32 {t10, t11}, $1;\n\t" + "cvt.rn.bf16.f16 t00, t00;\n\t" + "cvt.rn.bf16.f16 t01, t01;\n\t" + "cvt.rn.bf16.f16 t10, t10;\n\t" + "cvt.rn.bf16.f16 t11, t11;\n\t" + "mov.b32 $0, {t00, t01};\n\t" + "mov.b32 $1, {t10, t11};\n\t" + "}\n", + "=r,=r,r", + has_side_effects=False, + is_align_stack=False, + ) + vec = vector.from_elements( + ir.VectorType.get([2], T.i32(), loc=loc), + [llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)], + loc=loc, + ip=ip, + ) + return cute.TensorSSA(vec, 2, Uint32) + + +@dsl_user_op +def fp32x4_to_fp8x4( + a0: Float32, + a1: Float32, + a2: Float32, + a3: Float32, + *, + loc=None, + ip=None, +) -> Uint32: + # Pack four FP32 values into one b32 of four e4m3 bytes, byte order + # {a0, a1, a2, a3} from low to high address. + out = llvm.inline_asm( + T.i32(), + [ + a0.ir_value(loc=loc, ip=ip), + a1.ir_value(loc=loc, ip=ip), + a2.ir_value(loc=loc, ip=ip), + a3.ir_value(loc=loc, ip=ip), + ], + "{\n\t" + ".reg .b16 t0, t1;\n\t" + "cvt.rn.satfinite.e4m3x2.f32 t0, $2, $1;\n\t" + "cvt.rn.satfinite.e4m3x2.f32 t1, $4, $3;\n\t" + "mov.b32 $0, {t0, t1};\n\t" + "}\n", + "=r,f,f,f,f", + has_side_effects=False, + is_align_stack=False, + ) + return Uint32(out) + + +@dsl_user_op +def fp32x8_to_fp4x8( + vals: cute.Tensor, + offset: Constexpr[int], + *, + loc=None, + ip=None, +) -> Uint32: + # Pack eight scaled FP32 values into four E2M1x2 bytes, returned as one b32. + assert vals.element_type is Float32 + out = llvm.inline_asm( + T.i32(), + [vals[offset + i].ir_value(loc=loc, ip=ip) for i in range(8)], + "{\n\t" + ".reg .b8 x0, x1, x2, x3;\n\t" + "cvt.rn.satfinite.e2m1x2.f32 x0, $2, $1;\n\t" + "cvt.rn.satfinite.e2m1x2.f32 x1, $4, $3;\n\t" + "cvt.rn.satfinite.e2m1x2.f32 x2, $6, $5;\n\t" + "cvt.rn.satfinite.e2m1x2.f32 x3, $8, $7;\n\t" + "mov.b32 $0, {x0, x1, x2, x3};\n\t" + "}\n", + "=r,f,f,f,f,f,f,f,f", + has_side_effects=False, + is_align_stack=False, + ) + return Uint32(out) diff --git a/python/sglang/srt/layers/attention/cutlass_mla_backend.py b/python/sglang/srt/layers/attention/cutlass_mla_backend.py index e81e761bcefd..05641ea1fede 100644 --- a/python/sglang/srt/layers/attention/cutlass_mla_backend.py +++ b/python/sglang/srt/layers/attention/cutlass_mla_backend.py @@ -241,14 +241,14 @@ def forward_decode( assert v is not None if save_kv_cache: if k_rope is not None: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, k_rope, ) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, @@ -269,7 +269,7 @@ def forward_decode( q_nope = q_nope.to(self.q_data_type) q_rope = q_rope.to(self.q_data_type) - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) o = cutlass_mla_decode( q_nope=q_nope, diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index f9f396428557..883e38d288a1 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -53,6 +53,7 @@ ) from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc from sglang.srt.speculative.spec_info import SpecInput from sglang.srt.utils import ceil_align @@ -354,8 +355,10 @@ def __init__( self.page_size = model_runner.page_size assert self.page_size == 256, "the system hardcodes page_size=256" - self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.req_to_token_pool = model_runner.req_to_token_pool self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool + self.hisparse_coordinator = model_runner.hisparse_coordinator + self.req_to_token = model_runner.req_to_token_pool.req_to_token self.MAX_SEQ_LEN_FOR_CAPTURE = self.req_to_token.shape[1] assert isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool) @@ -667,18 +670,29 @@ def init_forward_metadata(self, forward_batch: ForwardBatch) -> None: req_pool_indices = forward_batch.req_pool_indices seq_lens = forward_batch.seq_lens.to(torch.int32) seq_lens_cpu = forward_batch.seq_lens_cpu - assert forward_batch.req_to_token_pool.req_to_token is self.req_to_token + assert self.req_to_token_pool.req_to_token is self.req_to_token assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0 assert seq_lens_cpu is not None max_seq_len = int(seq_lens_cpu.max().item()) if forward_batch.forward_mode.is_decode_or_idle(): + # DSv4 bakes this step's KV write target (c4/c128) into metadata, + # so slice the shared multi-step out_cache_loc now rather than at + # forward time. + out_cache_loc = forward_batch.out_cache_loc + if self.topk > 0 and self.speculative_num_steps > 1: + out_cache_loc = per_step_draft_out_cache_loc( + out_cache_loc, + forward_batch.batch_size, + self.topk, + self.speculative_num_steps, + )[self.speculative_step_id] metadata = self.init_forward_metadata_decode( max_seq_len=max_seq_len, req_pool_indices=req_pool_indices, seq_lens=seq_lens, - out_cache_loc=forward_batch.out_cache_loc, + out_cache_loc=out_cache_loc, ) elif forward_batch.forward_mode.is_target_verify(): metadata = self.init_forward_metadata_target_verify( @@ -960,7 +974,7 @@ def forward( layer_id = layer.layer_id metadata = self.forward_metadata core_attn_metadata = metadata.core_attn_metadata - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = self.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) if isinstance(core_attn_metadata, DSV4AttnMetadata): @@ -1183,7 +1197,6 @@ def __init__( self, model_runner: ModelRunner, topk: int, speculative_num_steps: int ): super().__init__(model_runner) - self.model_runner = model_runner self.topk = topk self.speculative_num_steps = speculative_num_steps self.attn_backends: List[DeepseekV4AttnBackend] = [] diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 9a9a7225a8ab..3e0ee41ab71c 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -20,11 +20,19 @@ from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import AttentionBackend -from sglang.srt.layers.attention.dsv4.compressor import ( - CompressorBackendMixin, - FusedCompressMetadata, - create_paged_compressor_data, -) + +if envs.SGLANG_OPT_USE_COMPRESSOR_V2.get(): + from sglang.srt.layers.attention.dsv4.compressor_v2 import ( + CompressorBackendMixin, + FusedCompressMetadata, + create_paged_compressor_data, + ) +else: + from sglang.srt.layers.attention.dsv4.compressor import ( + CompressorBackendMixin, + FusedCompressMetadata, + create_paged_compressor_data, + ) from sglang.srt.layers.attention.dsv4.indexer import C4IndexerBackendMixin from sglang.srt.layers.attention.dsv4.metadata import ( PagedIndexerMetadata, @@ -348,8 +356,10 @@ def __init__( self.page_size = model_runner.page_size assert self.page_size == 256, "the system hardcodes page_size=256" - self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.req_to_token_pool = model_runner.req_to_token_pool self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool + self.hisparse_coordinator = model_runner.hisparse_coordinator + self.req_to_token = model_runner.req_to_token_pool.req_to_token self.MAX_SEQ_LEN_FOR_CAPTURE = self.req_to_token.shape[1] assert isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool) @@ -661,7 +671,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch) -> None: req_pool_indices = forward_batch.req_pool_indices seq_lens = forward_batch.seq_lens.to(torch.int32) seq_lens_cpu = forward_batch.seq_lens_cpu - assert forward_batch.req_to_token_pool.req_to_token is self.req_to_token + assert self.req_to_token_pool.req_to_token is self.req_to_token assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0 assert seq_lens_cpu is not None @@ -954,7 +964,7 @@ def forward( layer_id = layer.layer_id metadata = self.forward_metadata core_attn_metadata = metadata.core_attn_metadata - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = self.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) if isinstance(core_attn_metadata, DSV4AttnMetadata): @@ -1183,7 +1193,6 @@ def __init__( self, model_runner: ModelRunner, topk: int, speculative_num_steps: int ): super().__init__(model_runner) - self.model_runner = model_runner self.topk = topk self.speculative_num_steps = speculative_num_steps self.attn_backends: List[DeepseekV4HipRadixBackend] = [] diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index 6ffb9719b7b8..d18e4dd695e6 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -12,6 +12,10 @@ can_use_dsa_fused_store, fused_store_index_k_cache, ) +from sglang.srt.compilation.piecewise_context_manager import ( + get_forward_context, + is_in_piecewise_cuda_graph, +) from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa.utils import ( aiter_can_use_preshuffle_paged_mqa, @@ -80,6 +84,11 @@ from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_output from sglang.srt.model_executor.cuda_graph_runner import get_is_capture_mode from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_req_to_token_pool, + get_token_to_kv_pool, +) from sglang.srt.server_args import get_global_server_args _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() @@ -90,6 +99,80 @@ DUAL_STREAM_TOKEN_THRESHOLD = 1024 if _is_cuda else 0 +if _is_cuda: + from sglang.srt.compilation.compilation_config import register_split_op + from sglang.srt.utils.custom_op import register_custom_op + + @register_custom_op(mutates_args=["topk_result"]) + @register_split_op() + def k_cache_and_topk_result( + layer_id: int, + key: torch.Tensor, + q_fp8: torch.Tensor, + weights: torch.Tensor, + topk_result: torch.Tensor, + ) -> None: + assert ( + _is_cuda + ), "Internal error: piecewise CUDA graph is only supported on CUDA" + from sglang.srt.layers.attention.dsa.triton_kernel import act_quant + + forward_batch = get_forward_context().forward_batch + indexer = get_forward_context().dsa_indexers[layer_id] + metadata = get_attn_backend().get_indexer_metadata(layer_id, forward_batch) + + # slice off padding from piecewise CUDA graph + extend_num_tokens = forward_batch.extend_num_tokens + + indexer._store_index_k_cache( + forward_batch=forward_batch, + layer_id=layer_id, + key=key[:extend_num_tokens], + act_quant=act_quant, + out_cache_loc=forward_batch.out_cache_loc[:extend_num_tokens], + ) + indexer._get_topk_ragged( + False, + forward_batch, + layer_id, + q_fp8[:extend_num_tokens], + weights, + metadata, + topk_result, + ) + + def _logits_head_gate_pcg_fake_impl( + x: torch.Tensor, + weight: torch.Tensor, + n_heads_inv_sqrt: float, + softmax_scale: float, + q_scale: torch.Tensor, + ) -> torch.Tensor: + return torch.empty( + (x.shape[0], weight.shape[0], q_scale.shape[-1]), + dtype=torch.float32, + device=x.device, + ) + + @register_custom_op(fake_impl=_logits_head_gate_pcg_fake_impl) + def logits_head_gate_pcg( + x: torch.Tensor, + weight: torch.Tensor, + n_heads_inv_sqrt: float, + softmax_scale: float, + q_scale: torch.Tensor, + ) -> torch.Tensor: + from sglang.srt.layers.deep_gemm_wrapper import entrypoint as deep_gemm_wrapper + + out = torch.empty( + (x.shape[0], weight.shape[0]), dtype=torch.float32, device=x.device + ) + deep_gemm_wrapper.gemm_nt_bf16bf16f32(x, weight, out) + weights = out * n_heads_inv_sqrt + weights = weights.unsqueeze(-1) * q_scale * softmax_scale + return weights + + class BaseIndexerMetadata(ABC): @abstractmethod def get_seqlens_int32(self) -> torch.Tensor: @@ -436,7 +519,8 @@ def _get_k_bf16( def _update_rope_guarded(dst: torch.Tensor, src: torch.Tensor) -> None: # On AMD with in-place RoPE kernels, self-aliasing can occur; # skip write-back when src/dst tensors point to a single memory. - if src.data_ptr() == dst.data_ptr(): + # data_ptr() is not comparable inside torch.compile, so skip the guard there. + if not torch.compiler.is_compiling() and src.data_ptr() == dst.data_ptr(): return dst.copy_(src) @@ -449,9 +533,9 @@ def _get_topk_paged( metadata: BaseIndexerMetadata, ) -> torch.Tensor: if TYPE_CHECKING: - assert isinstance(forward_batch.token_to_kv_pool, DSATokenToKVPool) + assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) - page_size = forward_batch.token_to_kv_pool.page_size + page_size = get_token_to_kv_pool().page_size # NOTE(dark): blocksize = 64 is hardcoded in deep_gemm if _is_hip: if _use_aiter_preshuffle: @@ -471,7 +555,7 @@ def _get_topk_paged( block_tables = metadata.get_page_table_64() max_seq_len = block_tables.shape[1] * page_size - kv_cache_fp8 = forward_batch.token_to_kv_pool.get_index_k_with_scale_buffer( + kv_cache_fp8 = get_token_to_kv_pool().get_index_k_with_scale_buffer( layer_id=layer_id ) @@ -622,13 +706,14 @@ def _get_topk_ragged( q_fp8: torch.Tensor, weights: torch.Tensor, metadata: BaseIndexerMetadata, + topk_result: Optional[torch.Tensor] = None, ) -> torch.Tensor: if TYPE_CHECKING: - assert isinstance(forward_batch.token_to_kv_pool, DSATokenToKVPool) + assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) assert forward_batch.forward_mode.is_extend_without_speculative() - page_size = forward_batch.token_to_kv_pool.page_size + page_size = get_token_to_kv_pool().page_size if _is_hip: if _use_aiter_preshuffle: assert ( @@ -664,9 +749,10 @@ def _get_topk_ragged( device_index = device.index assert device_index is not None, "q_fp8 must be on an indexed CUDA device" - topk_result = torch.full( - (token_nums, self.index_topk), -1, device=device, dtype=torch.int32 - ) + if topk_result is None: + topk_result = torch.full( + (token_nums, self.index_topk), -1, device=device, dtype=torch.int32 + ) if batch_size == 0: return topk_result @@ -675,7 +761,7 @@ def _get_topk_ragged( indexer_seq_lens_cpu = metadata.get_indexer_seq_len_cpu() seq_len_sum = torch.sum(indexer_seq_lens_cpu).item() max_seq_len = torch.max(indexer_seq_lens_cpu).item() - k_fp8, k_scale = forward_batch.token_to_kv_pool.get_index_k_scale_buffer( + k_fp8, k_scale = get_token_to_kv_pool().get_index_k_scale_buffer( layer_id, metadata.get_indexer_seq_len(), block_tables, @@ -850,10 +936,13 @@ def _get_topk_ragged_with_cp( actual_seq_q: int, cp_index: List[Tuple[int, int, int]] = None, ) -> torch.Tensor: + assert ( + not is_in_piecewise_cuda_graph() + ), "DSA context parallel (_get_topk_ragged_with_cp) not supported under piecewise CUDA graph" if TYPE_CHECKING: - assert isinstance(forward_batch.token_to_kv_pool, DSATokenToKVPool) + assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) - page_size = forward_batch.token_to_kv_pool.page_size + page_size = get_token_to_kv_pool().page_size assert page_size == 64, "only support page size 64" assert len(weights.shape) == 3 weights = weights.squeeze(-1) @@ -882,12 +971,12 @@ def _get_topk_ragged_with_cp( end_seq_position += pre_chunk_offset if offset == 0 and batch_idx != 0: offset += forward_batch.extend_seq_lens_cpu[batch_idx - 1] - k_fp8 = forward_batch.token_to_kv_pool.get_index_k_continuous( + k_fp8 = get_token_to_kv_pool().get_index_k_continuous( layer_id, end_seq_position, block_tables[batch_idx], ) - k_scale = forward_batch.token_to_kv_pool.get_index_k_scale_continuous( + k_scale = get_token_to_kv_pool().get_index_k_scale_continuous( layer_id, end_seq_position, block_tables[batch_idx], @@ -943,12 +1032,12 @@ def _get_topk_ragged_with_cp( - forward_batch.extend_seq_lens_cpu[0] + kv_len ) - k_fp8 = forward_batch.token_to_kv_pool.get_index_k_continuous( + k_fp8 = get_token_to_kv_pool().get_index_k_continuous( layer_id, kv_len, block_tables[0], ) - k_scale = forward_batch.token_to_kv_pool.get_index_k_scale_continuous( + k_scale = get_token_to_kv_pool().get_index_k_scale_continuous( layer_id, kv_len, block_tables[0], @@ -996,10 +1085,13 @@ def forward_indexer( topk: int, layer_id: int, ) -> Optional[torch.Tensor]: + assert ( + not is_in_piecewise_cuda_graph() + ), "DSA forward_indexer (non-CUDA loop path) not supported under piecewise CUDA graph" if not _is_npu: from sglang.srt.layers.attention.dsa.tilelang_kernel import fp8_index - page_size = forward_batch.token_to_kv_pool.page_size + page_size = get_token_to_kv_pool().page_size assert page_size == 64, "only support page size 64" assert len(weights.shape) == 3 @@ -1011,7 +1103,7 @@ def forward_indexer( topk_indices_list = [] - block_tables = forward_batch.req_to_token_pool.req_to_token[ + block_tables = get_req_to_token_pool().req_to_token[ forward_batch.req_pool_indices, : ] strided_indices = torch.arange( @@ -1036,12 +1128,12 @@ def forward_indexer( weights_partial = weights[q_len_start:q_len_end] weights_partial = weights_partial.squeeze(-1).unsqueeze(0).contiguous() - k_fp8 = forward_batch.token_to_kv_pool.get_index_k_continuous( + k_fp8 = get_token_to_kv_pool().get_index_k_continuous( layer_id, seq_len, block_tables[i], ) - k_scale = forward_batch.token_to_kv_pool.get_index_k_scale_continuous( + k_scale = get_token_to_kv_pool().get_index_k_scale_continuous( layer_id, seq_len, block_tables[i], @@ -1078,33 +1170,38 @@ def _store_index_k_cache( key: torch.Tensor, *, act_quant=None, # fallback only + out_cache_loc: Optional[torch.Tensor] = None, ) -> None: """ Store DSA indexer K cache for current step. Preferred: fused_store_index_k_cache(key, cache, out_cache_loc, page_size) Fallback : act_quant(key) + token_to_kv_pool.set_index_k_scale_buffer(...) + + out_cache_loc will default to forward_batch.out_cache_loc if not provided. """ - # Fast path: JIT fused store (CUDA, page_size=64, non-fnuz) + if out_cache_loc is None: + out_cache_loc = forward_batch.out_cache_loc + if ( _is_cuda and (not _is_fp8_fnuz) and can_use_dsa_fused_store( key.dtype, - forward_batch.out_cache_loc.dtype, - forward_batch.token_to_kv_pool.page_size, + out_cache_loc.dtype, + get_token_to_kv_pool().page_size, ) ): # NOTE: wrapper already normalizes shape/contiguity and asserts dtypes. - buf = forward_batch.token_to_kv_pool.get_index_k_with_scale_buffer( + buf = get_token_to_kv_pool().get_index_k_with_scale_buffer( layer_id=layer_id ) fused_store_index_k_cache( key, buf, - forward_batch.out_cache_loc, - forward_batch.token_to_kv_pool.page_size, + out_cache_loc, + get_token_to_kv_pool().page_size, ) return @@ -1114,8 +1211,8 @@ def _store_index_k_cache( # layout with page_size=1; the same kv_cache.view works for both cases # because page_size is 1 there. if _use_aiter: - page_size = forward_batch.token_to_kv_pool.page_size - buf = forward_batch.token_to_kv_pool.get_index_k_with_scale_buffer( + page_size = get_token_to_kv_pool().page_size + buf = get_token_to_kv_pool().get_index_k_with_scale_buffer( layer_id=layer_id ) kv_cache = buf.view(-1, page_size, 132).view(fp8_dtype) @@ -1136,13 +1233,12 @@ def _store_index_k_cache( assert act_quant is not None k_fp8, k_scale = act_quant(key, self.block_size, self.scale_fmt) - out_loc = forward_batch.out_cache_loc - if not out_loc.is_contiguous(): - out_loc = out_loc.contiguous() + if not out_cache_loc.is_contiguous(): + out_cache_loc = out_cache_loc.contiguous() - forward_batch.token_to_kv_pool.set_index_k_scale_buffer( + get_token_to_kv_pool().set_index_k_scale_buffer( layer_id=layer_id, - loc=out_loc, + loc=out_cache_loc, index_k=k_fp8, index_k_scale=k_scale, ) @@ -1175,15 +1271,21 @@ def forward_cuda( from sglang.srt.layers.attention.dsa.triton_kernel import act_quant if TYPE_CHECKING: - assert isinstance(forward_batch.token_to_kv_pool, DSATokenToKVPool) + assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) # When upstream uses fused FP8 RMSNorm+quant, activations may be passed as # a tuple like (x_fp8, x_scale[, y]). Use `x_meta` for shape/device queries. x_meta = x[0] if isinstance(x, tuple) else x - metadata = forward_batch.attn_backend.get_indexer_metadata( - layer_id, forward_batch - ) + # In piecewise CUDA graph mode, metadata is fetched inside custom ops via get_forward_context() to + # prevent Dynamo from guarding on forward_metadata identity (which changes each + # replay when init_forward_metadata creates a new ForwardMetadata object). + if not is_in_piecewise_cuda_graph(): + metadata = get_attn_backend().get_indexer_metadata(layer_id, forward_batch) + if metadata is None: + return None + else: + metadata = None enable_dual_stream = ( self.alt_stream is not None @@ -1192,14 +1294,13 @@ def forward_cuda( and q_lora.shape[0] <= DUAL_STREAM_TOKEN_THRESHOLD ) - # skip DSA if attention backend choose to skip this batch - if metadata is None: - return None - # Determine if should skip topk based on sequence length # We can only skip the logits computation if cuda graph is not involved skip_logits_computation = False - if forward_batch.forward_mode.is_extend_without_speculative(): + if ( + not is_in_piecewise_cuda_graph() + and forward_batch.forward_mode.is_extend_without_speculative() + ): if forward_batch.seq_lens_cpu is not None: max_kv_len = forward_batch.seq_lens_cpu.max().item() skip_logits_computation = max_kv_len <= self.index_topk @@ -1255,7 +1356,7 @@ def forward_cuda( act_quant=act_quant, ) current_stream.wait_stream(self.alt_stream) - else: + elif not is_in_piecewise_cuda_graph(): q_fp8, q_scale = act_quant(query, self.block_size, self.scale_fmt) self._store_index_k_cache( forward_batch=forward_batch, @@ -1263,6 +1364,10 @@ def forward_cuda( key=key, act_quant=act_quant, ) + else: + # piecewise CUDA graph need to split graph on store_k_cache and mqa_logits, + # so delay store_k_cache after weights proj. + q_fp8, q_scale = act_quant(query, self.block_size, self.scale_fmt) # aiter (ROCm gfx95): the 3-tuple (fp8, scale, bf16) from # fused_rms_fp8_group_quant is passed directly to _get_logits_head_gate, @@ -1305,25 +1410,37 @@ def forward_cuda( else: x_for_gate = x - weights = self._get_logits_head_gate(x_for_gate, q_scale) + if is_in_piecewise_cuda_graph(): + weights = logits_head_gate_pcg( + x_for_gate, + self.weights_proj.weight, + self.n_heads**-0.5, + self.softmax_scale, + q_scale, + ) + else: + weights = self._get_logits_head_gate(x_for_gate, q_scale) if _is_cuda or _is_hip: - assert forward_batch.seq_lens_cpu is not None - if len(forward_batch.seq_lens_cpu) == 0: - # this seems b/c max-pad, no worries? - # if x.shape[0] != 0: - # print( - # "HACK: seq_lens empty but x not empty, hackily return all-invalid topk_result" - # ) - return maybe_capture_indexer_topk( - layer_id, - torch.full( - (x_meta.shape[0], self.index_topk), - -1, - dtype=torch.int, - device=x_meta.device, - ), - ) + # In piecewise CUDA graph, any access to seq_lens_cpu creates a Dynamo shape guard. + # Piecewise CUDA graph never has empty batches. + if not is_in_piecewise_cuda_graph(): + assert forward_batch.seq_lens_cpu is not None + if len(forward_batch.seq_lens_cpu) == 0: + # this seems b/c max-pad, no worries? + # if x.shape[0] != 0: + # print( + # "HACK: seq_lens empty but x not empty, hackily return all-invalid topk_result" + # ) + return maybe_capture_indexer_topk( + layer_id, + torch.full( + (x_meta.shape[0], self.index_topk), + -1, + dtype=torch.int, + device=x_meta.device, + ), + ) if ( forward_batch.forward_mode.is_decode_or_idle() @@ -1376,6 +1493,24 @@ def forward_cuda( layer_id, torch.cat([topk_result_prev, topk_result_next], dim=0), ) + elif is_in_piecewise_cuda_graph(): + assert ( + not enable_dual_stream + ), "Internal error: piecewise CUDA graph should not be enabled with dual stream" + + topk_result = torch.full( + (q_fp8.shape[0], self.index_topk), + -1, + device=q_fp8.device, + dtype=torch.int32, + ) + k_cache_and_topk_result( + layer_id=layer_id, + key=key, + q_fp8=q_fp8, + weights=weights, + topk_result=topk_result, + ) else: topk_result = self._get_topk_ragged( enable_dual_stream, @@ -1405,12 +1540,10 @@ def forward_npu( layer_scatter_modes=None, dynamic_scale: torch.Tensor = None, ) -> torch.Tensor: - if forward_batch.attn_backend.forward_metadata.seq_lens_cpu_int is None: - actual_seq_lengths_kv = forward_batch.attn_backend.forward_metadata.seq_lens + if get_attn_backend().forward_metadata.seq_lens_cpu_int is None: + actual_seq_lengths_kv = get_attn_backend().forward_metadata.seq_lens else: - actual_seq_lengths_kv = ( - forward_batch.attn_backend.forward_metadata.seq_lens_cpu_int - ) + actual_seq_lengths_kv = get_attn_backend().forward_metadata.seq_lens_cpu_int is_prefill = ( forward_batch.forward_mode.is_extend() and not forward_batch.forward_mode.is_draft_extend_v2() @@ -1558,7 +1691,7 @@ def forward_npu( torch.npu.current_stream(), ) - forward_batch.token_to_kv_pool.set_index_k_buffer( + get_token_to_kv_pool().set_index_k_buffer( layer_id, forward_batch.out_cache_loc, k ) if is_prefill: @@ -1566,7 +1699,7 @@ def forward_npu( self.dsa_enable_prefill_cp and forward_batch.attn_cp_metadata is not None ): - forward_batch.attn_backend.forward_metadata.actual_seq_lengths_q = ( + get_attn_backend().forward_metadata.actual_seq_lengths_q = ( forward_batch.attn_cp_metadata.actual_seq_q_prev_tensor, forward_batch.attn_cp_metadata.actual_seq_q_next_tensor, ) @@ -1579,34 +1712,32 @@ def forward_npu( forward_batch.attn_cp_metadata.kv_len_next_tensor + forward_batch.extend_prefix_lens.squeeze() ) - forward_batch.attn_backend.forward_metadata.actual_seq_lengths_kv = ( + get_attn_backend().forward_metadata.actual_seq_lengths_kv = ( total_kv_len_prev_tensor, total_kv_len_next_tensor, ) else: - forward_batch.attn_backend.forward_metadata.actual_seq_lengths_kv = ( + get_attn_backend().forward_metadata.actual_seq_lengths_kv = ( forward_batch.attn_cp_metadata.kv_len_prev_tensor, forward_batch.attn_cp_metadata.kv_len_next_tensor, ) actual_seq_lengths_q = ( - forward_batch.attn_backend.forward_metadata.actual_seq_lengths_q + get_attn_backend().forward_metadata.actual_seq_lengths_q ) actual_seq_lengths_kv = ( - forward_batch.attn_backend.forward_metadata.actual_seq_lengths_kv + get_attn_backend().forward_metadata.actual_seq_lengths_kv ) else: actual_seq_lengths_kv = forward_batch.seq_lens actual_seq_lengths_q = forward_batch.extend_seq_lens.cumsum(dim=0) else: - if forward_batch.attn_backend.forward_metadata.actual_seq_lengths_q is None: + if get_attn_backend().forward_metadata.actual_seq_lengths_q is None: if ( forward_batch.forward_mode.is_draft_extend_v2() or forward_batch.forward_mode.is_target_verify() or forward_batch.forward_mode.is_draft_extend() ): - num_draft_tokens = ( - forward_batch.attn_backend.speculative_num_draft_tokens - ) + num_draft_tokens = get_attn_backend().speculative_num_draft_tokens actual_seq_lengths_q = torch.arange( num_draft_tokens, num_draft_tokens + bs, @@ -1622,10 +1753,10 @@ def forward_npu( ) else: actual_seq_lengths_q = ( - forward_batch.attn_backend.forward_metadata.actual_seq_lengths_q + get_attn_backend().forward_metadata.actual_seq_lengths_q ) - past_key_states = forward_batch.token_to_kv_pool.get_index_k_buffer(layer_id) + past_key_states = get_token_to_kv_pool().get_index_k_buffer(layer_id) if self.rotary_emb.is_neox_style and self.alt_stream is not None: torch.npu.current_stream().wait_event(q_rope_event) @@ -1637,7 +1768,7 @@ def forward_npu( and layer_scatter_modes.attn_mode == ScatterMode.TP_ATTN_FULL ): weights = scattered_to_tp_attn_full(weights, forward_batch) - block_table = forward_batch.attn_backend.forward_metadata.block_tables + block_table = get_attn_backend().forward_metadata.block_tables if ( is_prefill and self.dsa_enable_prefill_cp diff --git a/python/sglang/srt/layers/attention/dsa/dsa_topk_backend.py b/python/sglang/srt/layers/attention/dsa/dsa_topk_backend.py new file mode 100644 index 000000000000..8b76557e26a5 --- /dev/null +++ b/python/sglang/srt/layers/attention/dsa/dsa_topk_backend.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +from enum import Enum, IntEnum, auto +from typing import Callable, Dict, List, Optional, Tuple + +import torch + +from sglang.srt.environ import envs + +_FLASHINFER_TIE_BREAK_VALUES = { + "small": 1, + "large": 2, +} + + +class TopkTransformMethod(IntEnum): + # Transform topk indices to indices to the page table (page_size = 1) + PAGED = auto() + # Transform topk indices to indices to ragged kv (non-paged) + RAGGED = auto() + + +class DSATopKBackend(Enum): + SGL_KERNEL = "sgl-kernel" + TORCH = "torch" + FLASHINFER = "flashinfer" + + def is_sgl_kernel(self) -> bool: + return self == DSATopKBackend.SGL_KERNEL + + def is_torch(self) -> bool: + return self == DSATopKBackend.TORCH + + def is_flashinfer(self) -> bool: + return self == DSATopKBackend.FLASHINFER + + def topk_func( + self, + score: torch.Tensor, + lengths: torch.Tensor, + topk: int, + row_starts: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.is_sgl_kernel(): + from sgl_kernel import fast_topk_v2 + + return fast_topk_v2(score, lengths, topk, row_starts=row_starts) + if self.is_torch(): + return _topk_unfused( + score, + lengths, + topk, + row_starts=row_starts, + topk_op=torch.topk, + topk_op_kwargs={"dim": -1}, + ) + if self.is_flashinfer(): + import flashinfer + + return _topk_unfused( + score, + lengths, + topk, + row_starts=row_starts, + topk_op=flashinfer.top_k, + topk_op_kwargs={ + "sorted": False, + "deterministic": envs.SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTIC.get(), + "tie_break": _flashinfer_tie_break_value(), + "dsa_graph_safe": True, + }, + ) + raise RuntimeError(f"Unsupported {self = }.") + + def topk_transform( + self, + logits: torch.Tensor, + lengths: torch.Tensor, + topk: int, + topk_transform_method: TopkTransformMethod, + attn_metadata, + cu_seqlens_q_topk: Optional[torch.Tensor] = None, + topk_indices_offset: Optional[torch.Tensor] = None, + row_starts: Optional[torch.Tensor] = None, + batch_idx_list: Optional[List[int]] = None, + force_unfused_topk: bool = False, + ) -> torch.Tensor: + if not envs.SGLANG_DSA_FUSE_TOPK.get() or force_unfused_topk: + return self.topk_func(logits, lengths, topk, row_starts=row_starts) + + if self.is_sgl_kernel(): + from sgl_kernel import ( + fast_topk_transform_fused, + fast_topk_transform_ragged_fused, + ) + + if topk_transform_method == TopkTransformMethod.PAGED: + page_table_size_1 = ( + attn_metadata.page_table_1[batch_idx_list] + if batch_idx_list is not None + else attn_metadata.page_table_1 + ) + return fast_topk_transform_fused( + score=logits, + lengths=lengths, + page_table_size_1=page_table_size_1, + cu_seqlens_q=cu_seqlens_q_topk, + topk=topk, + row_starts=row_starts, + ) + if topk_transform_method == TopkTransformMethod.RAGGED: + if topk_indices_offset is None: + raise RuntimeError( + "RAGGED topk_transform requires topk_indices_offset; " + "expected extend-without-speculative metadata." + ) + return fast_topk_transform_ragged_fused( + score=logits, + lengths=lengths, + topk_indices_offset=topk_indices_offset, + topk=topk, + row_starts=row_starts, + ) + raise RuntimeError(f"Unsupported {topk_transform_method = }.") + + if self.is_flashinfer(): + import flashinfer + + if topk_transform_method == TopkTransformMethod.PAGED: + row_to_batch, local_row_starts = _build_flashinfer_paged_args( + attn_metadata=attn_metadata, + row_starts=row_starts, + cu_seqlens_q_topk=cu_seqlens_q_topk, + batch_idx_list=batch_idx_list, + device=logits.device, + num_rows=logits.shape[0], + ) + return flashinfer.top_k_page_table_transform( + logits.contiguous(), + attn_metadata.page_table_1.contiguous(), + lengths.contiguous(), + topk, + row_to_batch=row_to_batch, + deterministic=envs.SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTIC.get(), + tie_break=_flashinfer_tie_break_value(), + dsa_graph_safe=True, + row_starts=local_row_starts, + ) + if topk_transform_method == TopkTransformMethod.RAGGED: + if topk_indices_offset is None: + raise RuntimeError( + "RAGGED topk_transform requires topk_indices_offset; " + "expected extend-without-speculative metadata." + ) + return flashinfer.top_k_ragged_transform( + logits.contiguous(), + topk_indices_offset.contiguous(), + lengths.contiguous(), + topk, + deterministic=envs.SGLANG_DSA_TOPK_FLASHINFER_DETERMINISTIC.get(), + tie_break=_flashinfer_tie_break_value(), + dsa_graph_safe=True, + row_starts=row_starts, + ) + raise RuntimeError(f"Unsupported {topk_transform_method = }.") + + raise RuntimeError(f"Unsupported {self = } for SGLANG_DSA_FUSE_TOPK.") + + +def _topk_unfused( + score: torch.Tensor, + lengths: torch.Tensor, + topk: int, + row_starts: Optional[torch.Tensor] = None, + topk_op: Callable[..., Tuple[torch.Tensor, torch.Tensor]] = torch.topk, + topk_op_kwargs: Optional[Dict[str, object]] = None, +) -> torch.Tensor: + batch_size, max_score_len = score.shape + topk_indices = score.new_full((batch_size, topk), -1, dtype=torch.int32) + if batch_size == 0 or topk == 0 or max_score_len == 0: + return topk_indices + + if row_starts is None: + row_starts = torch.zeros_like(lengths, dtype=torch.int32, device=score.device) + else: + row_starts = row_starts.to(dtype=torch.int32, device=score.device) + lengths = lengths.to(dtype=torch.int32, device=score.device) + + col_indices = torch.arange(max_score_len, dtype=torch.int32, device=score.device) + col_indices = col_indices.unsqueeze(0) + row_starts_unsqueezed = row_starts.unsqueeze(1) + row_ends_unsqueezed = (row_starts + lengths).unsqueeze(1) + valid_mask = (col_indices >= row_starts_unsqueezed) & ( + col_indices < row_ends_unsqueezed + ) + + masked_logits = score.masked_fill(~valid_mask, float("-inf")) + valid_topk = min(topk, max_score_len) + topk_kwargs = topk_op_kwargs or {} + topk_scores, topk_col_indices = topk_op(masked_logits, valid_topk, **topk_kwargs) + topk_local_indices = topk_col_indices.to(torch.int32) - row_starts_unsqueezed + topk_local_indices = topk_local_indices.masked_fill( + topk_scores == float("-inf"), -1 + ) + topk_indices[:, :valid_topk] = topk_local_indices + + return topk_indices + + +def _build_flashinfer_paged_args( + attn_metadata, + row_starts: Optional[torch.Tensor], + cu_seqlens_q_topk: Optional[torch.Tensor], + batch_idx_list: Optional[List[int]], + device: torch.device, + num_rows: int, +) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: + row_to_batch = ( + torch.as_tensor(batch_idx_list, dtype=torch.int32, device=device) + if batch_idx_list is not None + else None + ) + + if ( + row_to_batch is not None + and cu_seqlens_q_topk is not None + and row_to_batch.shape[0] != num_rows + ): + q_lens = (cu_seqlens_q_topk[1:] - cu_seqlens_q_topk[:-1]).to( + dtype=torch.int32, device=device + ) + row_to_batch = torch.repeat_interleave(row_to_batch, q_lens) + + if row_to_batch is None and cu_seqlens_q_topk is not None: + # Decode-like case (one query row per batch) does not need an explicit mapping. + # Avoid dynamic tensor construction in this branch to keep CUDA graph capture safe. + num_batches = cu_seqlens_q_topk.shape[0] - 1 + if not (row_starts is None and num_rows == num_batches): + q_lens = (cu_seqlens_q_topk[1:] - cu_seqlens_q_topk[:-1]).to( + dtype=torch.int32, device=device + ) + row_to_batch = torch.repeat_interleave( + torch.arange(q_lens.shape[0], dtype=torch.int32, device=device), + q_lens, + ) + + if row_starts is not None and row_to_batch is None: + raise RuntimeError( + "PAGED topk_transform with row_starts requires cu_seqlens_q metadata." + ) + + local_row_starts = row_starts + if local_row_starts is not None and row_to_batch is not None: + local_row_starts = ( + local_row_starts - attn_metadata.cu_seqlens_k[:-1][row_to_batch] + ) + + return row_to_batch, local_row_starts + + +def _flashinfer_tie_break_value() -> int: + mode = envs.SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK.get() + if mode is None: + return 0 + mode = mode.lower() + if mode not in _FLASHINFER_TIE_BREAK_VALUES: + raise RuntimeError( + "SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK must be one of " + f"{tuple(_FLASHINFER_TIE_BREAK_VALUES)} or unset, got {mode!r}." + ) + return _FLASHINFER_TIE_BREAK_VALUES[mode] diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index a2c062b91f08..92479f3dcb6f 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -1,12 +1,22 @@ from __future__ import annotations +import logging from dataclasses import dataclass -from enum import IntEnum, auto -from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, TypeAlias +from typing import ( + TYPE_CHECKING, + Dict, + List, + Literal, + Optional, + Tuple, + TypeAlias, +) import torch from sglang.srt.configs.model_config import get_dsa_index_topk, is_deepseek_dsa + +logger = logging.getLogger(__name__) from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.dsa.dequant_k_cache import dequantize_k_cache_paged @@ -16,6 +26,10 @@ compute_cu_seqlens, ) from sglang.srt.layers.attention.dsa.dsa_indexer import BaseIndexerMetadata +from sglang.srt.layers.attention.dsa.dsa_topk_backend import ( + DSATopKBackend, + TopkTransformMethod, +) from sglang.srt.layers.attention.dsa.quant_k_cache import quantize_k_cache from sglang.srt.layers.attention.dsa.transform_index import ( transform_index_page_table_decode, @@ -158,13 +172,6 @@ class DSAMetadata: token_to_batch_idx: Optional[torch.Tensor] = None -class TopkTransformMethod(IntEnum): - # Transform topk indices to indices to the page table (page_size = 1) - PAGED = auto() - # Transform topk indices to indices to ragged kv (non-paged) - RAGGED = auto() - - @torch.compile def _compiled_cat(tensors: list[torch.Tensor], dim: int = -1) -> torch.Tensor: return torch.cat(tensors, dim=dim) @@ -190,6 +197,7 @@ def _cat(tensors: list[torch.Tensor], dim: int = -1) -> torch.Tensor: class DSAIndexerMetadata(BaseIndexerMetadata): attn_metadata: DSAMetadata topk_transform_method: TopkTransformMethod + topk_backend: DSATopKBackend = DSATopKBackend.SGL_KERNEL paged_mqa_schedule_metadata: Optional[torch.Tensor] = None force_unfused_topk: bool = False @@ -228,17 +236,11 @@ def topk_transform( logits: torch.Tensor, topk: int, ks: Optional[torch.Tensor] = None, - cu_seqlens_q: torch.Tensor = None, - ke_offset: torch.Tensor = None, - batch_idx_list: List[int] = None, + cu_seqlens_q: Optional[torch.Tensor] = None, + ke_offset: Optional[torch.Tensor] = None, + batch_idx_list: Optional[List[int]] = None, topk_indices_offset_override: Optional[torch.Tensor] = None, ) -> torch.Tensor: - from sgl_kernel import ( - fast_topk_transform_fused, - fast_topk_transform_ragged_fused, - fast_topk_v2, - ) - if topk_indices_offset_override is not None: cu_topk_indices_offset = topk_indices_offset_override cu_seqlens_q_topk = None @@ -256,38 +258,18 @@ def topk_transform( seq_lens_topk = ke_offset else: seq_lens_topk = self.get_seqlens_expanded() - if batch_idx_list is not None: - page_table_size_1 = self.attn_metadata.page_table_1[batch_idx_list] - else: - page_table_size_1 = self.attn_metadata.page_table_1 - - if not envs.SGLANG_DSA_FUSE_TOPK.get() or self.force_unfused_topk: - return fast_topk_v2(logits, seq_lens_topk, topk, row_starts=ks) - elif self.topk_transform_method == TopkTransformMethod.PAGED: - # NOTE(dark): if fused, we return a transformed page table directly - return fast_topk_transform_fused( - score=logits, - lengths=seq_lens_topk, - page_table_size_1=page_table_size_1, - cu_seqlens_q=cu_seqlens_q_topk, - topk=topk, - row_starts=ks, - ) - elif self.topk_transform_method == TopkTransformMethod.RAGGED: - if cu_topk_indices_offset is None: - raise RuntimeError( - "RAGGED topk_transform requires topk_indices_offset; " - "expected extend-without-speculative metadata." - ) - return fast_topk_transform_ragged_fused( - score=logits, - lengths=seq_lens_topk, - topk_indices_offset=cu_topk_indices_offset, - topk=topk, - row_starts=ks, - ) - else: - assert False, f"Unsupported {self.topk_transform_method = }" + return self.topk_backend.topk_transform( + logits=logits, + lengths=seq_lens_topk, + topk=topk, + topk_transform_method=self.topk_transform_method, + attn_metadata=self.attn_metadata, + cu_seqlens_q_topk=cu_seqlens_q_topk, + topk_indices_offset=cu_topk_indices_offset, + row_starts=ks, + batch_idx_list=batch_idx_list, + force_unfused_topk=self.force_unfused_topk, + ) _DSA_IMPL_T: TypeAlias = Literal[ @@ -330,6 +312,9 @@ def __init__( self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim assert model_runner.req_to_token_pool is not None + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool + self.hisparse_coordinator = model_runner.hisparse_coordinator self.req_to_token = model_runner.req_to_token_pool.req_to_token self.use_mha: bool = False @@ -337,6 +322,9 @@ def __init__( model_runner.server_args.dsa_prefill_backend ) self.dsa_decode_impl: _DSA_IMPL_T = model_runner.server_args.dsa_decode_backend + self.dsa_topk_backend: DSATopKBackend = DSATopKBackend( + model_runner.server_args.dsa_topk_backend + ) if self.num_q_heads <= 64: self.flashmla_kv_num_q_heads = 64 elif self.num_q_heads <= 128: @@ -392,6 +380,16 @@ def __init__( else: self.workspace_buffer = None + def _get_fused_topk_page_table(self, topk_indices: torch.Tensor) -> torch.Tensor: + if ( + self.dsa_topk_backend.is_sgl_kernel() + or self.dsa_topk_backend.is_flashinfer() + ): + return topk_indices + raise RuntimeError( + f"Unsupported {self.dsa_topk_backend = } for SGLANG_DSA_FUSE_TOPK." + ) + def get_device_int32_arange(self, l: int) -> torch.Tensor: if l > len(self._arange_buf): next_pow_of_2 = 1 << (l - 1).bit_length() @@ -425,7 +423,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): assert forward_batch.seq_lens_cpu is not None max_seqlen_k = int(forward_batch.seq_lens_cpu.max().item() + draft_token_num) # [b, max_seqlen_k] - page_table = forward_batch.req_to_token_pool.req_to_token[ + page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, :max_seqlen_k ] @@ -580,8 +578,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): # Check if MHA FP8 dequantization is needed mha_dequantize_needed = ( - self.use_mha - and forward_batch.token_to_kv_pool.dtype == torch.float8_e4m3fn + self.use_mha and self.token_to_kv_pool.dtype == torch.float8_e4m3fn ) forward_batch.using_mha_one_shot_fp8_dequant = mha_dequantize_needed @@ -606,8 +603,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): # Validate indices when logical tokens exceed physical capacity # This is likely to be triggered by PP with high kv reuse & parallelism kv_cache_capacity = ( - forward_batch.token_to_kv_pool.size - + forward_batch.token_to_kv_pool.page_size + self.token_to_kv_pool.size + self.token_to_kv_pool.page_size ) if forward_batch.seq_lens_sum > kv_cache_capacity: max_idx = page_table_1_flattened.max().item() @@ -1380,7 +1376,7 @@ def forward_extend( if not layer.is_cross_attention else forward_batch.encoder_out_cache_loc ) - forward_batch.token_to_kv_pool.set_mla_kv_buffer( # type: ignore + self.token_to_kv_pool.set_mla_kv_buffer( # type: ignore layer, cache_loc, k, @@ -1405,7 +1401,7 @@ def forward_extend( # Do absorbed multi-latent attention (MLA path) assert q_rope is not None - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) if q_rope is not None: q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) @@ -1427,7 +1423,7 @@ def forward_extend( forward_batch.forward_mode ) if envs.SGLANG_DSA_FUSE_TOPK.get(): - page_table_1 = topk_indices + page_table_1 = self._get_fused_topk_page_table(topk_indices) else: if topk_transform_method == TopkTransformMethod.RAGGED: topk_indices_offset = metadata.topk_indices_offset @@ -1451,11 +1447,9 @@ def forward_extend( ) # todo hisparse: to cover more backends - if forward_batch.hisparse_coordinator is not None: - page_table_1 = ( - forward_batch.token_to_kv_pool.translate_loc_to_hisparse_device( - page_table_1 - ) + if self.hisparse_coordinator is not None: + page_table_1 = self.token_to_kv_pool.translate_loc_to_hisparse_device( + page_table_1 ) if dsa_impl == "tilelang": @@ -1580,7 +1574,7 @@ def forward_decode( if not layer.is_cross_attention else forward_batch.encoder_out_cache_loc ) - forward_batch.token_to_kv_pool.set_mla_kv_buffer( # type: ignore + self.token_to_kv_pool.set_mla_kv_buffer( # type: ignore layer, cache_loc, k, @@ -1588,7 +1582,7 @@ def forward_decode( ) # Do absorbed multi-latent attention - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) if q_rope is not None: q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) q_rope = q_rope.view( @@ -1609,15 +1603,15 @@ def forward_decode( if topk_indices is not None: topk_indices = self._pad_topk_indices(topk_indices, q_nope.shape[0]) - if forward_batch.hisparse_coordinator is not None: - page_table_1 = forward_batch.hisparse_coordinator.swap_in_selected_pages( + if self.hisparse_coordinator is not None: + page_table_1 = self.hisparse_coordinator.swap_in_selected_pages( forward_batch.req_pool_indices, forward_batch.seq_lens, topk_indices, layer.layer_id, ) elif envs.SGLANG_DSA_FUSE_TOPK.get(): - page_table_1 = topk_indices + page_table_1 = self._get_fused_topk_page_table(topk_indices) else: page_table_1 = transform_index_page_table_decode( page_table=metadata.page_table_1, @@ -2105,11 +2099,9 @@ def _forward_trtllm( if not layer.is_cross_attention else forward_batch.encoder_out_cache_loc ) - forward_batch.token_to_kv_pool.set_mla_kv_buffer( - layer, cache_loc, k, k_rope - ) + self.token_to_kv_pool.set_mla_kv_buffer(layer, cache_loc, k, k_rope) - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) kv_cache = k_cache.view(-1, self.real_page_size, self.kv_cache_dim).unsqueeze(1) if merge_query: @@ -2126,7 +2118,7 @@ def _forward_trtllm( topk_indices = self._pad_topk_indices(topk_indices, q.shape[0]) if envs.SGLANG_DSA_FUSE_TOPK.get(): - page_table_1 = topk_indices + page_table_1 = self._get_fused_topk_page_table(topk_indices) elif is_prefill: page_table_1 = transform_index_page_table_prefill( page_table=metadata.page_table_1, @@ -2172,8 +2164,8 @@ def _forward_trtllm( backend="trtllm-gen", skip_softmax_threshold_scale_factor=envs.SGLANG_SKIP_SOFTMAX_DECODE_THRESHOLD_SCALE_FACTOR.get(), ) - # Output: [batch, q_len=1, heads, v_dim] -> [batch, heads, v_dim] - return out.squeeze(1) + + return out def _pad_topk_indices( self, topk_indices: torch.Tensor, num_tokens: int @@ -2204,10 +2196,18 @@ def set_dsa_prefill_impl(self, forward_batch: Optional[ForwardBatch] = None): """ Decide all attention prefill dispatch strategies for this batch. """ + from sglang.srt.compilation.piecewise_context_manager import ( + is_in_piecewise_cuda_graph, + ) from sglang.srt.utils import get_device_sm, is_blackwell # Decide MHA vs MLA - if forward_batch and forward_batch.forward_mode.is_extend_without_speculative(): + if is_in_piecewise_cuda_graph(): + # Can't branch on seq_lens_cpu in PCG, force mha off to guarantee correctness. + self.use_mha = False + elif ( + forward_batch and forward_batch.forward_mode.is_extend_without_speculative() + ): # Check if sequence meets criteria for MHA_ONE_SHOT assert forward_batch.seq_lens_cpu is not None max_kv_len = forward_batch.seq_lens_cpu.max().item() @@ -2221,12 +2221,11 @@ def set_dsa_prefill_impl(self, forward_batch: Optional[ForwardBatch] = None): ) # SM90/SM100 only and max_kv_len <= envs.SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD.get() # Short enough for MHA - and forward_batch.token_to_kv_pool.dtype - in [torch.bfloat16, torch.float8_e4m3fn] + and self.token_to_kv_pool.dtype in [torch.bfloat16, torch.float8_e4m3fn] and sum_seq_lens <= forward_batch.get_max_chunk_capacity() # Fits in chunk and (not is_dsa_enable_prefill_cp()) # CP not enabled - and (forward_batch.hisparse_coordinator is None) + and (self.hisparse_coordinator is None) ) else: self.use_mha = False # Decode/verify always use MLA @@ -2272,7 +2271,7 @@ def get_indexer_metadata( self, layer_id: int, forward_batch: ForwardBatch ) -> DSAIndexerMetadata: force_unfused = ( - forward_batch.hisparse_coordinator is not None + self.hisparse_coordinator is not None and forward_batch.forward_mode.is_decode_or_idle() ) return DSAIndexerMetadata( @@ -2280,6 +2279,7 @@ def get_indexer_metadata( topk_transform_method=self.get_topk_transform_method( forward_batch.forward_mode ), + topk_backend=self.dsa_topk_backend, paged_mqa_schedule_metadata=self.forward_metadata.paged_mqa_schedule_metadata, force_unfused_topk=force_unfused, ) @@ -2311,7 +2311,6 @@ class DeepseekSparseAttnMultiStepBackend: def __init__( self, model_runner: ModelRunner, topk: int, speculative_num_steps: int ): - self.model_runner = model_runner self.topk = topk self.speculative_num_steps = speculative_num_steps self.attn_backends = [] diff --git a/python/sglang/srt/layers/attention/dsv4/compress_hip.py b/python/sglang/srt/layers/attention/dsv4/compress_hip.py index 1c69f7e46811..8c6b7df9bd03 100644 --- a/python/sglang/srt/layers/attention/dsv4/compress_hip.py +++ b/python/sglang/srt/layers/attention/dsv4/compress_hip.py @@ -12,10 +12,20 @@ from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa.dsa_indexer import rotate_activation from sglang.srt.layers.attention.dsv4.compressor import Compressor as _CompressorBase +from sglang.srt.layers.attention.dsv4.fused_compress_triton import ( + fused_ape_pool_norm_rope, +) +from sglang.srt.layers.attention.nsa.nsa_indexer import rotate_activation from sglang.srt.layers.deepseek_v4_rope import ( apply_rotary_emb_triton, fused_norm_rope_inplace_triton, + fused_softmax_pool_triton, ) + +try: + from sglang.srt.layers.deepseek_v4_rope import fused_softmax_pool_triton +except ImportError: + fused_softmax_pool_triton = None from sglang.srt.mem_cache.deepseek_v4_compress_state import ( CompressStatePool, KVAndScore, @@ -23,6 +33,7 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool if TYPE_CHECKING: + from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( DeepseekV4HipRadixBackend, ) @@ -90,33 +101,44 @@ class CompressorHip(_CompressorBase): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.norm = DeepseekRefRMSNorm(self.head_dim, eps=self.norm.variance_epsilon) + self._freqs_cis_real: torch.Tensor | None = None @cached_property def use_fused_compress(self) -> bool: - return False + return envs.SGLANG_OPT_USE_FUSED_COMPRESS.get() @cached_property def use_hip_fused_compress(self) -> bool: return envs.SGLANG_OPT_USE_FUSED_COMPRESS.get() - def _get_states(self, forward_batch: ForwardBatch) -> KVAndScore: - token_to_kv_pool = forward_batch.token_to_kv_pool + @cached_property + def use_fused_compress_triton(self) -> bool: + # The fused Triton kernel only benefits non-overlap (HCA, ratio=128) + # but HCA's K=128 loop is too sequential to outperform batched ops. + # CSA (overlap=True) has a reshape/overlap-transform semantic mismatch. + # Disabled until a tiled kernel for CSA overlap is implemented. + return False + + def _get_states( + self, + forward_batch: ForwardBatch, + attn_backend: AttentionBackend, + ) -> KVAndScore: + token_to_kv_pool = attn_backend.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) if self.is_in_indexer: return token_to_kv_pool.get_indexer_compress_states(self.layer_id) else: return token_to_kv_pool.get_attention_compress_states(self.layer_id) - def _get_state_pool(self, forward_batch: ForwardBatch) -> CompressStatePool: - token_to_kv_pool = forward_batch.token_to_kv_pool + def _get_state_pool(self, attn_backend: AttentionBackend) -> CompressStatePool: + token_to_kv_pool = attn_backend.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) if self.is_in_indexer: ret = token_to_kv_pool.get_indexer_compress_states(self.layer_id) else: ret = token_to_kv_pool.get_attention_compress_states(self.layer_id) - assert isinstance(ret, CompressStatePool) - return ret def overlap_transform(self, tensor: torch.Tensor, fill_value: Any) -> torch.Tensor: @@ -155,18 +177,19 @@ def compress_extend_paged( self, kv_and_scores: KVAndScore, forward_batch: ForwardBatch, + attn_backend: AttentionBackend, ): - backend = forward_batch.attn_backend + backend = attn_backend if TYPE_CHECKING: assert isinstance(backend, DeepseekV4HipRadixBackend) - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = backend.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - state_pool = self._get_state_pool(forward_batch) + state_pool = self._get_state_pool(backend) prefix_lens = forward_batch.extend_prefix_lens_cpu extend_lens = forward_batch.extend_seq_lens_cpu req_pool_indices = forward_batch.req_pool_indices - req_to_token = forward_batch.req_to_token_pool.req_to_token + req_to_token = backend.req_to_token_pool.req_to_token assert not self.forward_mode.is_target_verify() assert extend_lens is not None and prefix_lens is not None @@ -243,15 +266,22 @@ def compress_extend_paged( pt += extend_lens[i] continue - kv_compressed = ( - kv_and_score_to_compress.kv - * kv_and_score_to_compress.score.softmax(dim=1) - ).sum(dim=1) + beg_idx = prefix_lens[i] // self.ratio * self.ratio + end_idx = (prefix_lens[i] + extend_lens[i]) // self.ratio * self.ratio + + if self.use_hip_fused_compress: + kv_compressed = fused_softmax_pool_triton( + kv_and_score_to_compress.kv_score, + kv_and_score_to_compress._item_size, + ) + else: + kv_compressed = ( + kv_and_score_to_compress.kv + * kv_and_score_to_compress.score.softmax(dim=1) + ).sum(dim=1) assert kv_compressed.dtype == torch.float32 - beg_idx = prefix_lens[i] // self.ratio * self.ratio - end_idx = (prefix_lens[i] + extend_lens[i]) // self.ratio * self.ratio freqs_cis = self.freqs_cis[beg_idx : end_idx : self.ratio] assert freqs_cis.size(0) == kv_compressed.size( 0 @@ -289,18 +319,19 @@ def compress_decode_paged( self, kv_and_scores: KVAndScore, forward_batch: ForwardBatch, + attn_backend: AttentionBackend, ): """Paged and cudagraph compatible version of compress_decode""" assert self.ape_converted - state_pool = self._get_state_pool(forward_batch) - token_to_kv_pool = forward_batch.token_to_kv_pool + state_pool = self._get_state_pool(attn_backend) + token_to_kv_pool = attn_backend.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) req_pool_indices = forward_batch.req_pool_indices - req_to_token = forward_batch.req_to_token_pool.req_to_token + req_to_token = attn_backend.req_to_token_pool.req_to_token seq_lens = forward_batch.seq_lens if forward_batch.forward_mode.is_target_verify(): - draft_tokens = forward_batch.attn_backend.speculative_num_draft_tokens + draft_tokens = attn_backend.speculative_num_draft_tokens offsets = torch.arange(1, draft_tokens + 1, device=seq_lens.device) seq_lens_2d = seq_lens[:, None] + offsets[None, :] seq_lens = seq_lens_2d.view(-1) @@ -331,9 +362,39 @@ def compress_decode_paged( kv_and_score_to_compress = state_pool.get_state_by_state_loc( compress_indices_state.view(-1) ).view(-1, self.ratio, self.coff * self.head_dim) + bs = seq_lens.size(0) + + if self.use_fused_compress_triton and not self.overlap: + # Fused path for non-overlap (HCA, ratio=128, coff=1): + # APE + softmax-pool + norm + RoPE in one kernel. + # Overlap (CSA) is excluded because the overlap_transform_decode + # rearranges A/B halves across the coff dimension in a way + # that simple reshape cannot replicate correctly. + raw = kv_and_score_to_compress.kv_score + gathered = raw.reshape(bs, self.ratio, raw.shape[-1]).contiguous() + + comp_positions = (seq_lens - 1) // self.ratio * self.ratio + freqs_real_table = self._get_freqs_cis_real() + freqs_batch = freqs_real_table[comp_positions] + + kv_compressed = fused_ape_pool_norm_rope( + kv_score_gathered=gathered, + ape=self.ape, + rms_weight=self.norm.weight, + rms_eps=self.norm.eps, + freqs_cis_real=freqs_batch, + head_dim=self.head_dim, + rope_head_dim=self.rope_head_dim, + ratio=self.ratio, + overlap=self.overlap, + ) + if self.rotate: + kv_compressed = rotate_activation(kv_compressed) + return kv_compressed + + # Unfused reference path kv_and_score_to_compress.score.add_(self.ape.unsqueeze(0)) - bs = seq_lens.size(0) if self.overlap: kv_and_score_to_compress = kv_and_score_to_compress.view( bs, self.coff * self.ratio, self.coff * self.head_dim @@ -343,17 +404,20 @@ def compress_decode_paged( score=self.overlap_transform_decode(kv_and_score_to_compress.score), ) - self.print_tensor(kv_and_score_to_compress.kv, "kv_to_compress") - self.print_tensor(kv_and_score_to_compress.score, "score_to_compress") - kv_and_score_to_compress = kv_and_score_to_compress.view( bs, self.ratio * self.coff, self.head_dim ) - kv_compressed = ( - kv_and_score_to_compress.kv * kv_and_score_to_compress.score.softmax(dim=1) - ).sum(dim=1) - self.print_tensor(kv_compressed, "kv_before_norm") + if self.use_hip_fused_compress: + kv_compressed = fused_softmax_pool_triton( + kv_and_score_to_compress.kv_score, + kv_and_score_to_compress._item_size, + ) + else: + kv_compressed = ( + kv_and_score_to_compress.kv + * kv_and_score_to_compress.score.softmax(dim=1) + ).sum(dim=1) if self.use_hip_fused_compress: freqs_cis = self._init_freqs_cis_per_decode_step(forward_batch, seq_lens) fused_norm_rope_inplace_triton( @@ -361,28 +425,25 @@ def compress_decode_paged( ) else: kv_compressed = self.norm(kv_compressed) - self.print_tensor(kv_compressed, "kv_after_norm") freqs_cis = self.freqs_cis[(seq_lens - 1) // self.ratio * self.ratio] - self.print_tensor(freqs_cis, "freqs_cis") apply_rotary_emb_triton( kv_compressed[..., -self.rope_head_dim :], freqs_cis ) - self.print_tensor(kv_compressed, "kv_after_rope") if self.rotate: kv_compressed = rotate_activation(kv_compressed) - self.print_tensor(kv_compressed, "compressed_kv_output") return kv_compressed def compress_fused( self, kv_score: torch.Tensor, forward_batch: ForwardBatch, + attn_backend: AttentionBackend, ) -> torch.Tensor: - backend = forward_batch.attn_backend + backend = attn_backend if TYPE_CHECKING: assert isinstance(backend, DeepseekV4HipRadixBackend) - kv_score_buffer = self._get_state_pool(forward_batch) + kv_score_buffer = self._get_state_pool(backend) kv_score_buffer = kv_score_buffer.kv_score_buffer.kv_score return backend.forward_compress( @@ -398,13 +459,33 @@ def compress_fused( is_paged=True, ) + def _get_freqs_cis_real(self) -> torch.Tensor: + """Cache the float32 view of freqs_cis (complex64 -> real interleaved).""" + if self._freqs_cis_real is None: + if self.freqs_cis.is_complex(): + self._freqs_cis_real = ( + torch.view_as_real(self.freqs_cis).flatten(-2).contiguous() + ) + else: + self._freqs_cis_real = self.freqs_cis.contiguous() + return self._freqs_cis_real + def compress_dispatch( self, kv_score: torch.Tensor, forward_batch: ForwardBatch, + attn_backend: AttentionBackend, ) -> torch.Tensor: - if self.use_fused_compress: - return self.compress_fused(kv_score, forward_batch) + if self.use_fused_compress and ( + envs.SGLANG_OPT_DPSK_V4_RADIX.get() + and ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_extend_without_speculative() + ) + ): + return self.compress_fused( + kv_score, forward_batch, attn_backend=attn_backend + ) self.compress_decode = self.compress_decode_paged self.compress_extend = self.compress_extend_paged @@ -420,11 +501,13 @@ def compress_dispatch( result = self.compress_decode( kv_and_scores=kv_and_scores, forward_batch=forward_batch, + attn_backend=attn_backend, ) elif forward_batch.forward_mode.is_extend(): result = self.compress_extend( kv_and_scores=kv_and_scores, forward_batch=forward_batch, + attn_backend=attn_backend, ) else: msg = f"Forward mode {forward_batch.forward_mode} not supported in Compressor." @@ -445,11 +528,17 @@ def _init_freqs_cis_per_decode_step( setattr(forward_batch, attr, decoded) return decoded - def forward(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor: + def forward( + self, + x: torch.Tensor, + forward_batch: ForwardBatch, + attn_backend: AttentionBackend, + ) -> torch.Tensor: if forward_batch.forward_mode.is_idle(): assert x.shape[0] == 0 return x.new_empty(0, self.head_dim) - kv_score = self.compute_kv_score(x, forward_batch) self.forward_mode = forward_batch.forward_mode - return self.compress_dispatch(kv_score, forward_batch) + return self.compress_dispatch( + kv_score, forward_batch, attn_backend=attn_backend + ) diff --git a/python/sglang/srt/layers/attention/dsv4/compressor.py b/python/sglang/srt/layers/attention/dsv4/compressor.py index 092b98e2c1c3..fa326592f6d8 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor.py @@ -31,6 +31,7 @@ from sglang.srt.utils import add_prefix if TYPE_CHECKING: + from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend from sglang.srt.layers.rotary_embedding import RotaryEmbedding from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -56,6 +57,9 @@ def get_paged_compress_metadata(self, compress_ratio: int) -> FusedCompressMetad assert isinstance(metadata, FusedCompressMetadata) return metadata + def _maybe_upgrade_forward_metadata(self) -> None: + pass + def forward_compress( self, *, @@ -90,6 +94,37 @@ def forward_compress( metadata = (forward_batch.req_pool_indices.to(torch.int32), None, plan) indices, extra_data, plan = metadata + if _is_hip: + if not is_paged: + raise NotImplementedError("HIP fused compressor expects paged metadata") + + from sglang.srt.layers.attention.dsv4.fused_compress_triton import ( + hip_compress_forward, + hip_compress_fused_norm_rope_inplace, + ) + + kv_compressed = hip_compress_forward( + kv_score_buffer=kv_score_buffer, + kv_score_input=kv_score_input, + ape=ape, + indices=indices, + plan=plan, + compress_ratio=compress_ratio, + head_dim=head_dim, + extra_data=extra_data, + ) + norm_eps = ( + norm.variance_epsilon if hasattr(norm, "variance_epsilon") else norm.eps + ) + hip_compress_fused_norm_rope_inplace( + kv_compressed, + norm.weight, + norm_eps, + freqs_cis_cache, + plan, + ) + return rotate_activation(kv_compressed) if rotate else kv_compressed + kv_compressed = compress_forward( kv_score_buffer=kv_score_buffer, kv_score_input=kv_score_input, @@ -123,11 +158,11 @@ def forward_core_compressor( # attn_backend.forward(), so Raw -> DSV4Metadata must happen here too # (e.g. 1.6T layer 0 has compress_ratio=128 and needs cX_compress_metadata). self._maybe_upgrade_forward_metadata() - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = self.token_to_kv_pool if TYPE_CHECKING: assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - new_compressed_kv = compressor(x, forward_batch) + new_compressed_kv = compressor(x, forward_batch, attn_backend=self) core_metadata = self.forward_metadata.core_metadata out_loc = ( core_metadata.c4_out_loc @@ -154,11 +189,11 @@ def forward_indexer_compressor( assert is_overlap_compress(compressor.ratio) # PREP_IN_CG lazy upgrade (see forward_core_compressor for rationale). self._maybe_upgrade_forward_metadata() - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = self.token_to_kv_pool if TYPE_CHECKING: assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - new_compressed_kv = compressor(x, forward_batch) + new_compressed_kv = compressor(x, forward_batch, attn_backend=self) if envs.SGLANG_OPT_USE_FUSED_STORE_CACHE.get(): token_to_kv_pool.set_index_k_fused( layer_id=layer_id, @@ -278,6 +313,8 @@ def get_raw_loc(positions: torch.Tensor) -> torch.Tensor: if is_overlap: write_overlap_loc = get_raw_loc(write_positions - compress_ratio) extra_data = write_overlap_loc.view(-1, 1) + elif _is_hip: + extra_data = get_raw_loc(write_positions - compress_ratio) else: extra_data = None plan = CompressorDecodePlan(compress_ratio, seq_lens.to(torch.int32)) @@ -339,20 +376,16 @@ def apply_ape_hotfix(self): ape = torch.cat([ape[0], ape[1]], dim=0) self.ape.data.copy_(ape.view(self.ratio, -1)) - # NOTE: used by v2 compressor backend - def get_state_pool(self, forward_batch: ForwardBatch) -> CompressStatePool: - token_to_kv_pool = forward_batch.token_to_kv_pool + def get_state_pool(self, attn_backend: AttentionBackend) -> CompressStatePool: + token_to_kv_pool = attn_backend.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) if self.is_in_indexer: ret = token_to_kv_pool.get_indexer_compress_states(self.layer_id) else: ret = token_to_kv_pool.get_attention_compress_states(self.layer_id) - assert isinstance(ret, CompressStatePool) - return ret - # NOTE: used by v2 compressor backend def compute_kv_score(self, x: torch.Tensor, forward_batch: ForwardBatch): kv_score = linear_bf16_fp32(x, self.wkv_gate.weight) @@ -366,19 +399,22 @@ def compute_kv_score(self, x: torch.Tensor, forward_batch: ForwardBatch): ) return kv_score - def forward(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor: + def forward( + self, + x: torch.Tensor, + forward_batch: ForwardBatch, + attn_backend: AttentionBackend, + ) -> torch.Tensor: if forward_batch.forward_mode.is_idle(): assert x.shape[0] == 0 return x.new_empty(0, self.head_dim) kv_score = self.compute_kv_score(x, forward_batch) - backend = forward_batch.attn_backend if TYPE_CHECKING: - assert isinstance(backend, DeepseekV4AttnBackend) - kv_score_buffer = self.get_state_pool(forward_batch) - kv_score_buffer = kv_score_buffer.kv_score_buffer.kv_score - return backend.forward_compress( + assert isinstance(attn_backend, DeepseekV4AttnBackend) + kv_score_buffer = self.get_state_pool(attn_backend).kv_score_buffer.kv_score + return attn_backend.forward_compress( kv_score_buffer=kv_score_buffer, kv_score_input=kv_score, ape=self.ape.view(-1, self.head_dim), @@ -392,7 +428,7 @@ def forward(self, x: torch.Tensor, forward_batch: ForwardBatch) -> torch.Tensor: ) -if _is_hip: +if _is_hip and not envs.SGLANG_OPT_USE_COMPRESSOR_V2.get(): from sglang.srt.layers.attention.dsv4.compress_hip import ( # noqa: F811 CompressorHip as Compressor, ) diff --git a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py index 5d6dd1e0d619..41063e3a8c40 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py @@ -10,6 +10,7 @@ compress_forward, compress_norm_rope_store, ) +from sglang.jit_kernel.utils import is_hip_runtime from sglang.srt.environ import envs if TYPE_CHECKING: @@ -24,12 +25,380 @@ # NOTE: alias for backward compatibility FusedCompressMetadata: TypeAlias = CompressMetadata +_is_hip = is_hip_runtime() + +if _is_hip: + import triton + import triton.language as tl + + @triton.jit + def _c128_compress_decode_kernel( + buf_ptr, + input_ptr, + ape_ptr, + out_ptr, + plan_ptr, + buf_stride_slot, + input_stride_b, + ape_stride_r, + out_stride_b, + bs, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + ): + """Fused C128 decode: write to state buffer + online softmax-pool. + + plan_ptr points to int32 view: [bs, 4] where each row is + {seq_len, write_loc, read_page_0, read_page_1}. + """ + bid = tl.program_id(0) + if bid >= bs: + return + + # Parse plan + plan_base = plan_ptr + bid * 4 + seq_len = tl.load(plan_base).to(tl.int32) + write_loc = tl.load(plan_base + 1).to(tl.int32) + read_page_0 = tl.load(plan_base + 2).to(tl.int32) + + d = tl.arange(0, BLOCK_D) + last_dim: tl.constexpr = HEAD_DIM * 2 + + # Step 1: Write kv_score_input to state buffer at write_loc + d_mask_full = d < last_dim + input_val = tl.load( + input_ptr + bid * input_stride_b + d, mask=d_mask_full, other=0.0 + ) + tl.store(buf_ptr + write_loc * buf_stride_slot + d, input_val, mask=d_mask_full) + + # Step 2: Check boundary condition + d_mask_hd = d < HEAD_DIM + if seq_len % COMPRESS_RATIO != 0: + tl.store( + out_ptr + bid * out_stride_b + d, + tl.zeros([BLOCK_D], tl.float32), + mask=d_mask_hd, + ) + return + + # Step 3: Online softmax-pool over 128 slots in the page + page_base = read_page_0 * COMPRESS_RATIO * buf_stride_slot + m_prev = tl.full([BLOCK_D], float("-inf"), tl.float32) + kv_acc = tl.zeros([BLOCK_D], tl.float32) + w_acc = tl.zeros([BLOCK_D], tl.float32) + + for k in tl.static_range(COMPRESS_RATIO): + slot_addr = page_base + k * buf_stride_slot + kv_val = tl.load(buf_ptr + slot_addr + d, mask=d_mask_hd, other=0.0).to( + tl.float32 + ) + sc_val = tl.load( + buf_ptr + slot_addr + HEAD_DIM + d, mask=d_mask_hd, other=0.0 + ).to(tl.float32) + ape_val = tl.load( + ape_ptr + k * ape_stride_r + d, mask=d_mask_hd, other=0.0 + ).to(tl.float32) + score_k = sc_val + ape_val + + m_new = tl.maximum(m_prev, score_k) + exp_old = tl.where(m_prev == float("-inf"), 0.0, tl.exp(m_prev - m_new)) + exp_cur = tl.where(score_k == float("-inf"), 0.0, tl.exp(score_k - m_new)) + kv_acc = kv_acc * exp_old + exp_cur * kv_val + w_acc = w_acc * exp_old + exp_cur + m_prev = m_new + + compressed = kv_acc / w_acc + tl.store(out_ptr + bid * out_stride_b + d, compressed, mask=d_mask_hd) + + @triton.jit + def _c128_compress_prefill_write_kernel( + buf_ptr, + input_ptr, + plan_w_ptr, + buf_stride_slot, + input_stride_b, + num_w, + BLOCK_D: tl.constexpr, + LAST_DIM: tl.constexpr, + ): + """Prefill write phase: scatter kv_score_input tokens into state buffer.""" + wid = tl.program_id(0) + if wid >= num_w: + return + + # WritePlan: {ragged_id(u32), write_loc(i32)} = 8 bytes = 2 int32s + plan_base = plan_w_ptr + wid * 2 + ragged_id = (tl.load(plan_base).to(tl.int32)) & 0xFFFF + write_loc = tl.load(plan_base + 1).to(tl.int32) + + d = tl.arange(0, BLOCK_D) + d_mask = d < LAST_DIM + + if write_loc >= 0: + input_val = tl.load( + input_ptr + ragged_id * input_stride_b + d, mask=d_mask, other=0.0 + ) + tl.store(buf_ptr + write_loc * buf_stride_slot + d, input_val, mask=d_mask) + + @triton.jit + def _c128_compress_prefill_compress_kernel( + buf_ptr, + ape_ptr, + out_ptr, + plan_c_ptr, + buf_stride_slot, + ape_stride_r, + out_stride_b, + num_c, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + ): + """Prefill compress phase: online softmax-pool for each compress plan entry.""" + cid = tl.program_id(0) + if cid >= num_c: + return + + # CompressPlan: {seq_len(u32), ragged_id(u16)|buffer_len(u16), read_page_0(i32), read_page_1(i32)} + plan_base = plan_c_ptr + cid * 4 + read_page_0 = tl.load(plan_base + 2).to(tl.int32) + + d = tl.arange(0, BLOCK_D) + d_mask_hd = d < HEAD_DIM + + if read_page_0 < 0: + tl.store( + out_ptr + cid * out_stride_b + d, + tl.zeros([BLOCK_D], tl.float32), + mask=d_mask_hd, + ) + return + + page_base = read_page_0 * COMPRESS_RATIO * buf_stride_slot + m_prev = tl.full([BLOCK_D], float("-inf"), tl.float32) + kv_acc = tl.zeros([BLOCK_D], tl.float32) + w_acc = tl.zeros([BLOCK_D], tl.float32) + + for k in tl.static_range(COMPRESS_RATIO): + slot_addr = page_base + k * buf_stride_slot + kv_val = tl.load(buf_ptr + slot_addr + d, mask=d_mask_hd, other=0.0).to( + tl.float32 + ) + sc_val = tl.load( + buf_ptr + slot_addr + HEAD_DIM + d, mask=d_mask_hd, other=0.0 + ).to(tl.float32) + ape_val = tl.load( + ape_ptr + k * ape_stride_r + d, mask=d_mask_hd, other=0.0 + ).to(tl.float32) + score_k = sc_val + ape_val + + m_new = tl.maximum(m_prev, score_k) + exp_old = tl.where(m_prev == float("-inf"), 0.0, tl.exp(m_prev - m_new)) + exp_cur = tl.where(score_k == float("-inf"), 0.0, tl.exp(score_k - m_new)) + kv_acc = kv_acc * exp_old + exp_cur * kv_val + w_acc = w_acc * exp_old + exp_cur + m_prev = m_new + + compressed = kv_acc / w_acc + tl.store(out_ptr + cid * out_stride_b + d, compressed, mask=d_mask_hd) + + +def _compress_forward_c128_triton( + kv_score_buffer: torch.Tensor, + kv_score_input: torch.Tensor, + ape: torch.Tensor, + plan: Union[CompressorDecodePlan, CompressorPrefillPlan], + head_dim: int, +) -> torch.Tensor: + """Triton C128 compress_forward for HIP (wave64). + + Fuses write + online-softmax-pool into Triton kernels. + CUDA graph compatible. + """ + num_total_slots = kv_score_buffer.shape[0] * kv_score_buffer.shape[1] + num_pages = kv_score_buffer.shape[0] + last_dim = kv_score_buffer.shape[-1] + compress_ratio = 128 + + buf_flat = kv_score_buffer.view(-1, last_dim) + buf_stride_slot = last_dim # elements per slot + + BLOCK_D = triton.next_power_of_2(last_dim) + + if plan.is_decode: + # Decode path: single kernel does write + compress + plan_raw = plan[1].view(torch.int32) # [bs, 4] + bs = plan_raw.shape[0] + out = torch.empty( + bs, head_dim, dtype=torch.float32, device=kv_score_input.device + ) + + if bs > 0 and num_total_slots > 0: + grid = (bs,) + _c128_compress_decode_kernel[grid]( + buf_flat, + kv_score_input, + ape, + out, + plan_raw, + buf_stride_slot, + kv_score_input.stride(0), + ape.stride(0), + out.stride(0), + bs, + HEAD_DIM=head_dim, + BLOCK_D=triton.next_power_of_2(head_dim), + COMPRESS_RATIO=compress_ratio, + num_warps=8, + ) + return out + else: + # Prefill path: separate write kernel + compress kernel + plan_c_raw = plan[1].view(torch.int32) # [num_c, 4] + plan_w = plan[2] # [num_w, 8] uint8 + plan_w_raw = plan_w.view(torch.int32) # [num_w, 2] + num_c = plan_c_raw.shape[0] + num_w = plan_w_raw.shape[0] + + out = torch.empty( + num_c, head_dim, dtype=torch.float32, device=kv_score_input.device + ) + + # Phase 1: Write + if num_w > 0 and num_total_slots > 0: + grid_w = (num_w,) + _c128_compress_prefill_write_kernel[grid_w]( + buf_flat, + kv_score_input, + plan_w_raw, + buf_stride_slot, + kv_score_input.stride(0), + num_w, + BLOCK_D=BLOCK_D, + LAST_DIM=last_dim, + num_warps=4, + ) + + # Phase 2: Compress + if num_c > 0 and num_pages > 0: + grid_c = (num_c,) + _c128_compress_prefill_compress_kernel[grid_c]( + buf_flat, + ape, + out, + plan_c_raw, + buf_stride_slot, + ape.stride(0), + out.stride(0), + num_c, + HEAD_DIM=head_dim, + BLOCK_D=triton.next_power_of_2(head_dim), + COMPRESS_RATIO=compress_ratio, + num_warps=8, + ) + + return out + def _use_online_compress(compress_ratio: int) -> bool: """Online state-pool path is c128-only.""" return compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() +def _extract_positions_from_plan( + plan: Union[CompressorDecodePlan, CompressorPrefillPlan], + compress_ratio: int, +) -> torch.Tensor: + """Extract RoPE positions from plan tensors (decode or prefill). + + DecodePlan layout: [bs, 16] uint8, first 4 bytes = uint32 seq_len. + CompressPlan layout: [num_c, 16] uint8, first 4 bytes = uint32 seq_len. + Position for RoPE = seq_len - compress_ratio. + """ + plan_tensor = plan[1] # plan_d or plan_c + seq_lens = plan_tensor[:, :4].contiguous().view(torch.int32).squeeze(-1) + positions = seq_lens.to(torch.int32) - compress_ratio + return positions + + +def _compress_forward_c128_fallback( + kv_score_buffer: torch.Tensor, + kv_score_input: torch.Tensor, + ape: torch.Tensor, + plan: Union[CompressorDecodePlan, CompressorPrefillPlan], + head_dim: int, +) -> torch.Tensor: + """PyTorch fallback for C128 compress_forward on HIP (wave64). + + Fully vectorized, compatible with CUDA graph capture. + kv_score_buffer: [num_pages, 128, head_dim * 2] + ape: [128, head_dim] + + IMPORTANT: This also performs the write to state buffer (like the JIT kernel). + The JIT kernel does: (1) write kv_score_input to buffer, (2) compress from buffer. + """ + num_total_slots = kv_score_buffer.shape[0] * kv_score_buffer.shape[1] + num_pages = kv_score_buffer.shape[0] + last_dim = kv_score_buffer.shape[-1] + + # Step 1: WRITE kv_score_input to state buffer + if num_total_slots > 0: + buf_flat = kv_score_buffer.view(-1, last_dim) + if plan.is_decode: + # Decode: plan_d has write_loc per batch item + plan_raw = plan[1].view(torch.int32) # [bs, 4] + write_locs = plan_raw[:, 1].long() + # Only write valid locations (>= 0 and < buffer size) + valid_write = (write_locs >= 0) & (write_locs < num_total_slots) + if valid_write.any(): + buf_flat[write_locs[valid_write]] = kv_score_input[valid_write] + else: + # Prefill: plan_w has {ragged_id, write_loc} per write entry + plan_w = plan[2] # [num_w, 8] uint8 = WritePlan + if plan_w.shape[0] > 0: + plan_w_raw = plan_w.view(torch.int32) # [num_w, 2] + ragged_ids = plan_w_raw[:, 0].long() & 0xFFFF + write_locs = plan_w_raw[:, 1].long() + valid_write = (write_locs >= 0) & (write_locs < num_total_slots) + ragged_ids_safe = ragged_ids.clamp( + min=0, max=kv_score_input.shape[0] - 1 + ) + if valid_write.any(): + buf_flat[write_locs[valid_write]] = kv_score_input[ + ragged_ids_safe[valid_write] + ] + + # Step 2: COMPRESS (read from buffer page and do softmax-pool) + plan_c = plan[1] # plan_d for decode, plan_c for prefill + num_tokens = plan_c.shape[0] + if num_pages == 0 or num_tokens == 0: + return kv_score_input.new_zeros(num_tokens, head_dim) + + plan_c_raw = plan_c.view(torch.int32) # [N, 4] + read_page_0 = plan_c_raw[:, 2].long() + # Use torch.where instead of clamp to handle -1 (invalid) gracefully + valid_read = (read_page_0 >= 0) & (read_page_0 < num_pages) + read_page_0_safe = torch.where( + valid_read, read_page_0, torch.zeros_like(read_page_0) + ) + + gathered = kv_score_buffer[read_page_0_safe] # [N, 128, head_dim*2] + kv = gathered[:, :, :head_dim].float() + score = gathered[:, :, head_dim:].float() + ape.float().unsqueeze(0) + weights = score.softmax(dim=1) + out = (weights * kv).sum(dim=1) + + # For decode: zero out non-boundary tokens (seq_len % 128 != 0) + # so they don't corrupt kvcache location 0 when stored. + if plan.is_decode: + seq_lens = plan_c_raw[:, 0].to(torch.int32) + is_boundary = (seq_lens % 128 == 0).unsqueeze(-1) # [N, 1] + out = torch.where(is_boundary, out, torch.zeros_like(out)) + + return out.to(kv_score_input.dtype) + + class CompressorBackendMixin: def __init__(self): super().__init__() @@ -74,6 +443,8 @@ def _forward_compress_all_in_one( last_dim = 2 * head_dim * coff assert kv_score_buffer.shape[-1] == last_dim kv_score_buffer = kv_score_buffer.view(-1, compress_ratio, last_dim) + + # Step 1: compress_forward kv_compressed = compress_forward( kv_score_buffer=kv_score_buffer, kv_score_input=kv_score_input, @@ -83,7 +454,8 @@ def _forward_compress_all_in_one( head_dim=head_dim, is_online=is_online, ) - # NOTE: we use some hack here... + + # Step 2: norm + rope + store compress_norm_rope_store( kv_compressed, plan, @@ -106,38 +478,157 @@ def forward_unified( return self._maybe_upgrade_forward_metadata() - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = self.token_to_kv_pool token_to_kv_pool = cast("DeepSeekV4TokenToKVPool", token_to_kv_pool) kv_score_input = compressor.compute_kv_score(x, forward_batch) - state_pool = compressor.get_state_pool(forward_batch) - out_loc = self._get_out_loc(compressor.ratio) - if compressor.is_in_indexer: - kv_cache = token_to_kv_pool.get_index_k_with_scale_buffer(layer_id) - page_size = token_to_kv_pool.get_index_k_page_size() + + state_pool = compressor.get_state_pool(self) + if _is_hip and not envs.SGLANG_OPT_USE_JIT_NORM.get(): + self._forward_unified_hip( + token_to_kv_pool=token_to_kv_pool, + kv_score_input=kv_score_input, + state_pool=state_pool, + compressor=compressor, + layer_id=layer_id, + ) else: - _, _, compress_kv_pool = token_to_kv_pool.layer_mapping[layer_id] - assert compress_kv_pool is not None - kv_cache = token_to_kv_pool.get_extra_key_buffer(layer_id) - page_size = token_to_kv_pool.get_extra_key_page_size(layer_id) - if hasattr(compress_kv_pool, "translate_loc_to_hisparse_device"): - # The v2 compressor writes directly into the raw C4 KV tensor. - # HiSparse C4 therefore needs the physical C4 location here. - out_loc = compress_kv_pool.translate_loc_to_hisparse_device(out_loc) - self._forward_compress_all_in_one( - kv_score_buffer=state_pool.kv_score_buffer.kv_score, + out_loc = self._get_out_loc(compressor.ratio) + if compressor.is_in_indexer: + kv_cache = token_to_kv_pool.get_index_k_with_scale_buffer(layer_id) + page_size = token_to_kv_pool.get_index_k_page_size() + else: + _, _, compress_kv_pool = token_to_kv_pool.layer_mapping[layer_id] + assert compress_kv_pool is not None + kv_cache = token_to_kv_pool.get_extra_key_buffer(layer_id) + page_size = token_to_kv_pool.get_extra_key_page_size(layer_id) + if hasattr(compress_kv_pool, "translate_loc_to_hisparse_device"): + # The v2 compressor writes directly into the raw C4 KV tensor. + # HiSparse C4 therefore needs the physical C4 location here. + out_loc = compress_kv_pool.translate_loc_to_hisparse_device(out_loc) + self._forward_compress_all_in_one( + kv_score_buffer=state_pool.kv_score_buffer.kv_score, + kv_score_input=kv_score_input, + ape=compressor.ape, + head_dim=compressor.head_dim, + norm=compressor.norm, + freqs_cis_cache=compressor.freqs_cis, + kv_cache=kv_cache.view(dtype=torch.uint8), + is_indexer=compressor.is_in_indexer, + rotate=compressor.rotate, + compress_ratio=compressor.ratio, + page_size=page_size, + out_loc=out_loc, + ) + + def _forward_unified_hip( + self, + token_to_kv_pool: DeepSeekV4TokenToKVPool, + kv_score_input: torch.Tensor, + state_pool, + compressor: Compressor, + layer_id: int, + ) -> None: + """HIP-specific forward path using PyTorch/Triton fallbacks.""" + from sglang.srt.layers.attention.dsv4.quant_k_cache import ( + quant_to_nope_fp8_rope_bf16_pack_triton, + ) + from sglang.srt.layers.attention.nsa.nsa_indexer import rotate_activation + from sglang.srt.layers.attention.nsa.triton_kernel import act_quant + from sglang.srt.layers.deepseek_v4_rope import fused_norm_rope_inplace_triton + + compress_ratio = compressor.ratio + head_dim = compressor.head_dim + is_indexer = compressor.is_in_indexer + + plan = self._get_paged_compress_metadata(compress_ratio) + out_loc = self._get_out_loc(compress_ratio) + + # Step 1: compress_forward (always use JIT for both C4 and C128) + coff = 2 if is_overlap_compress(compress_ratio) else 1 + last_dim = 2 * head_dim * coff + kv_score_buffer = state_pool.kv_score_buffer.kv_score + kv_score_buffer = kv_score_buffer.view(-1, compress_ratio, last_dim) + + kv_compressed = compress_forward( + kv_score_buffer=kv_score_buffer, kv_score_input=kv_score_input, - ape=compressor.ape, - head_dim=compressor.head_dim, - norm=compressor.norm, - freqs_cis_cache=compressor.freqs_cis, - kv_cache=kv_cache.view(dtype=torch.uint8), - is_indexer=compressor.is_in_indexer, - rotate=compressor.rotate, - compress_ratio=compressor.ratio, - page_size=page_size, - out_loc=out_loc, + ape=compressor.ape.view(-1, head_dim), + plan=plan, + compress_ratio=compress_ratio, + head_dim=head_dim, + is_online=False, + ) + + if kv_compressed.shape[0] == 0: + return + + # For decode: zero out non-boundary tokens to prevent corrupting kvcache loc 0. + if plan.is_decode: + plan_raw = plan[1].view(torch.int32) + seq_lens_plan = plan_raw[:, 0].to(torch.int32) + is_boundary = (seq_lens_plan % compress_ratio == 0).unsqueeze(-1) + kv_compressed = torch.where( + is_boundary, kv_compressed, torch.zeros_like(kv_compressed) + ) + + # Step 2: norm + rope (Triton fallback for precision parity with V1) + positions = _extract_positions_from_plan(plan, compress_ratio) + positions_safe = positions.clamp(min=0) + + fused_norm_rope_inplace_triton( + kv_compressed, + compressor.norm.weight, + compressor.norm.variance_epsilon, + compressor.freqs_cis, + positions=positions_safe, ) + # Step 3: optional Hadamard rotation for indexer + if compressor.rotate: + kv_compressed = rotate_activation(kv_compressed) + + # Step 4: store to kvcache + # For decode: store ALL tokens. Non-boundary tokens have out_loc=0 (safe). + # For prefill: plan_c already only contains valid entries. + if plan.is_decode: + kv_to_store = kv_compressed + out_loc_to_store = out_loc + else: + kv_to_store = kv_compressed + plan_raw = plan[1].view(torch.int32) + ragged_ids = plan_raw[:, 1].to(torch.int32) & 0xFFFF + out_loc_to_store = out_loc[ragged_ids.long()] + + if kv_to_store.shape[0] == 0: + return + + if envs.SGLANG_OPT_USE_FUSED_STORE_CACHE.get(): + # fused kernel: BF16 in -> FP8 quant + paged scatter in one launch + if is_indexer: + token_to_kv_pool.set_index_k_fused( + layer_id=layer_id, + loc=out_loc_to_store, + cache_k=kv_to_store, + ) + else: + token_to_kv_pool.set_extra_key_buffer_fused( + layer_id=layer_id, + loc=out_loc_to_store, + cache_k=kv_to_store, + ) + else: + if is_indexer: + kv_fp8, kv_scale = act_quant(kv_to_store) + token_to_kv_pool.set_index_k_scale_buffer( + layer_id=layer_id, + loc=out_loc_to_store, + index_k=kv_fp8, + index_k_scale=kv_scale, + ) + else: + pack = quant_to_nope_fp8_rope_bf16_pack_triton(kv_to_store.bfloat16()) + token_to_kv_pool.set_extra_key_buffer(layer_id, out_loc_to_store, pack) + # NOTE: alias for backward compatibility forward_indexer_compressor = forward_unified forward_core_compressor = forward_unified diff --git a/python/sglang/srt/layers/attention/dsv4/fused_compress_triton.py b/python/sglang/srt/layers/attention/dsv4/fused_compress_triton.py new file mode 100644 index 000000000000..9434556c41bc --- /dev/null +++ b/python/sglang/srt/layers/attention/dsv4/fused_compress_triton.py @@ -0,0 +1,954 @@ +"""HIP fused compressor kernels using the NV/main metadata contract. + +The public wrappers mirror ``compress_forward``: + + decode: indices, seq_lens, extra_data + prefill: indices, compress_plan, write_plan, extra_data + +Prefill plans are the upstream 16-byte ``PrefillPlan`` structs stored as +``uint8[:, 16]``. The wrappers reinterpret them as ``int32[:, 4]`` before +launching Triton kernels. +""" + +from __future__ import annotations + +from typing import Optional, Union + +import torch +import triton +import triton.language as tl + +from sglang.jit_kernel.dsv4.compress_old import ( + CompressorDecodePlan, + CompressorPrefillPlan, +) + + +@triton.jit +def _fused_ape_pool_norm_rope_kernel( + kv_score_ptr, + kv_score_stride_b, + kv_score_stride_k, + ape_ptr, + ape_stride_r, + rms_weight_ptr, + rms_eps, + freqs_ptr, + freqs_stride_b, + out_ptr, + out_stride_b, + head_dim, + rope_head_dim, + half_dim, + RATIO: tl.constexpr, + K_POOL: tl.constexpr, + BLOCK_D: tl.constexpr, + HALF_ROPE: tl.constexpr, + OVERLAP: tl.constexpr, +): + bid = tl.program_id(0) + d = tl.arange(0, BLOCK_D) + d_mask = d < head_dim + + m_prev = tl.full([BLOCK_D], float("-inf"), tl.float32) + kv_acc = tl.zeros([BLOCK_D], tl.float32) + w_acc = tl.zeros([BLOCK_D], tl.float32) + + batch_base = bid * kv_score_stride_b + + for k in tl.range(0, K_POOL): + if OVERLAP: + is_b = k >= RATIO + col_off = tl.where(is_b, head_dim, 0) + else: + col_off = 0 + + row_off = batch_base + k * kv_score_stride_k + kv_val = tl.load( + kv_score_ptr + row_off + col_off + d, mask=d_mask, other=0.0 + ).to(tl.float32) + sc_val = tl.load( + kv_score_ptr + row_off + half_dim + col_off + d, mask=d_mask, other=0.0 + ).to(tl.float32) + + ape_val = tl.load( + ape_ptr + (k % RATIO) * ape_stride_r + col_off + d, mask=d_mask, other=0.0 + ).to(tl.float32) + score_k = sc_val + ape_val + + m_new = tl.maximum(m_prev, score_k) + exp_old = tl.where(m_prev == float("-inf"), 0.0, tl.exp(m_prev - m_new)) + exp_cur = tl.where(score_k == float("-inf"), 0.0, tl.exp(score_k - m_new)) + kv_acc = kv_acc * exp_old + exp_cur * kv_val + w_acc = w_acc * exp_old + exp_cur + m_prev = m_new + + compressed = kv_acc / w_acc + rms_w = tl.load(rms_weight_ptr + d, mask=d_mask, other=0.0) + c_sq = tl.where(d_mask, compressed * compressed, 0.0) + var = tl.sum(c_sq, axis=0) / head_dim + normed = compressed * tl.rsqrt(var + rms_eps) * rms_w + + out_base = out_ptr + bid * out_stride_b + tl.store(out_base + d, normed.to(out_ptr.dtype.element_ty), mask=d_mask) + + rope_start = head_dim - rope_head_dim + p = tl.arange(0, HALF_ROPE) + pmask = p < (rope_head_dim // 2) + xr = tl.load(out_base + rope_start + 2 * p, mask=pmask, other=0.0).to(tl.float32) + xi = tl.load(out_base + rope_start + 2 * p + 1, mask=pmask, other=0.0).to( + tl.float32 + ) + + freq_base = bid * freqs_stride_b + fr = tl.load(freqs_ptr + freq_base + 2 * p, mask=pmask, other=1.0).to(tl.float32) + fi = tl.load(freqs_ptr + freq_base + 2 * p + 1, mask=pmask, other=0.0).to( + tl.float32 + ) + + tl.store( + out_base + rope_start + 2 * p, + (xr * fr - xi * fi).to(out_ptr.dtype.element_ty), + mask=pmask, + ) + tl.store( + out_base + rope_start + 2 * p + 1, + (xr * fi + xi * fr).to(out_ptr.dtype.element_ty), + mask=pmask, + ) + + +def fused_ape_pool_norm_rope( + kv_score_gathered: torch.Tensor, + ape: torch.Tensor, + rms_weight: torch.Tensor, + rms_eps: float, + freqs_cis_real: torch.Tensor, + head_dim: int, + rope_head_dim: int, + ratio: int, + overlap: bool, +) -> torch.Tensor: + """Fused APE-add + overlap-transform + softmax-pool + RMSNorm + RoPE.""" + coff = 2 if overlap else 1 + bs = kv_score_gathered.shape[0] + k_in = kv_score_gathered.shape[1] + last_dim = kv_score_gathered.shape[2] + half_dim = last_dim // 2 + assert k_in == ratio * coff, f"k_in={k_in} != ratio*coff={ratio}*{coff}" + + out = torch.empty( + bs, head_dim, dtype=torch.float32, device=kv_score_gathered.device + ) + if bs == 0: + return out + + block_d = triton.next_power_of_2(head_dim) + half_rope = triton.next_power_of_2(rope_head_dim // 2) + num_warps = 4 if head_dim <= 256 else 8 + + _fused_ape_pool_norm_rope_kernel[(bs,)]( + kv_score_gathered, + kv_score_gathered.stride(0), + kv_score_gathered.stride(1), + ape, + ape.stride(0), + rms_weight, + rms_eps, + freqs_cis_real, + freqs_cis_real.stride(0), + out, + out.stride(0), + head_dim, + rope_head_dim, + half_dim, + RATIO=ratio, + K_POOL=k_in, + BLOCK_D=block_d, + HALF_ROPE=half_rope, + OVERLAP=int(overlap), + num_warps=num_warps, + ) + return out + + +@triton.jit +def _c4_decode_kernel( + kv_in_ptr, + out_ptr, + buffer_ptr, + ape_ptr, + indices_ptr, + seq_lens_ptr, + extra_ptr, + kv_in_row_stride, + out_row_stride, + buffer_page_stride, + buffer_slot_stride, + ape_row_stride, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +): + bid = tl.program_id(0) + pid_d = tl.program_id(1) + d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + + index = tl.load(indices_ptr + bid).to(tl.int64) + index_prev = tl.load(extra_ptr + bid).to(tl.int64) + seq_len = tl.load(seq_lens_ptr + bid).to(tl.int32) + write_slot = (seq_len + 3) % 4 + + in_base = bid.to(tl.int64) * kv_in_row_stride + page_base = ( + index * buffer_page_stride + write_slot.to(tl.int64) * buffer_slot_stride + ) + + valid_index = index >= 0 + for ch in tl.static_range(4): + ch_off = ch * HEAD_DIM + val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0) + tl.store( + buffer_ptr + page_base + ch_off + d_offs, + val, + mask=d_mask & valid_index, + ) + + NEG_BIG: tl.constexpr = -1.0e9 + running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) + running_sum = tl.zeros((BLOCK_D,), tl.float32) + weighted = tl.zeros((BLOCK_D,), tl.float32) + + for slot in tl.static_range(8): + if slot < 4: + page = index_prev + kv_off = 0 + score_off = 2 * HEAD_DIM + else: + page = index + kv_off = HEAD_DIM + score_off = 3 * HEAD_DIM + + src_pos = seq_len - 8 + slot + is_input = slot == 7 + write_pos = ((seq_len - 1) // 4) * 4 + page = tl.where(src_pos < write_pos, index_prev, index) + slot_in_page = src_pos % 4 + slot_base = ( + page * buffer_page_stride + slot_in_page.to(tl.int64) * buffer_slot_stride + ) + valid = src_pos >= 0 + if slot == 7: + kv = tl.load( + kv_in_ptr + in_base + kv_off + d_offs, + mask=d_mask & valid, + other=0.0, + ) + score = tl.load( + kv_in_ptr + in_base + score_off + d_offs, + mask=d_mask & valid, + other=NEG_BIG, + ) + else: + kv = tl.load( + buffer_ptr + slot_base + kv_off + d_offs, + mask=d_mask & valid, + other=0.0, + ) + score = tl.load( + buffer_ptr + slot_base + score_off + d_offs, + mask=d_mask & valid, + other=NEG_BIG, + ) + bias = tl.load(ape_ptr + slot * ape_row_stride + d_offs, mask=d_mask, other=0.0) + s = score + bias + new_max = tl.maximum(running_max, s) + factor = tl.exp(running_max - new_max) + e = tl.where(valid, tl.exp(s - new_max), 0.0) + running_sum = running_sum * factor + e + weighted = weighted * factor + kv * e + running_max = new_max + + tl.store( + out_ptr + bid.to(tl.int64) * out_row_stride + d_offs, + weighted / running_sum, + mask=d_mask, + ) + + +@triton.jit +def _c4_prefill_compress_kernel( + kv_in_ptr, + out_ptr, + buffer_ptr, + ape_ptr, + indices_ptr, + extra_ptr, + plan_ptr, + kv_in_row_stride, + out_row_stride, + buffer_page_stride, + buffer_slot_stride, + ape_row_stride, + plan_row_stride, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid_p = tl.program_id(0) + pid_d = tl.program_id(1) + d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + + plan_base = plan_ptr + pid_p * plan_row_stride + ragged_id = tl.load(plan_base + 0).to(tl.int32) + batch_id = tl.load(plan_base + 1).to(tl.int32) + position = tl.load(plan_base + 2).to(tl.int32) + window_len = tl.load(plan_base + 3).to(tl.int32) + if ragged_id < 0: + return + + extra_base = extra_ptr + batch_id.to(tl.int64) * 4 + load_first_page = tl.load(extra_base + 0).to(tl.int64) + load_second_page = tl.load(extra_base + 1).to(tl.int64) + + NEG_BIG: tl.constexpr = -1.0e9 + running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) + running_sum = tl.zeros((BLOCK_D,), tl.float32) + weighted = tl.zeros((BLOCK_D,), tl.float32) + + for slot in tl.static_range(8): + in_state = slot < window_len + if slot < 4: + page = tl.where(window_len <= 4, load_second_page, load_first_page) + kv_off = 0 + score_off = 2 * HEAD_DIM + slot_in_page = slot + else: + page = load_second_page + kv_off = HEAD_DIM + score_off = 3 * HEAD_DIM + slot_in_page = slot - 4 + + src_pos = position - 7 + slot + state_valid = in_state & (src_pos >= 0) + slot_base = page * buffer_page_stride + slot_in_page * buffer_slot_stride + in_row = ragged_id - (7 - slot) + in_row_safe = tl.where(in_state, 0, in_row) + in_base = in_row_safe.to(tl.int64) * kv_in_row_stride + + kv_state = tl.load( + buffer_ptr + slot_base + kv_off + d_offs, + mask=d_mask & state_valid, + other=0.0, + ) + score_state = tl.load( + buffer_ptr + slot_base + score_off + d_offs, + mask=d_mask & state_valid, + other=NEG_BIG, + ) + kv_input = tl.load( + kv_in_ptr + in_base + kv_off + d_offs, + mask=d_mask & (~in_state), + other=0.0, + ) + score_input = tl.load( + kv_in_ptr + in_base + score_off + d_offs, + mask=d_mask & (~in_state), + other=NEG_BIG, + ) + kv = tl.where(in_state, kv_state, kv_input) + score = tl.where(in_state, score_state, score_input) + bias = tl.load(ape_ptr + slot * ape_row_stride + d_offs, mask=d_mask, other=0.0) + + s = score + bias + new_max = tl.maximum(running_max, s) + factor = tl.exp(running_max - new_max) + e = tl.exp(s - new_max) + running_sum = running_sum * factor + e + weighted = weighted * factor + kv * e + running_max = new_max + + tl.store( + out_ptr + ragged_id.to(tl.int64) * out_row_stride + d_offs, + weighted / running_sum, + mask=d_mask, + ) + + +@triton.jit +def _c4_prefill_write_kernel( + kv_in_ptr, + buffer_ptr, + indices_ptr, + extra_ptr, + plan_ptr, + kv_in_row_stride, + buffer_page_stride, + buffer_slot_stride, + plan_row_stride, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid_p = tl.program_id(0) + pid_d = tl.program_id(1) + d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + + plan_base = plan_ptr + pid_p * plan_row_stride + ragged_id = tl.load(plan_base + 0).to(tl.int32) + batch_id = tl.load(plan_base + 1).to(tl.int32) + position = tl.load(plan_base + 2).to(tl.int32) + if ragged_id < 0: + return + + extra_base = extra_ptr + batch_id.to(tl.int64) * 4 + write_first_page = tl.load(extra_base + 2).to(tl.int64) + last_position = tl.load(extra_base + 3).to(tl.int32) + write_second_page = tl.load(indices_ptr + batch_id).to(tl.int64) + page = tl.where(position < last_position, write_first_page, write_second_page) + slot = position % 4 + + in_base = ragged_id.to(tl.int64) * kv_in_row_stride + dst_base = page * buffer_page_stride + slot.to(tl.int64) * buffer_slot_stride + for ch in tl.static_range(4): + ch_off = ch * HEAD_DIM + val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0) + tl.store(buffer_ptr + dst_base + ch_off + d_offs, val, mask=d_mask) + + +@triton.jit +def _c128_decode_kernel( + kv_in_ptr, + out_ptr, + buffer_ptr, + ape_ptr, + indices_ptr, + seq_lens_ptr, + extra_ptr, + kv_in_row_stride, + out_row_stride, + buffer_page_stride, + buffer_slot_stride, + ape_row_stride, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, +): + bid = tl.program_id(0) + pid_d = tl.program_id(1) + d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + + index = tl.load(indices_ptr + bid).to(tl.int64) + index_prev = tl.load(extra_ptr + bid).to(tl.int64) + seq_len = tl.load(seq_lens_ptr + bid).to(tl.int32) + write_slot = (seq_len + 127) % 128 + in_base = bid.to(tl.int64) * kv_in_row_stride + dst_base = index * buffer_page_stride + write_slot.to(tl.int64) * buffer_slot_stride + + for ch in tl.static_range(2): + ch_off = ch * HEAD_DIM + val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0) + tl.store(buffer_ptr + dst_base + ch_off + d_offs, val, mask=d_mask) + + NEG_BIG: tl.constexpr = -1.0e9 + running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) + running_sum = tl.zeros((BLOCK_D,), tl.float32) + weighted = tl.zeros((BLOCK_D,), tl.float32) + + for chunk_start in tl.static_range(0, 128, BLOCK_S): + slot_offs = chunk_start + tl.arange(0, BLOCK_S) + src_pos = seq_len - 128 + slot_offs + valid = src_pos >= 0 + is_input = slot_offs == 127 + write_pos = ((seq_len - 1) // 128) * 128 + pages = tl.where(src_pos < write_pos, index_prev, index) + slot_in_page = src_pos % 128 + slot_bases = ( + pages * buffer_page_stride + slot_in_page.to(tl.int64) * buffer_slot_stride + ) + kv_tile = tl.load( + buffer_ptr + slot_bases[:, None] + d_offs[None, :], + mask=valid[:, None] & (~is_input)[:, None] & d_mask[None, :], + other=0.0, + ) + score_tile = tl.load( + buffer_ptr + slot_bases[:, None] + HEAD_DIM + d_offs[None, :], + mask=valid[:, None] & (~is_input)[:, None] & d_mask[None, :], + other=NEG_BIG, + ) + kv_input_tile = tl.load( + kv_in_ptr + in_base + d_offs[None, :], + mask=valid[:, None] & is_input[:, None] & d_mask[None, :], + other=0.0, + ) + score_input_tile = tl.load( + kv_in_ptr + in_base + HEAD_DIM + d_offs[None, :], + mask=valid[:, None] & is_input[:, None] & d_mask[None, :], + other=NEG_BIG, + ) + kv_tile = tl.where(is_input[:, None], kv_input_tile, kv_tile) + score_tile = tl.where(is_input[:, None], score_input_tile, score_tile) + bias_tile = tl.load( + ape_ptr + slot_offs[:, None] * ape_row_stride + d_offs[None, :], + mask=d_mask[None, :], + other=0.0, + ) + s = score_tile + bias_tile + local_max = tl.max(s, axis=0) + new_max = tl.maximum(running_max, local_max) + exp_s = tl.exp(s - new_max[None, :]) + exp_s = tl.where(valid[:, None], exp_s, 0.0) + factor = tl.exp(running_max - new_max) + running_sum = running_sum * factor + tl.sum(exp_s, axis=0) + weighted = weighted * factor + tl.sum(kv_tile * exp_s, axis=0) + running_max = new_max + + tl.store( + out_ptr + bid.to(tl.int64) * out_row_stride + d_offs, + weighted / running_sum, + mask=d_mask, + ) + + +@triton.jit +def _c128_prefill_compress_kernel( + kv_in_ptr, + out_ptr, + buffer_ptr, + ape_ptr, + indices_ptr, + plan_ptr, + kv_in_row_stride, + out_row_stride, + buffer_page_stride, + buffer_slot_stride, + ape_row_stride, + plan_row_stride, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_S: tl.constexpr, +): + pid_p = tl.program_id(0) + pid_d = tl.program_id(1) + d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + + plan_base = plan_ptr + pid_p * plan_row_stride + ragged_id = tl.load(plan_base + 0).to(tl.int32) + batch_id = tl.load(plan_base + 1).to(tl.int32) + position = tl.load(plan_base + 2).to(tl.int32) + window_len = tl.load(plan_base + 3).to(tl.int32) + if ragged_id < 0: + return + + index = tl.load(indices_ptr + batch_id).to(tl.int64) + NEG_BIG: tl.constexpr = -1.0e9 + running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) + running_sum = tl.zeros((BLOCK_D,), tl.float32) + weighted = tl.zeros((BLOCK_D,), tl.float32) + + for chunk_start in tl.static_range(0, 128, BLOCK_S): + slot_offs = chunk_start + tl.arange(0, BLOCK_S) + is_state = slot_offs < window_len + src_pos = position - 127 + slot_offs + state_valid = is_state & (src_pos >= 0) + slot_bases = ( + index * buffer_page_stride + slot_offs.to(tl.int64) * buffer_slot_stride + ) + in_rows = ragged_id - (127 - slot_offs) + in_rows_safe = tl.where(is_state, tl.zeros_like(in_rows), in_rows) + in_bases = in_rows_safe.to(tl.int64) * kv_in_row_stride + + kv_state = tl.load( + buffer_ptr + slot_bases[:, None] + d_offs[None, :], + mask=state_valid[:, None] & d_mask[None, :], + other=0.0, + ) + score_state = tl.load( + buffer_ptr + slot_bases[:, None] + HEAD_DIM + d_offs[None, :], + mask=state_valid[:, None] & d_mask[None, :], + other=NEG_BIG, + ) + kv_input = tl.load( + kv_in_ptr + in_bases[:, None] + d_offs[None, :], + mask=(~is_state)[:, None] & d_mask[None, :], + other=0.0, + ) + score_input = tl.load( + kv_in_ptr + in_bases[:, None] + HEAD_DIM + d_offs[None, :], + mask=(~is_state)[:, None] & d_mask[None, :], + other=NEG_BIG, + ) + kv_tile = tl.where(is_state[:, None], kv_state, kv_input) + score_tile = tl.where(is_state[:, None], score_state, score_input) + bias_tile = tl.load( + ape_ptr + slot_offs[:, None] * ape_row_stride + d_offs[None, :], + mask=d_mask[None, :], + other=0.0, + ) + + s = score_tile + bias_tile + local_max = tl.max(s, axis=0) + new_max = tl.maximum(running_max, local_max) + exp_s = tl.exp(s - new_max[None, :]) + # Keep input-path entries valid; only state-path entries need src_pos guard. + valid = state_valid | (~is_state) + exp_s = tl.where(valid[:, None], exp_s, 0.0) + factor = tl.exp(running_max - new_max) + running_sum = running_sum * factor + tl.sum(exp_s, axis=0) + weighted = weighted * factor + tl.sum(kv_tile * exp_s, axis=0) + running_max = new_max + + tl.store( + out_ptr + ragged_id.to(tl.int64) * out_row_stride + d_offs, + weighted / running_sum, + mask=d_mask, + ) + + +@triton.jit +def _c128_prefill_write_kernel( + kv_in_ptr, + buffer_ptr, + indices_ptr, + plan_ptr, + kv_in_row_stride, + buffer_page_stride, + buffer_slot_stride, + plan_row_stride, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid_p = tl.program_id(0) + pid_d = tl.program_id(1) + d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + + plan_base = plan_ptr + pid_p * plan_row_stride + ragged_id = tl.load(plan_base + 0).to(tl.int32) + batch_id = tl.load(plan_base + 1).to(tl.int32) + position = tl.load(plan_base + 2).to(tl.int32) + if ragged_id < 0: + return + + index = tl.load(indices_ptr + batch_id).to(tl.int64) + slot = position % 128 + in_base = ragged_id.to(tl.int64) * kv_in_row_stride + dst_base = index * buffer_page_stride + slot.to(tl.int64) * buffer_slot_stride + + for ch in tl.static_range(2): + ch_off = ch * HEAD_DIM + val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0) + tl.store(buffer_ptr + dst_base + ch_off + d_offs, val, mask=d_mask) + + +@triton.jit +def _compress_norm_rope_kernel( + kv_ptr, + weight_ptr, + freqs_ptr, + handle_ptr, + eps, + kv_row_stride, + freqs_row_stride, + plan_row_stride, + HEAD_DIM: tl.constexpr, + ROPE_DIM: tl.constexpr, + HEAD_BLOCK: tl.constexpr, + ROPE_PAIR_BLOCK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + IS_DECODE: tl.constexpr, +): + work_id = tl.program_id(0) + + if IS_DECODE: + row = work_id + seq_len = tl.load(handle_ptr + work_id).to(tl.int32) + position = ((seq_len - 1) // COMPRESS_RATIO) * COMPRESS_RATIO + else: + plan_base = handle_ptr + work_id * plan_row_stride + row = tl.load(plan_base + 0).to(tl.int32) + plan_position = tl.load(plan_base + 2).to(tl.int32) + if row < 0: + return + position = plan_position + 1 - COMPRESS_RATIO + + base = row.to(tl.int64) * kv_row_stride + offs = tl.arange(0, HEAD_BLOCK) + mask = offs < HEAD_DIM + x = tl.load(kv_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) + w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32) + rms_inv = tl.rsqrt(tl.sum(x * x, axis=0) / HEAD_DIM + eps) + x_normed = x * rms_inv * w + + rope_start: tl.constexpr = HEAD_DIM - ROPE_DIM + pair_offs = tl.arange(0, ROPE_PAIR_BLOCK) + pair_mask = pair_offs < (ROPE_DIM // 2) + x_real = tl.load( + kv_ptr + base + rope_start + 2 * pair_offs, + mask=pair_mask, + other=0.0, + ).to(tl.float32) + x_imag = tl.load( + kv_ptr + base + rope_start + 2 * pair_offs + 1, + mask=pair_mask, + other=0.0, + ).to(tl.float32) + w_real = tl.load( + weight_ptr + rope_start + 2 * pair_offs, + mask=pair_mask, + other=1.0, + ).to(tl.float32) + w_imag = tl.load( + weight_ptr + rope_start + 2 * pair_offs + 1, + mask=pair_mask, + other=1.0, + ).to(tl.float32) + x_real = x_real * rms_inv * w_real + x_imag = x_imag * rms_inv * w_imag + + freq_base = position.to(tl.int64) * freqs_row_stride + f_real = tl.load(freqs_ptr + freq_base + 2 * pair_offs, mask=pair_mask, other=0.0) + f_imag = tl.load( + freqs_ptr + freq_base + 2 * pair_offs + 1, + mask=pair_mask, + other=0.0, + ) + out_real = x_real * f_real - x_imag * f_imag + out_imag = x_real * f_imag + x_imag * f_real + + tl.store(kv_ptr + base + offs, x_normed, mask=mask & (offs < rope_start)) + tl.store(kv_ptr + base + rope_start + 2 * pair_offs, out_real, mask=pair_mask) + tl.store(kv_ptr + base + rope_start + 2 * pair_offs + 1, out_imag, mask=pair_mask) + + +def _plan_as_i32(plan: torch.Tensor) -> torch.Tensor: + assert plan.dtype == torch.uint8 and plan.dim() == 2 and plan.shape[1] == 16 + return plan.view(torch.int32).view(-1, 4) + + +def _block_d(head_dim: int) -> int: + return min(32, triton.next_power_of_2(head_dim)) + + +def _check_common( + kv_score_buffer: torch.Tensor, + kv_score_input: torch.Tensor, + out: torch.Tensor, + ape: torch.Tensor, + indices: torch.Tensor, + head_dim: int, + compress_ratio: int, +) -> None: + coff = 2 if compress_ratio == 4 else 1 + assert kv_score_input.is_cuda and kv_score_buffer.is_cuda + assert kv_score_input.dim() == 2 and kv_score_input.dtype == torch.float32 + assert kv_score_input.shape[1] == 2 * coff * head_dim + assert kv_score_buffer.dim() == 3 and kv_score_buffer.dtype == torch.float32 + assert kv_score_buffer.shape[1:] == (compress_ratio, 2 * coff * head_dim) + assert out.shape == (kv_score_input.shape[0], head_dim) + assert out.dtype == torch.float32 and out.is_cuda + assert ape.shape == (compress_ratio * coff, head_dim) + assert ape.dtype == torch.float32 and ape.is_cuda + assert indices.dtype == torch.int32 and indices.is_cuda + + +def _is_decode_plan(plan: Union[CompressorDecodePlan, CompressorPrefillPlan]) -> bool: + return isinstance(plan, CompressorDecodePlan) + + +def hip_compress_forward( + *, + kv_score_buffer: torch.Tensor, + kv_score_input: torch.Tensor, + ape: torch.Tensor, + indices: torch.Tensor, + plan: Union[CompressorDecodePlan, CompressorPrefillPlan], + extra_data: Optional[torch.Tensor], + head_dim: int, + compress_ratio: int, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if compress_ratio not in (4, 128): + raise ValueError(f"unsupported {compress_ratio=}") + if out is None: + out = kv_score_input.new_empty((kv_score_input.shape[0], head_dim)) + is_decode = _is_decode_plan(plan) + if not is_decode: + out.fill_(10000.0) + + _check_common( + kv_score_buffer, + kv_score_input, + out, + ape, + indices, + head_dim, + compress_ratio, + ) + + BLOCK_D = _block_d(head_dim) + num_d_chunks = triton.cdiv(head_dim, BLOCK_D) + + if is_decode: + seq_lens = plan.seq_lens + assert seq_lens.dtype == torch.int32 and seq_lens.is_cuda + assert seq_lens.shape == indices.shape + grid = (seq_lens.numel(), num_d_chunks) + if compress_ratio == 4: + assert extra_data is not None + assert extra_data.shape == (seq_lens.numel(), 1) + _c4_decode_kernel[grid]( + kv_score_input, + out, + kv_score_buffer, + ape, + indices, + seq_lens, + extra_data, + kv_score_input.stride(0), + out.stride(0), + kv_score_buffer.stride(0), + kv_score_buffer.stride(1), + ape.stride(0), + HEAD_DIM=head_dim, + BLOCK_D=BLOCK_D, + ) + else: + assert extra_data is not None + assert extra_data.shape == seq_lens.shape + _c128_decode_kernel[grid]( + kv_score_input, + out, + kv_score_buffer, + ape, + indices, + seq_lens, + extra_data, + kv_score_input.stride(0), + out.stride(0), + kv_score_buffer.stride(0), + kv_score_buffer.stride(1), + ape.stride(0), + HEAD_DIM=head_dim, + BLOCK_D=BLOCK_D, + BLOCK_S=64, + ) + return out + + compress_plan = _plan_as_i32(plan.compress_plan) + write_plan = _plan_as_i32(plan.write_plan) + if compress_ratio == 4: + assert extra_data is not None + assert extra_data.dim() == 2 and extra_data.shape[1] == 4 + compress_grid = (compress_plan.shape[0], num_d_chunks) + write_grid = (write_plan.shape[0], num_d_chunks) + _c4_prefill_compress_kernel[compress_grid]( + kv_score_input, + out, + kv_score_buffer, + ape, + indices, + extra_data, + compress_plan, + kv_score_input.stride(0), + out.stride(0), + kv_score_buffer.stride(0), + kv_score_buffer.stride(1), + ape.stride(0), + compress_plan.stride(0), + HEAD_DIM=head_dim, + BLOCK_D=BLOCK_D, + ) + _c4_prefill_write_kernel[write_grid]( + kv_score_input, + kv_score_buffer, + indices, + extra_data, + write_plan, + kv_score_input.stride(0), + kv_score_buffer.stride(0), + kv_score_buffer.stride(1), + write_plan.stride(0), + HEAD_DIM=head_dim, + BLOCK_D=BLOCK_D, + ) + else: + load_indices = indices if extra_data is None else extra_data + assert load_indices.dim() == 1 and load_indices.dtype == torch.int32 + compress_grid = (compress_plan.shape[0], num_d_chunks) + write_grid = (write_plan.shape[0], num_d_chunks) + _c128_prefill_compress_kernel[compress_grid]( + kv_score_input, + out, + kv_score_buffer, + ape, + load_indices, + compress_plan, + kv_score_input.stride(0), + out.stride(0), + kv_score_buffer.stride(0), + kv_score_buffer.stride(1), + ape.stride(0), + compress_plan.stride(0), + HEAD_DIM=head_dim, + BLOCK_D=BLOCK_D, + BLOCK_S=64, + ) + _c128_prefill_write_kernel[write_grid]( + kv_score_input, + kv_score_buffer, + indices, + write_plan, + kv_score_input.stride(0), + kv_score_buffer.stride(0), + kv_score_buffer.stride(1), + write_plan.stride(0), + HEAD_DIM=head_dim, + BLOCK_D=BLOCK_D, + ) + return out + + +def hip_compress_fused_norm_rope_inplace( + kv: torch.Tensor, + weight: torch.Tensor, + eps: float, + freqs_cis: torch.Tensor, + plan: Union[CompressorDecodePlan, CompressorPrefillPlan], +) -> None: + assert kv.dim() == 2 and kv.stride(-1) == 1 + assert weight.shape == (kv.shape[1],) + freqs_real = torch.view_as_real(freqs_cis).flatten(-2) + head_dim = kv.shape[1] + rope_dim = freqs_real.shape[-1] + assert head_dim >= rope_dim and rope_dim % 2 == 0 + + is_decode = _is_decode_plan(plan) + if is_decode: + handle = plan.seq_lens + else: + handle = _plan_as_i32(plan.compress_plan) + + if handle.numel() == 0: + return + + HEAD_BLOCK = triton.next_power_of_2(head_dim) + ROPE_PAIR_BLOCK = max(triton.next_power_of_2(rope_dim // 2), 1) + _compress_norm_rope_kernel[(handle.shape[0],)]( + kv, + weight, + freqs_real, + handle, + eps, + kv.stride(0), + freqs_real.stride(0), + handle.stride(0) if not is_decode else 0, + HEAD_DIM=head_dim, + ROPE_DIM=rope_dim, + HEAD_BLOCK=HEAD_BLOCK, + ROPE_PAIR_BLOCK=ROPE_PAIR_BLOCK, + COMPRESS_RATIO=plan.compress_ratio, + IS_DECODE=is_decode, + ) diff --git a/python/sglang/srt/layers/attention/dsv4/indexer.py b/python/sglang/srt/layers/attention/dsv4/indexer.py index f8899264d6d0..afde82d7a052 100644 --- a/python/sglang/srt/layers/attention/dsv4/indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/indexer.py @@ -22,6 +22,7 @@ from sglang.srt.utils import add_prefix, is_hip if TYPE_CHECKING: + from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.dsv4.compressor import ( CompressorBackendMixin, ) @@ -38,6 +39,9 @@ FP8_MAX = torch.finfo(FP8_DTYPE).max +_arange_cache = {} + + def fp8_paged_mqa_logits_torch( q_fp8: torch.Tensor, kvcache_fp8: torch.Tensor, @@ -48,12 +52,13 @@ def fp8_paged_mqa_logits_torch( max_seq_len: int, clean_logits: bool = True, ) -> torch.Tensor: + """Vectorized implementation compatible with CUDA graph capture.""" _ = deep_gemm_metadata batch_size, _, num_heads, head_dim = q_fp8.shape block_size = kvcache_fp8.shape[1] - assert head_dim == 128, "torch reference impl hardcodes DSV4 indexer head_dim=128" - assert block_size == 64, "torch reference impl hardcodes block_size=64 cache layout" + assert head_dim == 128 + assert block_size == 64 assert q_fp8.shape == (batch_size, 1, num_heads, head_dim) assert kvcache_fp8.shape[1:] == (block_size, 1, head_dim + 4) assert weight.shape == (batch_size, num_heads) @@ -61,32 +66,85 @@ def fp8_paged_mqa_logits_torch( assert page_table.shape[0] == batch_size assert clean_logits == False - logits = page_table.new_empty((batch_size, max_seq_len), dtype=torch.float32) - for i in range(batch_size): - q = q_fp8[i, 0] - q = q.to(torch.float32) - q_scale = weight[i] - seq_len = int(seq_lens[i].item()) - assert seq_len <= max_seq_len - num_pages = (seq_len + block_size - 1) // block_size - padded_seq_len = num_pages * block_size - pages = page_table[i, :num_pages] - kvcache_fp8 = kvcache_fp8.view(-1, block_size * (head_dim + 4)) - kvcache = kvcache_fp8[pages] - SCALE_OFFSET = block_size * head_dim - kvcache_value = kvcache[..., :SCALE_OFFSET].view(dtype=FP8_DTYPE) - kvcache_scale = kvcache[..., SCALE_OFFSET:].view(dtype=torch.float32) - kvcache_value = kvcache_value.to(torch.float32) - kvcache_scale = kvcache_scale.contiguous() - kvcache_value = kvcache_value.view(padded_seq_len, head_dim) - kvcache_scale = kvcache_scale.view(padded_seq_len) - score = F.linear(kvcache_value, q) - score = F.relu(score) - score *= q_scale[None, :] - score = score.sum(dim=1) - score *= kvcache_scale - logits[i, :seq_len] = score[:seq_len] + max_num_pages = page_table.shape[1] + SCALE_OFFSET = block_size * head_dim + total_dim = block_size * (head_dim + 4) + + kvcache_flat = kvcache_fp8.view(-1, total_dim) + + pages_clamped = page_table.clamp(min=0) + kvcache_gathered = kvcache_flat[pages_clamped] + + kv_values_raw = kvcache_gathered[..., :SCALE_OFFSET].contiguous() + kv_values_fp8 = kv_values_raw.view(dtype=FP8_DTYPE) + kv_values = kv_values_fp8.to(torch.float32) + kv_values = kv_values.reshape(batch_size, max_num_pages * block_size, head_dim) + + kv_scales_raw = kvcache_gathered[..., SCALE_OFFSET:].contiguous() + kv_scales = kv_scales_raw.view(dtype=torch.float32) + kv_scales = kv_scales.reshape(batch_size, max_num_pages * block_size) + + q_float = q_fp8[:, 0].to(torch.float32) + scores = torch.bmm(kv_values, q_float.transpose(1, 2)) + scores = F.relu(scores) + scores = scores * weight.unsqueeze(1) + scores = scores.sum(dim=2) + scores = scores * kv_scales + + padded_seq_len = max_num_pages * block_size + cache = _arange_cache + arange_key = f"arange_{padded_seq_len}_{scores.device}" + if arange_key not in cache: + cache[arange_key] = torch.arange(padded_seq_len, device=scores.device) + positions = cache[arange_key].unsqueeze(0) + valid_mask = positions < seq_lens.unsqueeze(1) + scores = scores.masked_fill(~valid_mask, 0.0) + + if padded_seq_len < max_seq_len: + scores = F.pad(scores, (0, max_seq_len - padded_seq_len), value=0.0) + else: + scores = scores[:, :max_seq_len] + + return scores + + +def _aiter_fp8_paged_mqa_logits( + q_fp8: torch.Tensor, + kvcache_fp8: torch.Tensor, + weight: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + deep_gemm_metadata: Any, + max_seq_len: int, + clean_logits: bool = False, +) -> torch.Tensor: + """Wrapper adapting aiter's deepgemm_fp8_paged_mqa_logits to SGLang's interface.""" + from aiter.ops.triton.attention.pa_mqa_logits import ( + deepgemm_fp8_paged_mqa_logits, + ) + batch_size = q_fp8.shape[0] + next_n = q_fp8.shape[1] + total_tokens = batch_size * next_n + _sl = seq_lens.squeeze(-1) if seq_lens.dim() == 2 else seq_lens + kv_block_size = kvcache_fp8.shape[1] + logits = torch.empty( + total_tokens, + max_seq_len, + dtype=torch.float32, + device=q_fp8.device, + ) + deepgemm_fp8_paged_mqa_logits( + q_fp8, + kvcache_fp8, + weight, + logits, + _sl.to(torch.int32), + page_table.to(torch.int32), + max_seq_len, + KVBlockSize=kv_block_size, + Preshuffle=True, + ) return logits @@ -98,6 +156,9 @@ def topk_transform_512_pytorch_vectorized( page_size: int, out_raw_indices: Optional[torch.Tensor] = None, ) -> None: + """Vectorized PyTorch fallback for topk_transform_512. + All helper tensors (arange, zeros) are cached to avoid device-tensor + creation during HIP/CUDA graph capture.""" TOPK = out_page_indices.shape[1] batch_size = scores.shape[0] @@ -107,13 +168,22 @@ def topk_transform_512_pytorch_vectorized( page_bits = (page_size - 1).bit_length() if page_size > 1 else 0 page_mask = page_size - 1 - positions = ( - torch.arange(max_seq_len, device=device).unsqueeze(0).expand(batch_size, -1) - ) + cache = _arange_cache + key_seq = f"arange_{max_seq_len}_{device}" + key_topk = f"arange_{TOPK}_{device}" + key_bs = f"arange_{batch_size}_{device}" + if key_seq not in cache: + cache[key_seq] = torch.arange(max_seq_len, device=device) + if key_topk not in cache: + cache[key_topk] = torch.arange(TOPK, device=device, dtype=torch.int32) + if key_bs not in cache: + cache[key_bs] = torch.arange(batch_size, device=device) + + positions = cache[key_seq].unsqueeze(0).expand(batch_size, -1) valid_mask = positions < seq_lens.unsqueeze(1) masked_scores = scores.clone() - masked_scores[~valid_mask] = float("-inf") + masked_scores.masked_fill_(~valid_mask, float("-inf")) actual_k = min(TOPK, max_seq_len) _, raw_indices = torch.topk( @@ -122,44 +192,28 @@ def topk_transform_512_pytorch_vectorized( raw_indices = raw_indices.to(torch.int32) if actual_k < TOPK: - padding = torch.zeros( - (batch_size, TOPK - actual_k), dtype=torch.int32, device=device - ) - raw_indices = torch.cat([raw_indices, padding], dim=1) + raw_indices = F.pad(raw_indices, (0, TOPK - actual_k), value=0) - batch_indices = ( - torch.arange(batch_size, device=device).unsqueeze(1).expand(-1, TOPK) - ) + batch_indices = cache[key_bs].unsqueeze(1).expand(-1, TOPK) gathered_scores = scores[ batch_indices.flatten(), raw_indices.clamp(min=0).flatten() ].view(batch_size, TOPK) valid_topk = gathered_scores != float("-inf") if actual_k < TOPK: - pad_mask = torch.arange(TOPK, device=device).unsqueeze(0) >= actual_k + pad_mask = cache[key_topk].unsqueeze(0) >= actual_k valid_topk = valid_topk & ~pad_mask needs_sequential = seq_lens <= TOPK - if needs_sequential.any(): - sequential_indices = ( - torch.arange(TOPK, device=device, dtype=torch.int32) - .unsqueeze(0) - .expand(batch_size, -1) - ) - sequential_valid = sequential_indices < seq_lens.unsqueeze(1) - - raw_indices = torch.where( - needs_sequential.unsqueeze(1).expand(-1, TOPK), - torch.where( - sequential_valid, - sequential_indices, - torch.tensor(-1, device=device, dtype=torch.int32), - ), - raw_indices, - ) - valid_topk = torch.where( - needs_sequential.unsqueeze(1).expand(-1, TOPK), sequential_valid, valid_topk - ) + sequential_indices = cache[key_topk].unsqueeze(0).expand(batch_size, -1) + sequential_valid = sequential_indices < seq_lens.unsqueeze(1) + + seq_indices_or_neg1 = sequential_indices.clone() + seq_indices_or_neg1.masked_fill_(~sequential_valid, -1) + + needs_seq_mask = needs_sequential.unsqueeze(1).expand(-1, TOPK) + raw_indices = torch.where(needs_seq_mask, seq_indices_or_neg1, raw_indices) + valid_topk = torch.where(needs_seq_mask, sequential_valid, valid_topk) page_idx = raw_indices >> page_bits offset_in_page = raw_indices & page_mask @@ -169,17 +223,13 @@ def topk_transform_512_pytorch_vectorized( page_indices = (physical_pages << page_bits) | offset_in_page page_indices = page_indices.to(torch.int32) - - page_indices = torch.where( - valid_topk, page_indices, torch.tensor(-1, device=device, dtype=torch.int32) - ) + page_indices.masked_fill_(~valid_topk, -1) out_page_indices.copy_(page_indices) if out_raw_indices is not None: - raw_indices = torch.where( - valid_topk, raw_indices, torch.tensor(-1, device=device, dtype=torch.int32) - ) + raw_indices = raw_indices.clone() + raw_indices.masked_fill_(~valid_topk, -1) out_raw_indices.copy_(raw_indices) @@ -289,18 +339,20 @@ def _forward_prepare_normal( positions: torch.Tensor, forward_batch: ForwardBatch, token_to_kv_pool: DeepSeekV4TokenToKVPool, + skip_compressor: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: if TYPE_CHECKING: assert isinstance(self, CompressorBackendMixin) weights = c4_indexer.compute_weights(x, skip_scale=True) q_fp8, weights = c4_indexer.compute_q(q_lora, positions, weights) - self.forward_indexer_compressor( - x=x, - forward_batch=forward_batch, - layer_id=c4_indexer.layer_id, - compressor=c4_indexer.compressor, - ) + if not skip_compressor: + self.forward_indexer_compressor( + x=x, + forward_batch=forward_batch, + layer_id=c4_indexer.layer_id, + compressor=c4_indexer.compressor, + ) c4_indexer_kv_cache = token_to_kv_pool.get_index_k_with_scale_buffer( layer_id=c4_indexer.layer_id, ) @@ -315,13 +367,14 @@ def forward_c4_indexer( alt_streams: Optional[List[torch.cuda.Stream]] = None, enable_multi_stream: bool = False, q_lora_ready: Optional[torch.cuda.Event] = None, + skip_compressor: bool = False, ) -> None: if forward_batch.forward_mode.is_idle(): return # PREP_IN_CG lazy upgrade: this runs from MQALayer._forward_prepare, # before attn_backend.forward() would trigger the upgrade. self._maybe_upgrade_forward_metadata() - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = self.token_to_kv_pool if TYPE_CHECKING: assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) @@ -353,6 +406,7 @@ def forward_c4_indexer( positions=core_metadata.positions, forward_batch=forward_batch, token_to_kv_pool=token_to_kv_pool, + skip_compressor=skip_compressor, ) assert len(q_fp8.shape) == 3 @@ -371,6 +425,8 @@ def forward_c4_indexer( from sglang.srt.layers.attention.dsa.tilelang_kernel import ( tilelang_fp8_paged_mqa_logits as fn, ) + elif envs.SGLANG_OPT_USE_AITER_INDEXER.get(): + fn = _aiter_fp8_paged_mqa_logits elif envs.SGLANG_FP8_PAGED_MQA_LOGITS_TORCH.get(): fn = fp8_paged_mqa_logits_torch else: @@ -378,7 +434,8 @@ def forward_c4_indexer( _c4sl = indexer_metadata.c4_seq_lens _use_tilelang = envs.SGLANG_OPT_USE_TILELANG_INDEXER.get() - if _c4sl.dim() == 1 and not _use_tilelang: + _use_aiter = envs.SGLANG_OPT_USE_AITER_INDEXER.get() + if _c4sl.dim() == 1 and not _use_tilelang and not _use_aiter: _c4sl = _c4sl.unsqueeze(-1) logits = fn( q_fp8, @@ -398,7 +455,7 @@ def forward_c4_indexer( indexer_capturer = get_global_indexer_capturer() capture_enabled = indexer_capturer is not None - hisparse_coordinator = forward_batch.hisparse_coordinator + hisparse_coordinator = self.hisparse_coordinator hisparse_decode = ( hisparse_coordinator is not None and forward_batch.forward_mode.is_decode() ) @@ -541,10 +598,12 @@ def forward( x: torch.Tensor, q_lora: torch.Tensor, forward_batch: ForwardBatch, + attn_backend: AttentionBackend, enable_multi_stream: bool = False, q_lora_ready: Optional[torch.cuda.Event] = None, + skip_compressor: bool = False, ) -> None: - return forward_batch.attn_backend.forward_c4_indexer( + return attn_backend.forward_c4_indexer( x=x, q_lora=q_lora, forward_batch=forward_batch, @@ -552,4 +611,5 @@ def forward( alt_streams=self.alt_streams, enable_multi_stream=enable_multi_stream, q_lora_ready=q_lora_ready, + skip_compressor=skip_compressor, ) diff --git a/python/sglang/srt/layers/attention/dsv4/metadata.py b/python/sglang/srt/layers/attention/dsv4/metadata.py index c3e8041032d5..c4d4668085da 100644 --- a/python/sglang/srt/layers/attention/dsv4/metadata.py +++ b/python/sglang/srt/layers/attention/dsv4/metadata.py @@ -103,7 +103,10 @@ class PagedIndexerMetadata: topk_metadata: torch.Tensor = field(init=False, repr=False) def __post_init__(self): - if envs.SGLANG_FP8_PAGED_MQA_LOGITS_TORCH.get(): + if ( + envs.SGLANG_FP8_PAGED_MQA_LOGITS_TORCH.get() + or envs.SGLANG_OPT_USE_AITER_INDEXER.get() + ): self.deep_gemm_metadata = None else: import deep_gemm @@ -148,14 +151,17 @@ def max_c4_seq_len(self) -> int: def copy_(self, other: "PagedIndexerMetadata"): if is_hip(): copy_fields = ["page_table", "c4_seq_lens"] + assign_fields = ["deep_gemm_metadata"] else: copy_fields = ["page_table", "c4_seq_lens", "deep_gemm_metadata"] + assign_fields = [] copy_fields += ["topk_metadata"] copy_metadata( src=other, dst=self, check_eq_fields=["page_size"], copy_fields=copy_fields, + assign_fields=assign_fields, ) diff --git a/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py index a84015a803f8..fa0da5c46c05 100644 --- a/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py +++ b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py @@ -117,6 +117,10 @@ def __init__( ) self.head_size = model_runner.model_config.head_dim + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype @@ -183,7 +187,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): metadata.orig_seq_lens_tensor = forward_batch.orig_seq_lens metadata.orig_seq_lens = forward_batch.orig_seq_lens.tolist() - metadata.block_tables = forward_batch.req_to_token_pool.req_to_token[ + metadata.block_tables = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len ] # Convert the block table to a strided format. @@ -346,9 +350,7 @@ def forward_extend( assert current_end <= self.max_context_len # Do multi-head attention - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim ) @@ -358,7 +360,7 @@ def forward_extend( if key is not None and value is not None: if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, key, @@ -442,9 +444,7 @@ def forward_decode( key = k.view(-1, self.num_kv_heads, self.head_size) value = v.view(-1, self.num_kv_heads, self.head_size) - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim ) @@ -454,7 +454,7 @@ def forward_decode( if key is not None and value is not None: if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, key, diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index cc3b1ca32ca6..1d0d1b8ba5f9 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -126,6 +126,10 @@ def __init__( self.device = model_runner.device self.decode_cuda_graph_metadata = {} self.target_verify_metadata = {} + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype @@ -138,8 +142,6 @@ def __init__( isinstance(model_runner.token_to_kv_pool, SWAKVPool) and model_runner.token_to_kv_pool.swa_layer_nums > 0 ) - if self.use_sliding_window_kv_pool: - self.token_to_kv_pool = model_runner.token_to_kv_pool self.topk = model_runner.server_args.speculative_eagle_topk or 0 self.speculative_num_steps = speculative_num_steps @@ -295,7 +297,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] else: @@ -315,7 +317,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] metadata_expand = FlashAttentionMetadata() @@ -358,7 +360,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): metadata.cu_seqlens_k = torch.nn.functional.pad( torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] # Precompute FA3 scheduler metadata to avoid per-layer @@ -394,7 +396,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -416,7 +418,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -477,7 +479,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ) _, sort_order = torch.sort(keys, dim=1) non_masked_page_table = ( - forward_batch.req_to_token_pool.req_to_token[ + self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : ] .gather(1, cols) @@ -506,7 +508,26 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): metadata.cu_seqlens_k = torch.nn.functional.pad( torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + + # MLA/MHA CP: prepare_mlp_sync_batch pads extend tokens up to + # lcm(attn_tp_size, attn_cp_size), so cache_seqlens_cp can exceed + # seq_lens_cpu.max(). Widen page_table by the pad delta to keep + # FA3's causal reads in-bounds; widened columns index KV slot 0 + # (req_to_token is zero-init) and outputs for padding queries are + # discarded downstream. + if ( + self.attn_cp_size > 1 + and forward_batch.global_num_tokens_cpu is not None + and forward_batch.extend_num_tokens is not None + and forward_batch.extend_seq_lens_cpu is not None + ): + padded_extend = int(forward_batch.extend_num_tokens) + real_extend = int(sum(forward_batch.extend_seq_lens_cpu)) + pad_delta = padded_extend - real_extend + if pad_delta > 0: + metadata.max_seq_len_k += pad_delta + + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -526,28 +547,37 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): if forward_batch.forward_mode == ForwardMode.EXTEND: self._maybe_init_local_attn_metadata(forward_batch, metadata, device) - # Encoder metadata for cross attention + # Encoder metadata for cross attention. Supports per-request varlen + # encoder lengths (e.g. MossVL with different image sizes per request). if forward_batch.encoder_lens is not None: - assert ( - forward_batch.encoder_lens.numel() == 1 - ), "Only encoder size 1 is supported for now" - metadata.encoder_lens_int32 = forward_batch.encoder_lens.to(torch.int32) metadata.encoder_cu_seqlens_k = torch.nn.functional.pad( torch.cumsum(metadata.encoder_lens_int32, dim=0, dtype=torch.int32), (1, 0), ) metadata.encoder_max_seq_len_k = metadata.encoder_lens_int32.max().item() - metadata.encoder_page_table = forward_batch.req_to_token_pool.req_to_token[ + + # Cross-attn page_table: per-request rows. cache_seqlens + # (encoder_lens_int32) caps per-request reads so any garbage past + # encoder_lens[i] is never consumed. + metadata.encoder_page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.encoder_max_seq_len_k ] - # Currently only support forward_batch.encoder_lens.numel() == 1 - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ - forward_batch.req_pool_indices, - metadata.encoder_max_seq_len_k : ( - metadata.encoder_max_seq_len_k + metadata.max_seq_len_k - ), + # Self-attn (text) page_table: text starts at per-request offset + # encoder_lens[i], NOT at a single max. Use a fancy-index gather. + text_max = metadata.max_seq_len_k + arange_text = torch.arange( + text_max, device=forward_batch.req_pool_indices.device + ) + text_col = forward_batch.encoder_lens.long().unsqueeze( + 1 + ) + arange_text.unsqueeze( + 0 + ) # (bs, max_seq_len_k) + text_row = forward_batch.req_pool_indices.unsqueeze(1).expand(-1, text_max) + metadata.page_table = self.req_to_token_pool.req_to_token[ + text_row, text_col ] if self.use_sliding_window_kv_pool: @@ -625,36 +655,43 @@ def forward_extend( k_rope: Optional[torch.Tensor] = None, sinks: Optional[torch.Tensor] = None, ): + is_cp_mode = ( + forward_batch.forward_mode.is_context_parallel_extend() + and forward_batch.attn_cp_metadata is not None + and self.attn_cp_size > 1 + ) + if k is not None: assert v is not None - is_cp_mode = ( - forward_batch.forward_mode.is_context_parallel_extend() - and forward_batch.attn_cp_metadata is not None - and self.attn_cp_size > 1 - ) - - if save_kv_cache and not is_cp_mode and not self.fa_skip_kv_cache: + if save_kv_cache and not self.fa_skip_kv_cache: cache_loc = ( forward_batch.out_cache_loc if not layer.is_cross_attention else forward_batch.encoder_out_cache_loc ) - if not self.use_mla: - forward_batch.token_to_kv_pool.set_kv_buffer( - layer, cache_loc, k, v, layer.k_scale, layer.v_scale - ) - else: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + if self.use_mla: + # MLA: under CP, k and k_rope arrive full-sequence + # (rebuild_cp_kv_cache ran upstream in + # forward_absorb_prepare); rank-local otherwise. + # out_cache_loc is never zigzag-split, so the write + # lands in the right slots on every rank in either case. + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, k_rope, ) - if is_cp_mode: - cp_allgather_and_save_kv_cache( - forward_batch, layer, k, v, self.attn_cp_size - ) + elif is_cp_mode: + # Dense-MHA CP: k, v are still rank-local; backend + # all-gathers and writes to the per-rank pool. + cp_allgather_and_save_kv_cache( + forward_batch, layer, k, v, self.attn_cp_size + ) + else: + self.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) # Use precomputed metadata across all layers metadata = self.forward_metadata @@ -746,9 +783,7 @@ def forward_extend( # Use Flash Attention for prefill if not self.use_mla: # Do multi-head attention - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim @@ -950,9 +985,9 @@ def _fa_cp_attn( else: assert self.fa_impl_ver == 3, "Only FA3 support here" # Do absorbed multi-latent attention - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer( - layer.layer_id - ).to(q.dtype) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to( + q.dtype + ) k_rope = kv_cache[:, :, layer.v_head_dim :] c_kv = kv_cache[:, :, : layer.v_head_dim] k_rope_cache = k_rope.view( @@ -974,57 +1009,103 @@ def _fa_cp_attn( q_nope = q_all[:, :, : layer.v_head_dim] q_rope = q_all[:, :, layer.v_head_dim :] - result = flash_attn_with_kvcache( - q=q_rope, - k_cache=k_rope_cache, - v_cache=c_kv_cache, - qv=q_nope, - page_table=page_table, - cache_seqlens=cache_seqlens, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k_new=cu_seqlens_k if not use_local_attn else None, - max_seqlen_q=max_seqlen_q, - softmax_scale=layer.scaling, - causal=False if use_cascade_attn else causal, - softcap=layer.logit_cap, - k_descale=k_descale, - v_descale=v_descale, - return_softmax_lse=use_cascade_attn, - num_splits=self.num_splits, - ver=self.fa_impl_ver, - ) - if use_cascade_attn: - o, softmax_lse, *rest = result - o_expand, softmax_lse_expand, *rest_expand = ( - flash_attn_with_kvcache( - q=q_rope, + if is_cp_mode: + # MLA CP: q is rank-local zigzag-split; run the + # absorbed-MLA kernel twice (prev/next halves) against + # the full latent KV pool (which rebuild_cp_kv_cache + # populated upstream) via cp_attn_forward_extend. + # Concat q_nope + q_rope along dim=-1 so the wrapper's + # chunk(2, dim=0) keeps their alignment; split back + # inside the closure. + assert ( + not use_cascade_attn + ), "Cascade attention under MLA CP is not supported in v1." + q_fused = torch.cat([q_nope, q_rope], dim=-1) + + def _mla_cp_attn( + q_chunk, + cu_seqlens_q_cp, + cache_seqlens_cp, + max_seqlen_q_cp, + ): + q_nope_chunk = q_chunk[..., : layer.v_head_dim] + q_rope_chunk = q_chunk[..., layer.v_head_dim :] + return flash_attn_with_kvcache( + q=q_rope_chunk, + qv=q_nope_chunk, k_cache=k_rope_cache, v_cache=c_kv_cache, - qv=q_nope, - page_table=self.forward_metadata_spec_decode_expand.page_table, - cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32, - cu_seqlens_q=self.forward_metadata_spec_decode_expand.cu_seqlens_q, - cu_seqlens_k_new=self.forward_metadata_spec_decode_expand.cu_seqlens_k, - max_seqlen_q=self.forward_metadata_spec_decode_expand.max_seq_len_q, + page_table=page_table, + cache_seqlens=cache_seqlens_cp, + cu_seqlens_q=cu_seqlens_q_cp, + cu_seqlens_k_new=( + cu_seqlens_k if not use_local_attn else None + ), + max_seqlen_q=max_seqlen_q_cp, softmax_scale=layer.scaling, - causal=False, - window_size=window_size, + causal=causal, softcap=layer.logit_cap, k_descale=k_descale, v_descale=v_descale, - return_softmax_lse=True, num_splits=self.num_splits, ver=self.fa_impl_ver, ) - ) - o, _ = merge_state_v2_wrapper( - o, - softmax_lse.T.contiguous(), - o_expand, - softmax_lse_expand.T.contiguous(), + + o = cp_attn_forward_extend( + forward_batch, q_fused, self.device, _mla_cp_attn ) else: - o = result + result = flash_attn_with_kvcache( + q=q_rope, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + qv=q_nope, + page_table=page_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k if not use_local_attn else None, + max_seqlen_q=max_seqlen_q, + softmax_scale=layer.scaling, + causal=False if use_cascade_attn else causal, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=use_cascade_attn, + num_splits=self.num_splits, + ver=self.fa_impl_ver, + ) + if use_cascade_attn: + o, softmax_lse, *rest = result + o_expand, softmax_lse_expand, *rest_expand = ( + flash_attn_with_kvcache( + q=q_rope, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + qv=q_nope, + page_table=self.forward_metadata_spec_decode_expand.page_table, + cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32, + cu_seqlens_q=self.forward_metadata_spec_decode_expand.cu_seqlens_q, + cu_seqlens_k_new=self.forward_metadata_spec_decode_expand.cu_seqlens_k, + max_seqlen_q=self.forward_metadata_spec_decode_expand.max_seq_len_q, + softmax_scale=layer.scaling, + causal=False, + window_size=window_size, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=True, + num_splits=self.num_splits, + ver=self.fa_impl_ver, + ) + ) + o, _ = merge_state_v2_wrapper( + o, + softmax_lse.T.contiguous(), + o_expand, + softmax_lse_expand.T.contiguous(), + ) + else: + o = result return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) @@ -1050,11 +1131,11 @@ def forward_decode( else forward_batch.encoder_out_cache_loc ) if not self.use_mla: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) else: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, @@ -1113,9 +1194,7 @@ def forward_decode( if not self.use_mla: # Do multi-head attention - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim ) @@ -1248,9 +1327,7 @@ def forward_decode( o = result else: # Do absorbed multi-latent attention - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( - q.dtype - ) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype) k_rope = kv_cache[:, :, layer.v_head_dim :] c_kv = kv_cache[:, :, : layer.v_head_dim] k_rope_cache = k_rope.view( @@ -2244,25 +2321,25 @@ def init_forward_metadata_replay_cuda_graph( metadata.page_table[:, :max_seq_pages].copy_(page_indices // self.page_size) if encoder_lens is not None: - # Only support encoder size 1 for now - metadata.encoder_max_seq_len_k = encoder_lens[0] - metadata.encoder_lens_int32.copy_(encoder_lens[:1]) - metadata.encoder_cu_seqlens_k[1:].copy_( - torch.cumsum(metadata.encoder_lens_int32, dim=0, dtype=torch.int32) + # Per-request varlen encoder support (e.g. MossVL different images). + metadata.encoder_max_seq_len_k = int(encoder_lens.max().item()) + metadata.encoder_lens_int32[:bs].copy_(encoder_lens[:bs].to(torch.int32)) + metadata.encoder_cu_seqlens_k[1 : bs + 1].copy_( + torch.cumsum(metadata.encoder_lens_int32[:bs], dim=0, dtype=torch.int32) ) - metadata.encoder_page_table[:, : metadata.encoder_max_seq_len_k].copy_( + metadata.encoder_page_table[:bs, : metadata.encoder_max_seq_len_k].copy_( self.req_to_token[req_pool_indices, : metadata.encoder_max_seq_len_k] ) - # Update the regular page table - page_table = self.req_to_token[ - req_pool_indices, - metadata.encoder_max_seq_len_k : ( - metadata.encoder_max_seq_len_k + metadata.max_seq_len_k - ), - ] - metadata.page_table[:, : metadata.max_seq_len_k].copy_(page_table) + # Self-attn (text) page_table: per-request offset = encoder_lens[i]. + text_max = metadata.max_seq_len_k + arange_text = torch.arange(text_max, device=req_pool_indices.device) + text_col = encoder_lens[:bs].long().unsqueeze(1) + arange_text.unsqueeze(0) + text_row = req_pool_indices.unsqueeze(1).expand(-1, text_max) + metadata.page_table[:bs, :text_max].copy_( + self.req_to_token[text_row, text_col] + ) self.forward_metadata = metadata self.forward_metadata_spec_decode_expand = metadata_expand diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 27705a4b8793..cd8cc2dcda02 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -126,7 +126,8 @@ def __init__( self.prefill_backend = "fa2" self.decode_backend = "fa2" - # Store multi-item scoring flag for efficient access + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.enable_mis = model_runner.server_args.enable_mis # FIXME: remove dllm workarounds from flashinfer @@ -242,12 +243,10 @@ def __init__( fmha_backend = "auto" if is_sm100_supported(): - # Disable CUTLASS backend when piecewise cuda graph is enabled - # due to TMA descriptor initialization issues on B200 if not model_runner.server_args.disable_piecewise_cuda_graph: - logger.warning( + logger.info( "CUTLASS backend is disabled when piecewise cuda graph is enabled " - "due to TMA descriptor initialization issues on B200. " + "due to TMA descriptor initialization issues on SM100 GPUs. " "Using auto backend instead for stability." ) else: @@ -802,7 +801,7 @@ def forward_extend( if k is not None: assert v is not None if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) @@ -812,7 +811,7 @@ def forward_extend( ) o = prefill_wrapper_paged.forward( q.view(-1, layer.tp_q_head_num, layer.head_dim), - forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), + self.token_to_kv_pool.get_kv_buffer(layer.layer_id), causal=causal, sm_scale=layer.scaling, # Disable sliding window attention for multi-item scoring: @@ -836,12 +835,12 @@ def forward_extend( ) else: # If `k`/`v` are not explicitly provided, fall back to the KV cache stored in - # `forward_batch.token_to_kv_pool` for this layer. This enables attention over + # `self.token_to_kv_pool` for this layer. This enables attention over # previously cached context without re-materializing KV tensors (e.g., the # IQuestLoopCoder path uses token_to_kv_pool as the KV source). if k is None and v is None: - k = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id)[0] - v = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id)[1] + k = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)[0] + v = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)[1] causal = True if ( layer.is_cross_attention @@ -875,7 +874,7 @@ def forward_extend( ) o2, s2 = prefill_wrapper_paged.forward_return_lse( q.view(-1, layer.tp_q_head_num, layer.head_dim), - forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), + self.token_to_kv_pool.get_kv_buffer(layer.layer_id), causal=False, sm_scale=layer.scaling, logits_soft_cap=logits_soft_cap, @@ -884,7 +883,7 @@ def forward_extend( o, _ = merge_state(o1, s1, o2, s2) if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) @@ -912,14 +911,14 @@ def forward_decode( if k is not None: assert v is not None if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) # Call the wrapped function o = decode_wrapper.forward( q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), - forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), + self.token_to_kv_pool.get_kv_buffer(layer.layer_id), sm_scale=layer.scaling, logits_soft_cap=layer.logit_cap, # Must use _float to avoid device-to-host copy that breaks cuda graph capture. @@ -1547,6 +1546,7 @@ def __init__( # Cached variables for generate_draft_decode_kv_indices self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] + self.req_to_token_pool = model_runner.req_to_token_pool def common_template( self, @@ -1562,7 +1562,7 @@ def common_template( (self.speculative_num_steps, num_seqs, self.topk) ]( forward_batch.req_pool_indices, - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, diff --git a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py index 601a80cea52b..61b6c49a56da 100644 --- a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py @@ -204,6 +204,10 @@ def __init__( self.max_context_len = model_runner.model_config.context_len self.device = model_runner.device self.skip_prefill = skip_prefill + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.enable_chunk_kv = ( not skip_prefill and get_global_server_args().disaggregation_mode != "decode" @@ -544,11 +548,9 @@ def forward_extend( assert v is not None if save_kv_cache: if k_rope is not None: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( - layer, cache_loc, k, k_rope - ) + self.token_to_kv_pool.set_mla_kv_buffer(layer, cache_loc, k, k_rope) else: - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) if q_rope is not None: q = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) q_rope = q_rope.view( @@ -572,9 +574,7 @@ def forward_extend( ) else: # mla paged prefill - k_buf = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( - q.dtype - ) + k_buf = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype) if q_rope is None: qall = q.view(-1, layer.tp_q_head_num, layer.head_dim) q, q_rope = ( @@ -611,14 +611,14 @@ def forward_decode( assert v is not None if save_kv_cache: if k_rope is not None: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, k_rope, ) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, @@ -636,9 +636,7 @@ def forward_decode( q_nope = reshaped_q[:, :, : layer.v_head_dim] q_rope = reshaped_q[:, :, layer.v_head_dim :] - k_buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( - q.dtype - ) + k_buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype) o = q_nope.new_empty(q_nope.shape) # Direct call to run without the wrapper @@ -944,6 +942,7 @@ def __init__( self.max_context_len = self.attn_backends[0].max_context_len # Cached variables for generate_draft_decode_kv_indices + self.req_to_token_pool = model_runner.req_to_token_pool self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] self.page_size = model_runner.server_args.page_size @@ -961,7 +960,7 @@ def common_template( (self.speculative_num_steps, num_seqs, self.topk) ]( forward_batch.req_pool_indices, - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py index 1e63f9b5cd84..c0bce60ceafd 100644 --- a/python/sglang/srt/layers/attention/flashmla_backend.py +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -410,14 +410,14 @@ def forward_decode( if k is not None: assert v is not None if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, ) bs = forward_batch.batch_size - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) reshape_q = q.view(bs, -1, layer.tp_q_head_num, layer.head_dim) if self.is_fp8_kvcache: @@ -489,10 +489,10 @@ def forward_extend( if k is not None: assert v is not None if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) bs = forward_batch.batch_size - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) reshape_q = q.view(bs, -1, layer.tp_q_head_num, layer.head_dim) if self.is_fp8_kvcache: diff --git a/python/sglang/srt/layers/attention/hip_flash_mla.py b/python/sglang/srt/layers/attention/hip_flash_mla.py index c22d4f38f0e9..ae6da641f939 100644 --- a/python/sglang/srt/layers/attention/hip_flash_mla.py +++ b/python/sglang/srt/layers/attention/hip_flash_mla.py @@ -12,10 +12,6 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs): if is_hip(): import os - from sglang.srt.layers.attention.dsa.tilelang_kernel import ( - dpsk_v4_fp8_attention_fwd, - ) - backend = os.environ.get("SGLANG_HACK_FLASHMLA_BACKEND", "tilelang") else: import flash_mla @@ -36,8 +32,19 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs): return flash_mla_with_kvcache_torch(**kwargs) if backend == "tilelang": + from sglang.srt.layers.attention.dsa.tilelang_kernel import ( + dpsk_v4_fp8_attention_fwd, + ) + return dpsk_v4_fp8_attention_fwd(**kwargs) + if backend == "triton": + from sglang.srt.layers.attention.nsa.triton_decode import ( + triton_fp8_attention_fwd, + ) + + return triton_fp8_attention_fwd(**kwargs) + if backend == "kernel": return flash_mla.flash_mla_with_kvcache(**kwargs) diff --git a/python/sglang/srt/layers/attention/hybrid_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_attn_backend.py index 69e80149e292..df0c70dc5841 100644 --- a/python/sglang/srt/layers/attention/hybrid_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_attn_backend.py @@ -23,6 +23,8 @@ def __init__( self.prefill_backend = prefill_backend self.decode_backend = decode_backend self.data_type = model_runner.kv_cache_dtype + self.token_to_kv_pool = model_runner.token_to_kv_pool + self.req_to_token_pool = model_runner.req_to_token_pool def _select_backend(self, forward_mode: ForwardMode) -> AttentionBackend: """ diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index 7b41b66e6032..a1876944d821 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -23,12 +23,6 @@ from sglang.srt.server_args import get_global_server_args from sglang.srt.speculative.eagle_info import EagleDraftInput, EagleVerifyInput from sglang.srt.speculative.spec_info import SpecInput -from sglang.srt.utils import is_cpu - -if not is_cpu(): - from sglang.srt.layers.attention.fla.chunk_delta_h import ( - CHUNK_SIZE as FLA_CHUNK_SIZE, - ) logger = logging.getLogger(__name__) @@ -143,6 +137,7 @@ def __init__(self, model_runner: ModelRunner): self.topk = model_runner.server_args.speculative_eagle_topk or 0 self.is_draft_worker = model_runner.is_draft_worker self.req_to_token_pool: HybridReqToTokenPool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.forward_metadata: ForwardMetadata = None self.state_indices_list = [] self.query_start_loc_list = [] @@ -277,9 +272,9 @@ def _init_track_conv_indices( After processing a prefill chunk, we need to save the last `conv_state_len` tokens of the processed region for prefix caching. - The key insight is that FLA (Flash Linear Attention) processes sequences in chunks - of FLA_CHUNK_SIZE. We only track the conv state up to the last complete chunk boundary - (aligned_len). + The key insight is that FLA (Flash Linear Attention) and Mamba2 processes sequences in chunks + of the chunk size (FLA_CHUNK_SIZE=64 for FLA, mamba_chunk_size for Mamba2). + We only track the conv state up to the last complete chunk boundary (aligned_len). start_indices is the starting token index of the conv state to track in this extend batch. indices include all pos to track in this extend batch, conv_state_len for each req that @@ -316,28 +311,33 @@ def _init_track_ssm_indices( Compute source and destination indices for tracking SSM states for prefix caching. After processing a prefill, we need to save the SSM recurrent state for prefix caching. - The FLA kernel outputs intermediate hidden states `h` at each chunk boundary, + The kernel outputs intermediate hidden states `h` at each chunk boundary, plus a `last_recurrent_state` at the end of the chunked prefill size. + The chunk size varies by model type: + - FLA models: FLA_CHUNK_SIZE (64) + - Mamba2 models: mamba_chunk_size (256) + The challenge is that sequences may or may not end on a chunk boundary: - - Aligned case (len % FLA_CHUNK_SIZE == 0): In this case, FLA will store the to-cache - state in the last_recurrent_state. - - Unaligned case (len % FLA_CHUNK_SIZE != 0): The last_recurrent_state includes the + - Aligned case (len % chunk_size == 0): The to-cache state is stored in + the last_recurrent_state. + - Unaligned case (len % chunk_size != 0): The last_recurrent_state includes the unaligned position, but we only want state up to the last chunk boundary. We must extract from the intermediate `h` tensor at the appropriate chunk index. We compute the src and dst indices for all requests that need to be cached (i.e. mamba_track_mask is True) based on the rule above. - For example: - 1. If chunked prefill length is < 64, then only final state has value. In this case we - cache `final` state. - 2. if chunked prefill length == 64, then only final state has value. In this case we - cache pos 64, from `final` state - 3. if chunked prefill length >64 and < 128, then both h and final state have value. - We cache pos 64 from `h` state - 4. if chunked prefill length ==128, then both h and final state have value. We cache - pos 128 from `final` state. Note `h` doesn't include the pos 128. + For example (assuming chunk_size=64): + 1. If chunked prefill length is < chunk_size, then only final state has value. + In this case we cache `final` state. + 2. If chunked prefill length == chunk_size, then only final state has value. + In this case we cache pos chunk_size, from `final` state. + 3. If chunked prefill length > chunk_size and < 2 * chunk_size, then both h and + final state have value. We cache pos chunk_size from `h` state. + 4. If chunked prefill length == 2 * chunk_size, then both h and final state have + value. We cache pos 2 * chunk_size from `final` state. Note `h` doesn't include + the final position. Returns: track_ssm_h_src: Source indices into the packed `h` tensor (for unaligned seqs) @@ -345,6 +345,7 @@ def _init_track_ssm_indices( track_ssm_final_src: Source indices into last_recurrent_state buffer (for aligned seqs) track_ssm_final_dst: Destination cache slot indices (for aligned seqs) """ + mamba_cache_chunk_size = get_global_server_args().mamba_cache_chunk_size # Move to CPU to avoid kernel launches for masking operations mamba_track_mask = forward_batch.mamba_track_mask.cpu() extend_seq_lens = forward_batch.extend_seq_lens.cpu() @@ -354,7 +355,10 @@ def _init_track_ssm_indices( prefix_lens = forward_batch.extend_prefix_lens.cpu() # Calculate the number of hidden states per request - num_h_states = (extend_seq_lens - 1) // FLA_CHUNK_SIZE + 1 + if isinstance(self, Mamba2AttnBackend): + num_h_states = extend_seq_lens // mamba_cache_chunk_size + else: + num_h_states = (extend_seq_lens - 1) // mamba_cache_chunk_size + 1 # Calculate the starting offset for each sequence in the packed batch track_ssm_src_offset = torch.zeros_like(num_h_states) @@ -367,17 +371,17 @@ def _init_track_ssm_indices( dst_masked = mamba_track_indices[mamba_track_mask] # Determine if the sequence ends at a chunk boundary - is_aligned = (lens_masked % FLA_CHUNK_SIZE) == 0 + is_aligned = (lens_masked % mamba_cache_chunk_size) == 0 # Case 1: Aligned. Use last_recurrent_state from ssm_states. track_ssm_final_src = mamba_cache_indices[mamba_track_mask][is_aligned] track_ssm_final_dst = dst_masked[is_aligned] # Case 2: Unaligned. Use intermediate state from h. - # TODO: if support FLA_CHUNK_SIZE % page size != 0, then need to modify this + # TODO: if support mamba_cache_chunk_size % page size != 0, then need to modify this not_aligned = ~is_aligned track_ssm_h_src = offset_masked[not_aligned] + ( - lens_masked[not_aligned] // FLA_CHUNK_SIZE + lens_masked[not_aligned] // mamba_cache_chunk_size ) track_ssm_h_dst = dst_masked[not_aligned] @@ -638,10 +642,10 @@ def _track_mamba_state_extend( """ Track and copy SSM states during extend for prefix caching. - After the FLA chunked prefill kernel runs, we need to save the SSM recurrent + After the chunked prefill kernel runs, we need to save the SSM recurrent state at the last chunk boundary so it can be reused for prefix caching. The source of the state depends on whether the sequence length is aligned - to FLA_CHUNK_SIZE. See `_init_track_ssm_indices` for more details on how + to the chunk size. See `_init_track_ssm_indices` for more details on how the source and destination indices are computed. Note: Conv state tracking for extend is handled separately via gather operations @@ -668,6 +672,17 @@ def __init__(self, model_runner: ModelRunner): config = model_runner.mamba2_config assert config is not None self.mamba_chunk_size = config.mamba_chunk_size + self.conv_states_shape = ( + model_runner.req_to_token_pool.mamba_pool.mamba_cache.conv[0].shape + ) + + if model_runner.server_args.enable_mamba_extra_buffer(): + assert ( + self.conv_states_shape[-1] < self.mamba_chunk_size + ), f"{self.conv_states_shape[-1]=} should be less than {self.mamba_chunk_size}" + assert ( + model_runner.server_args.mamba_track_interval >= self.mamba_chunk_size + ), f"mamba_track_interval ({model_runner.server_args.mamba_track_interval}) must be >= mamba_chunk_size ({self.mamba_chunk_size})" def init_forward_metadata(self, forward_batch: ForwardBatch): self._execute_deferred_mamba_cow_and_clear(forward_batch) @@ -725,20 +740,46 @@ def forward( hidden_states: torch.Tensor, output: torch.Tensor, layer_id: int, + forward_batch: ForwardBatch, mup_vector: Optional[torch.Tensor] = None, use_triton_causal_conv: bool = False, ): assert isinstance(self.forward_metadata, Mamba2Metadata) layer_cache = self.req_to_token_pool.mamba2_layer_cache(layer_id) - return mixer.forward( + intermediate_states = mixer.forward( hidden_states=hidden_states, output=output, layer_cache=layer_cache, metadata=self.forward_metadata, + forward_batch=forward_batch, mup_vector=mup_vector, use_triton_causal_conv=use_triton_causal_conv, ) + if forward_batch.mamba_track_mask is not None: + if ( + intermediate_states is not None + and forward_batch.mamba_track_mask is not None + and forward_batch.mamba_track_mask.any() + ): + self._track_mamba_state_extend( + forward_batch, + intermediate_states, + layer_cache.temporal, + self.forward_metadata, + ) + + if self.forward_metadata.num_decodes > 0: + num_decodes = self.forward_metadata.num_decodes + track_mamba_states_if_needed( + layer_cache.conv[0], + layer_cache.temporal, + self.forward_metadata.mamba_cache_indices[-num_decodes:], + forward_batch.mamba_track_mask[-num_decodes:], + forward_batch.mamba_track_indices[-num_decodes:], + num_decodes, + ) + def forward_decode(self, *args, **kwargs): raise NotImplementedError( "Mamba2AttnBackend's forward is called directly instead of through HybridLinearAttnBackend, as it supports mixed prefill and decode" @@ -763,6 +804,9 @@ def __init__( self.full_attn_backend = full_attn_backend self.linear_attn_backend = linear_attn_backend self.attn_backend_list = [full_attn_backend, linear_attn_backend] + # Dispatcher aliases the full-attn backend's pool refs. + self.token_to_kv_pool = full_attn_backend.token_to_kv_pool + self.req_to_token_pool = full_attn_backend.req_to_token_pool def _is_full_attn( self, layer: Optional[RadixAttention], layer_id: Optional[int] = None diff --git a/python/sglang/srt/layers/attention/intel_amx_backend.py b/python/sglang/srt/layers/attention/intel_amx_backend.py index 46b657d64c37..2f5e4141ba6a 100644 --- a/python/sglang/srt/layers/attention/intel_amx_backend.py +++ b/python/sglang/srt/layers/attention/intel_amx_backend.py @@ -19,6 +19,10 @@ def __init__(self, model_runner: ModelRunner): super().__init__() self.forward_metadata = None self.device = model_runner.device + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.num_head = ( model_runner.model_config.num_attention_heads // model_runner.tp_size @@ -105,7 +109,7 @@ def forward_extend( else forward_batch.encoder_out_cache_loc ) if save_kv_cache and k is not None and v is not None: - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) _, max_extend_len = self.forward_metadata self.extend_attention_fwd( @@ -113,9 +117,9 @@ def forward_extend( k, v, o.view(-1, layer.tp_q_head_num, layer.v_head_dim), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), - forward_batch.req_to_token_pool.req_to_token, + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, forward_batch.extend_seq_lens, @@ -152,14 +156,14 @@ def forward_decode( ) self.decode_attention_fwd( q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), o.view(-1, layer.tp_q_head_num, layer.v_head_dim), k, v, cache_loc, attn_logits, - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, layer.scaling, diff --git a/python/sglang/srt/layers/attention/linear/gdn_backend.py b/python/sglang/srt/layers/attention/linear/gdn_backend.py index 0e5453ebd0f6..d93a0f8ff60e 100644 --- a/python/sglang/srt/layers/attention/linear/gdn_backend.py +++ b/python/sglang/srt/layers/attention/linear/gdn_backend.py @@ -60,6 +60,7 @@ def __init__( ): triton_kernel = TritonGDNKernel() + cutedsl_kernel = None if decode_backend.is_triton(): self.decode_kernel = triton_kernel elif decode_backend.is_cutedsl(): @@ -69,7 +70,8 @@ def __init__( CuteDSLGDNKernel, ) - self.decode_kernel = CuteDSLGDNKernel() + cutedsl_kernel = CuteDSLGDNKernel() + self.decode_kernel = cutedsl_kernel elif decode_backend.is_flashinfer(): if not is_cuda(): raise ValueError("FlashInfer GDN backend requires CUDA") @@ -85,10 +87,26 @@ def __init__( if prefill_backend.is_triton(): self.extend_kernel = triton_kernel elif prefill_backend.is_cutedsl(): - raise ValueError( - "CuTe DSL backend only supports decode, not prefill. " - "Use --linear-attn-prefill-backend triton instead." - ) + if not is_cuda(): + raise ValueError("GDN CuTe DSL backend requires CUDA") + # Reuse the CuteDSL kernel if already created for decode + if cutedsl_kernel is None: + from sglang.srt.layers.attention.linear.kernels.gdn_cutedsl import ( + CuteDSLGDNKernel, + ) + + cutedsl_kernel = CuteDSLGDNKernel() + # The CuteDSL prefill kernel only exists on SM100+ (Blackwell). + # On SM90 (Hopper) fall back to Triton so users can pick + # `cutedsl` uniformly across hardware. + if cutedsl_kernel.supports_prefill: + self.extend_kernel = cutedsl_kernel + else: + rank0_log( + "CuTe DSL GDN prefill is not supported on this GPU " + "(requires SM100+). Falling back to Triton for prefill." + ) + self.extend_kernel = triton_kernel elif prefill_backend.is_flashinfer(): if not is_cuda(): raise ValueError("FlashInfer GDN backend requires CUDA") diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/__init__.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/__init__.py new file mode 100644 index 000000000000..7d61b18ec834 --- /dev/null +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/__init__.py @@ -0,0 +1,251 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/__init__.py + +from functools import cache + +import cutlass +import torch +import triton +from cuda.bindings.driver import CUstream +from cutlass import Int32, cute +from quack.compile_utils import make_fake_tensor + +from .kernel_h import h_cutedsl +from .kernel_kkt_inv_uw import kkt_inv_uw_cutedsl +from .kernel_o import o_cutedsl + + +class PrepMetaKernel: + def __init__(self, BT: int) -> None: + self.BT = BT + self.num_warps = 8 + + @cute.jit + def __call__( + self, + cu_seqlens: cute.Tensor, + chunk_indices: cute.Tensor, + chunk_offsets: cute.Tensor, + stream: CUstream, + ): + block = (self.num_warps * 32, 1, 1) + self.kernel( + cu_seqlens, + chunk_indices, + chunk_offsets, + ).launch(grid=(1, 1, 1), block=block, stream=stream) + + @cute.kernel + def kernel( + self, + cu_seqlens: cute.Tensor, + chunk_indices: cute.Tensor, + chunk_offsets: cute.Tensor, + ): + tid, _, _ = cute.arch.thread_idx() + warp_id = cute.arch.make_warp_uniform(tid // 32) + lane_id = tid % 32 + + num_seqs = cu_seqlens.shape[0] - 1 + num_warps = self.num_warps + tb_size = num_warps * 32 + + if tid == 0: + chunk_offsets[0] = 0 + + coarsen = cute.ceil_div(num_seqs, tb_size) + seq_start = tid * coarsen + num_iters = cutlass.min(seq_start + coarsen, num_seqs) - seq_start + + # First pass: compute this thread's total chunk count. + thread_sum = Int32(0) + for i in range(num_iters): + seq_id = seq_start + i + seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id] + thread_sum += cute.ceil_div(seqlen, self.BT) + + # warp parallel scan + cu_num_chunks = thread_sum + for i in cutlass.range_constexpr(5): + offset = cutlass.const_expr(1 << i) + lower = cute.arch.shuffle_sync_up( + cu_num_chunks, offset=offset, mask_and_clamp=0 + ) + if lane_id >= offset: + cu_num_chunks += lower + + # cross-warp cumsum (CTA-wide) + smem = cutlass.utils.SmemAllocator() + warp_num_chunks = smem.allocate_array(Int32, num_warps) + if lane_id == 31: + warp_num_chunks[warp_id] = cu_num_chunks + cute.arch.sync_threads() + + for i in cutlass.range_constexpr(1, num_warps): + if warp_id >= i: + cu_num_chunks += warp_num_chunks[i - 1] + + chunk_start = cu_num_chunks - thread_sum + + # Second pass: recompute per-sequence chunk counts and write results. + for i in range(num_iters): + seq_id = seq_start + i + seqlen = cu_seqlens[seq_id + 1] - cu_seqlens[seq_id] + num_chunks = cute.ceil_div(seqlen, self.BT) + chunk_end = chunk_start + num_chunks + chunk_offsets[seq_id + 1] = chunk_end + + for chunk_id in range(num_chunks): + chunk_indices[chunk_start + chunk_id, 0] = seq_id + chunk_indices[chunk_start + chunk_id, 1] = chunk_id + + chunk_start = chunk_end + + @cache + @staticmethod + def compile(BT: int): + cu_entries = cute.sym_int() + upper_bound_chunks = cute.sym_int() + + cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1) + chunk_indices = make_fake_tensor(Int32, (upper_bound_chunks, 2), divisibility=2) + chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1) + + kernel = PrepMetaKernel(BT) + stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + return cute.compile( + kernel, + cu_seqlens, + chunk_indices, + chunk_offsets, + stream, + options="--enable-tvm-ffi", + ) + + +def _upper_bound_chunks(num_seqs: int, total_tokens: int, chunk_size: int) -> int: + return (num_seqs - 1) + triton.cdiv(total_tokens - (num_seqs - 1), chunk_size) + + +def prepare_metadata_cutedsl( + cu_seqlens: torch.Tensor, + total_tokens: int, + chunk_size: int = 64, +) -> tuple[torch.Tensor, torch.Tensor]: + num_seqs = cu_seqlens.numel() - 1 + upper_bound_chunks = _upper_bound_chunks(num_seqs, total_tokens, chunk_size) + chunk_offsets = cu_seqlens.new_empty(num_seqs + 1, dtype=torch.int32) + chunk_indices = cu_seqlens.new_empty((upper_bound_chunks, 2), dtype=torch.int32) + + PrepMetaKernel.compile(chunk_size)(cu_seqlens, chunk_indices, chunk_offsets) + return chunk_indices, chunk_offsets + + +def chunk_gated_delta_rule_cutedsl( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + initial_state: torch.Tensor, + cu_seqlens: torch.Tensor, + chunk_indices: torch.Tensor, + chunk_offsets: torch.Tensor, + core_attn_out: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Run the GDN chunk CuteDSL prefill kernels. + + Args: + q: Query tensor with shape ``[1, T, H, K]``. + k: Key tensor with shape ``[1, T, H, K]``. + v: Value tensor with shape ``[1, T, Hv, V]``. + g: Log-space decay tensor with shape ``[1, T, Hv]``. + beta: Delta-rule beta tensor with shape ``[1, T, Hv]``. + initial_state: Recurrent state with shape ``[N, Hv, V, K]``. + cu_seqlens: Cumulative sequence lengths with shape ``[N + 1]``. + chunk_indices: Chunk index metadata with shape ``[NT, 2]``. + chunk_offsets: Cumulative chunk offsets with shape ``[N + 1]``. + core_attn_out: Optional output buffer with shape ``[T, Hv, V]``. + + Returns: + A tuple ``(output, final_state)`` where ``output`` has shape + ``[1, T, Hv, V]`` and ``final_state`` has shape ``[N, Hv, V, K]``. + When ``core_attn_out`` is provided, ``output`` is an unsqueezed view of + that buffer. + """ + q_3d = q.squeeze(0) + k_3d = k.squeeze(0) + v_3d = v.squeeze(0) + g_2d = g.squeeze(0) + beta_2d = beta.squeeze(0) + + _, _, head_k_dim = k_3d.shape + _, num_v_heads, head_v_dim = v_3d.shape + chunk_size = 64 + upper_bound_chunks = chunk_indices.shape[0] + pad_t = upper_bound_chunks * chunk_size + total_chunks_ptr = chunk_offsets[-1:] + + g_cu = torch.empty_like(g_2d, dtype=torch.float32) + u = q_3d.new_empty(pad_t, num_v_heads, head_v_dim) + w = q_3d.new_empty(pad_t, num_v_heads, head_k_dim) + + num_sms = torch.cuda.get_device_properties(q.device).multi_processor_count + kkt_inv_uw_cutedsl( + k_3d, + v_3d, + u, + w, + g_2d, + beta_2d, + g_cu, + cu_seqlens, + chunk_indices, + total_chunks_ptr, + num_sms=num_sms, + ) + + h = k_3d.new_empty( + upper_bound_chunks, + num_v_heads, + head_v_dim, + head_k_dim, + ) + v_new = q_3d.new_empty(pad_t, num_v_heads, head_v_dim) + final_state = torch.empty_like(initial_state) + h_cutedsl( + k_3d, + u, + w, + v_new, + g_cu, + h, + initial_state, + final_state, + cu_seqlens, + chunk_offsets, + ) + + output = core_attn_out if core_attn_out is not None else torch.empty_like(v_3d) + scale = head_k_dim**-0.5 + o_cutedsl( + q_3d, + k_3d, + v_new.view(upper_bound_chunks, chunk_size, num_v_heads, head_v_dim), + h, + g_cu, + output, + cu_seqlens, + chunk_indices, + total_chunks_ptr, + scale, + num_sms=num_sms, + ) + return output.unsqueeze(0), final_state + + +__all__ = [ + "chunk_gated_delta_rule_cutedsl", + "prepare_metadata_cutedsl", +] diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_h.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_h.py new file mode 100644 index 000000000000..17145b8b1b6e --- /dev/null +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_h.py @@ -0,0 +1,754 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_h.py +from functools import cache + +import cutlass +import torch +from cuda.bindings.driver import CUstream +from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute +from cutlass.cute.nvgpu import cpasync, warp +from quack.compile_utils import make_fake_tensor + +from sglang.srt.layers.attention.cute_utils import ( + EVICT_FIRST, + _tcgen05, + cvt, + fence_before_tma_store, + simple_tma_copy, +) + + +class Sm100ChunkHKernel: + """For each sequence, compute the chunk recurrent update. + + The input V tile is the U output from the KKT/UW kernel. For each chunk: + V_new = U - W @ H.T + (we actually do V_new.T = U.T - H @ W.T instead) + + H_scaled = H * exp(g_last) + V_scaled = V_new * exp(g_last - g) + H_new = H_scaled + V_scaled.T @ K + """ + + def __init__( + self, + H: int, + Hv: int, + K_dim: int, + V_dim: int, + h_dtype: cutlass.Numeric = Float32, + BT: int = 64, + num_stages: int = 2, + ) -> None: + assert Hv % H == 0 + assert K_dim == V_dim == 128 + assert BT == 64 + self.H = H + self.Hv = Hv + self.K_dim = K_dim + self.V_dim = V_dim + self.h_dtype = h_dtype + self.BT = BT + self.num_stages = num_stages + self.num_warps = 10 + + @cute.jit + def _make_bf16_tma_args( + self, + tensor: cute.Tensor, + dim: cutlass.Constexpr[int], + op: cpasync.TmaCopyOp, + stages: cutlass.Constexpr[int], + ): + swizzle_128B = cute.make_swizzle(3, 4, 3) + slayout = cute.make_layout( + (self.BT, 1, (64, dim // 64), stages), + stride=(64, 0, (1, self.BT * 64), self.BT * dim), + ) + slayout = cute.make_composed_layout(swizzle_128B, 0, slayout) + atom, tma_tensor = cpasync.make_tiled_tma_atom( + op, + cute.logical_divide(tensor, (None, None, 64)), + slayout, + cta_tiler=(self.BT, 1, dim), + ) + return atom, tma_tensor, slayout + + @cute.jit + def _make_h_tma_args(self, tensor: cute.Tensor, op: cpasync.TmaCopyOp): + # number of elements to fill 128B + num_elems = 128 // (tensor.element_type.width // 8) + swizzle_128B = cute.make_swizzle(3, 4, 3) + slayout = cute.make_layout( + (1, 1, self.V_dim, (num_elems, self.K_dim // num_elems)), + stride=(0, 0, num_elems, (1, self.V_dim * num_elems)), + ) + slayout = cute.make_composed_layout(swizzle_128B, 0, slayout) + atom, tma_tensor = cpasync.make_tiled_tma_atom( + op, + cute.logical_divide(tensor, (None, None, None, num_elems)), + slayout, + cta_tiler=(1, 1, self.V_dim, self.K_dim), + ) + return atom, tma_tensor, slayout + + @cute.jit + def __call__( + self, + K: cute.Tensor, + V: cute.Tensor, + W: cute.Tensor, + V_new: cute.Tensor, + g_cu: cute.Tensor, + h: cute.Tensor, + h0: cute.Tensor, + ht: cute.Tensor, + cu_seqlens: cute.Tensor, + chunk_offsets: cute.Tensor, + stream: CUstream, + ): + tma_g2s = cpasync.CopyBulkTensorTileG2SOp() + tma_s2g = cpasync.CopyBulkTensorTileS2GOp() + + K_args = self._make_bf16_tma_args(K, self.K_dim, tma_g2s, self.num_stages) + V_args = self._make_bf16_tma_args(V, self.V_dim, tma_g2s, self.num_stages) + W_args = self._make_bf16_tma_args(W, self.K_dim, tma_g2s, self.num_stages) + V_new_args = self._make_bf16_tma_args(V_new, self.V_dim, tma_s2g, 1) + H0_args = self._make_h_tma_args(h0, tma_g2s) + HT_args = self._make_h_tma_args(ht, tma_s2g) + H_args = self._make_h_tma_args(h, tma_s2g) + + grid = (self.Hv, h0.shape[0], 1) + block = (self.num_warps * 32, 1, 1) + self.kernel( + K_args, + V_args, + W_args, + V_new_args, + H0_args, + HT_args, + H_args, + g_cu, + cu_seqlens, + chunk_offsets, + ).launch(grid=grid, block=block, stream=stream) + + @cute.kernel + def kernel( + self, + K_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + V_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + W_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + V_new_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + H0_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + HT_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + H_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + g_cu: cute.Tensor, + cu_seqlens: cute.Tensor, + chunk_offsets: cute.Tensor, + ): + tid, _, _ = cute.arch.thread_idx() + head_id, seq_id, _ = cute.arch.block_idx() + warp_id = cute.arch.make_warp_uniform(tid // 32) + lane_id = tid % 32 + + BT = self.BT + V_dim = self.V_dim + K_dim = self.K_dim + num_stages = self.num_stages + is_f32 = self.h_dtype == Float32 + + K_tma_atom, tmaK, sK_layout = K_args + V_tma_atom, tmaV, sV_layout = V_args + W_tma_atom, tmaW, sW_layout = W_args + V_new_tma_atom, tmaV_new, sV_new_layout = V_new_args + H0_tma_atom, tmaH0, sH0_layout = H0_args + HT_tma_atom, tmaHT, _ = HT_args + H_tma_atom, tmaH, sH_layout = H_args + + def allocate_tensor(smem, dtype, layout): + return smem.allocate_tensor( + dtype, layout.outer, byte_alignment=128, swizzle=layout.inner + ) + + smem = cutlass.utils.SmemAllocator() + + # remove size=1 modes + sW = allocate_tensor(smem, BFloat16, sW_layout)[None, 0, None, None] + sV = allocate_tensor(smem, BFloat16, sV_layout)[None, 0, None, None] + sK = allocate_tensor(smem, BFloat16, sK_layout)[None, 0, None, None] + sH0 = allocate_tensor(smem, self.h_dtype, sH0_layout)[0, 0, None, None] + sH = allocate_tensor(smem, BFloat16, sH_layout)[0, 0, None, None] + sV_new = allocate_tensor(smem, BFloat16, sV_new_layout)[None, 0, None, 0] + + s_v_scale = smem.allocate_array(Float32, BT) + tma_mbar = smem.allocate_array(Int64, num_stages) + wh_in_mbar = smem.allocate_array(Int64, num_stages) + wh_done_mbar = smem.allocate_array(Int64, num_stages) + vk_in_mbar = smem.allocate_array(Int64, num_stages) + vk_done_mbar = smem.allocate_array(Int64, num_stages) + h0_mbar = smem.allocate_array(Int64, 1) + taddr = smem.allocate(Int32, 4) + + wh_tmem = 0 + vk_tmem = wh_tmem + BT + h_tmem_base = vk_tmem + K_dim + v_tmem_base = h_tmem_base + K_dim // 2 + + if warp_id == 0: + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_init(tma_mbar + i, 1) + cute.arch.mbarrier_init(wh_in_mbar + i, 256) + cute.arch.mbarrier_init(wh_done_mbar + i, 1) + cute.arch.mbarrier_init(vk_in_mbar + i, 256) + cute.arch.mbarrier_init(vk_done_mbar + i, 1) + cute.arch.mbarrier_init(h0_mbar, 1) + cute.arch.mbarrier_init_fence() + elif warp_id == 1: + cpasync.prefetch_descriptor(H0_tma_atom) + cpasync.prefetch_descriptor(W_tma_atom) + cpasync.prefetch_descriptor(V_tma_atom) + cpasync.prefetch_descriptor(K_tma_atom) + cpasync.prefetch_descriptor(HT_tma_atom) + cpasync.prefetch_descriptor(H_tma_atom) + cpasync.prefetch_descriptor(V_new_tma_atom) + cute.arch.sync_threads() + + bos = cu_seqlens[seq_id] + eos = cu_seqlens[seq_id + 1] + seqlen = eos - bos + num_chunks = cute.ceil_div(seqlen, BT) + + if warp_id == 9: + # TMA warp + stage_id = 0 + parity = 1 + + k_head_id = head_id // (self.Hv // self.H) + chunk_offset = chunk_offsets[seq_id] + + # load H0 + with cute.arch.elect_one(): + H0_size = V_dim * K_dim * self.h_dtype.width // 8 + cute.arch.mbarrier_arrive_and_expect_tx(h0_mbar, H0_size) + simple_tma_copy( + H0_tma_atom, tmaH0[seq_id, head_id, None, None], sH0, h0_mbar + ) + + # shape: ((BT, num_BT_tiles), (64, 2)) + gW_tiles = cute.logical_divide(tmaW[None, head_id, None], (BT, None)) + gV_tiles = cute.logical_divide(tmaV[None, head_id, None], (BT, None)) + gK_tiles = cute.logical_divide( + cute.domain_offset((bos, 0), tmaK[None, k_head_id, None]), + (BT, None), + ) + + for chunk_id in range(num_chunks): + mbar = tma_mbar + stage_id + gW = gW_tiles[(None, chunk_offset + chunk_id), None] + gV = gV_tiles[(None, chunk_offset + chunk_id), None] + gK = gK_tiles[(None, chunk_id), None] + + # wait for MMA to release the buffer + cute.arch.mbarrier_wait(vk_done_mbar + stage_id, parity) + + # load W, V (i.e. U), and K + with cute.arch.elect_one(): + STAGE_SIZE = BT * (K_dim + V_dim + K_dim) * 2 + cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE) + simple_tma_copy( + W_tma_atom, gW, sW[None, None, stage_id], mbar, EVICT_FIRST + ) + simple_tma_copy( + V_tma_atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST + ) + simple_tma_copy(K_tma_atom, gK, sK[None, None, stage_id], mbar) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + elif warp_id == 8: + # MMA warp + _tcgen05.alloc(taddr) + stage_id = 0 + parity = 0 + + wh_idesc = _tcgen05.make_bf16_idesc(V_dim, BT, negate_A=True) + vk_idesc = _tcgen05.make_bf16_idesc(V_dim, K_dim, transpose_B=True) + + # LBO=BT*128 is ignored for K-major + sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128) + + # when using BF16 state, H is read from smem for the 1st iteration + # variable names in this conditional branch can't be the same as those + # in the mainloop below due to CuteDSL restrictions. + if cutlass.const_expr(not is_f32): + ##### 1st MMA: V_new.T = V.T - H @ W.T ##### + Haddr0 = sH0[None, None].iterator.toint() + Waddr0 = sW[None, None, stage_id].iterator.toint() + hdesc0_base = sdesc_template | (Haddr0 >> 4) + wdesc0_base = sdesc_template | (Waddr0 >> 4) + + cute.arch.mbarrier_wait(tma_mbar + stage_id, parity) + cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity) + _tcgen05.fence_after_thread_sync() + + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(K_dim // 64): + for j in cutlass.range_constexpr(64 // 16): + hdesc0 = hdesc0_base | ((i * V_dim * 128 + j * 32) >> 4) + wdesc0 = wdesc0_base | ((i * BT * 128 + j * 32) >> 4) + _tcgen05.mma_f16(wh_tmem, hdesc0, wdesc0, wh_idesc, True) + _tcgen05.commit(wh_done_mbar + stage_id) + + ##### 2nd MMA: H_new = H + V_new.T @ K ##### + Kaddr0 = sK[None, None, stage_id].iterator.toint() + kdesc0_base = sdesc_template | (Kaddr0 >> 4) + + cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity) + _tcgen05.fence_after_thread_sync() + + with cute.arch.elect_one(): + for k in cutlass.range_constexpr(BT // 16): + vtmem0 = v_tmem_base + k * 8 + kdesc0 = kdesc0_base | ((k * 16 * 128) >> 4) + _tcgen05.mma_ts_f16(vk_tmem, vtmem0, kdesc0, vk_idesc, True) + _tcgen05.commit(vk_done_mbar + stage_id) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + num_iters = num_chunks - int(not is_f32) + for _ in range(num_iters): + ##### 1st MMA: V_new.T = V.T - H @ W.T ##### + Waddr = sW[None, None, stage_id].iterator.toint() + wdesc_base = sdesc_template | (Waddr >> 4) + + cute.arch.mbarrier_wait(tma_mbar + stage_id, parity) + cute.arch.mbarrier_wait(wh_in_mbar + stage_id, parity) + _tcgen05.fence_after_thread_sync() + + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(K_dim // 64): + for j in cutlass.range_constexpr(64 // 16): + htmem = h_tmem_base + i * 32 + j * 8 + wdesc = wdesc_base | ((i * BT * 128 + j * 32) >> 4) + _tcgen05.mma_ts_f16(wh_tmem, htmem, wdesc, wh_idesc, True) + _tcgen05.commit(wh_done_mbar + stage_id) + + ##### 2nd MMA: H_new = H + V_new.T @ K ##### + Kaddr = sK[None, None, stage_id].iterator.toint() + kdesc_base = sdesc_template | (Kaddr >> 4) + + cute.arch.mbarrier_wait(vk_in_mbar + stage_id, parity) + _tcgen05.fence_after_thread_sync() + + with cute.arch.elect_one(): + for k in cutlass.range_constexpr(BT // 16): + vtmem = v_tmem_base + k * 8 + kdesc = kdesc_base | ((k * 16 * 128) >> 4) + _tcgen05.mma_ts_f16(vk_tmem, vtmem, kdesc, vk_idesc, True) + _tcgen05.commit(vk_done_mbar + stage_id) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + elif warp_id >= 4: + # H warps + tid_ = tid % 128 + warp_id_ = warp_id % 4 + chunk_offset = chunk_offsets[seq_id] + + stage_id = 0 + vk_stage_id = 0 + vk_parity = 0 + + op = cute.nvgpu.CopyUniversalOp() + cp_16B = cute.make_copy_atom(op, Float32, num_bits_per_copy=128) + + ##### chunk_id = 0 ##### + if True: + chunk_id = 0 + end_t = min(bos + (chunk_id + 1) * BT, eos) + last_idx = end_t - 1 + h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True) + + # for 1st chunk, wait for H0 transfer from gmem + if warp_id_ == 0: + cute.arch.mbarrier_wait(h0_mbar, 0) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + + # when H0 is FP32, we need to pack it to BF16 + # also store to smem for TMA store later. + if cutlass.const_expr(is_f32): + for i in cutlass.range_constexpr(K_dim // 32): + # H0 smem layout: (V_dim, (32, K_dim/32)) + h_f32 = cute.make_rmem_tensor(32, Float32) + cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32) + + h_bf16 = cute.make_rmem_tensor(32, BFloat16) + h_bf16.store(h_f32.load().to(BFloat16)) + _tcgen05.st( + warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16 + ) + + # H smem layout: (V_dim, (64, K_dim/64)) + dst = cute.local_tile(sH[tid_, None], (32,), (i,)) + cute.copy(cp_16B, h_bf16, dst) + + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(wh_in_mbar + stage_id) + + # scale H for 2nd MMA + for i in cutlass.range_constexpr(K_dim // 32): + h_f32 = cute.make_rmem_tensor(32, Float32) + + if cutlass.const_expr(is_f32): + cute.copy(cp_16B, sH0[tid_, (None, i)], h_f32) + + else: + h_bf16 = cute.make_rmem_tensor(32, BFloat16) + sH_src = cute.local_tile(sH0[tid_, None], (32,), (i,)) + cute.copy(cp_16B, sH_src, h_bf16) + h_f32.store( + cvt.bf16x2_to_fp32x2( + cute.recast_tensor(h_bf16, Uint32) + ).load() + ) + + for j in cutlass.range_constexpr(32): + h_f32[j] *= h_scale + _tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32) + + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(vk_in_mbar + stage_id) + + # for BF16 H0, we issue TMA store from H0 smem + # for FP32 H0, we issue TMA store from H smem (after packing) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + fence_before_tma_store() + if warp_id_ == 3: + h_src = sH if cutlass.const_expr(is_f32) else sH0 + h_dst = tmaH[chunk_offset + chunk_id, head_id, None, None] + simple_tma_copy(H_tma_atom, h_src, h_dst) + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_commit_group() + + # When H0 is BF16, and there is only 1 chunk, storing + # the final state to sH0 can race before this store + # has finished. hence, we need to wait here. + if cutlass.const_expr(not is_f32): + cute.arch.cp_async_bulk_wait_group(0, read=True) + + stage_id = (stage_id + 1) % num_stages + + ##### subsequent chunks ##### + for chunk_id in range(1, num_chunks): + end_t = min(bos + (chunk_id + 1) * BT, eos) + last_idx = end_t - 1 + h_scale = cute.math.exp(g_cu[last_idx, head_id], fastmath=True) + + # wait for H from previous vk MMA + if warp_id_ == 0: + cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity) + vk_stage_id = (vk_stage_id + 1) % num_stages + if vk_stage_id == 0: + vk_parity ^= 1 + elif warp_id_ == 3: + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_wait_group(0, read=True) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + # load FP32 H from tmem, convert to BF16, store to tmem for 1st MMA, + # store to smem for TMA store later. + for i in cutlass.range_constexpr(K_dim // 32): + h_f32 = _tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32) + h_bf16 = cute.make_rmem_tensor(32, BFloat16) + h_bf16.store(h_f32.to(BFloat16)) + _tcgen05.st( + warp_id_ * 32, h_tmem_base + i * 16, "32x32b", 16, h_bf16 + ) + + # H smem layout: (V_dim, (64, K_dim/64)) + dst = cute.local_tile(sH[tid_, None], (32,), (i,)) + cute.copy(cp_16B, h_bf16, dst) + + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(wh_in_mbar + stage_id) + + # scale H for 2nd MMA + for i in cutlass.range_constexpr(K_dim // 32): + h_f32 = cute.make_rmem_tensor(32, Float32) + h_f32.store( + _tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32) + ) + for j in cutlass.range_constexpr(32): + h_f32[j] *= h_scale + _tcgen05.st(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32, h_f32) + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(vk_in_mbar + stage_id) + + # issue TMA store for O kernel + cute.arch.barrier(barrier_id=1, number_of_threads=128) + fence_before_tma_store() + if warp_id_ == 3: + h_dst = tmaH[chunk_offset + chunk_id, head_id, None, None] + simple_tma_copy(H_tma_atom, sH, h_dst) + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_commit_group() + + stage_id = (stage_id + 1) % num_stages + + # handle final state. reuse H0 smem. + if warp_id_ == 0: + cute.arch.mbarrier_wait(vk_done_mbar + vk_stage_id, vk_parity) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + for i in cutlass.range_constexpr(K_dim // 32): + h_f32 = cute.make_rmem_tensor(32, Float32) + h_f32.store(_tcgen05.ld(warp_id_ * 32, vk_tmem + i * 32, "32x32b", 32)) + + if cutlass.const_expr(is_f32): + cute.copy(cp_16B, h_f32, sH0[tid_, (None, i)]) + + else: + h_bf16 = cute.make_rmem_tensor(32, BFloat16) + h_bf16.store(h_f32.load().to(BFloat16)) + sH0_dst = cute.local_tile(sH0[tid_, None], (32,), (i,)) + cute.copy(cp_16B, h_bf16, sH0_dst) + + cute.arch.barrier(barrier_id=1, number_of_threads=128) + + if warp_id_ == 0: + ht_dst = tmaHT[seq_id, head_id, None, None] + simple_tma_copy(HT_tma_atom, sH0, ht_dst) + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_commit_group() + if warp_id_ == 1: + _tcgen05.dealloc() + + else: + # V warps + stage_id = 0 + parity = 0 + + chunk_offset = chunk_offsets[seq_id] + + ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True) + stsm_trans_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=True) + ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16) + stsm_trans_atom = cute.make_copy_atom(stsm_trans_op, BFloat16) + + # ((BT, num_BT_tiles), V_dim) + gV_new_tiles = cute.logical_divide( + tmaV_new[None, head_id, None], (BT, None) + ) + + # sV shape: [BT, (64, V_dim/64), num_stages] + # sV_view shape: [BT, (8, (8,2)), num_stages] + sV_view = cute.logical_divide(sV, (None, 8, None)) + sV_new_view = cute.logical_divide(sV_new, (None, 8)) + + # [BT, 8, num_stages] + s_col = warp_id * 4 + (lane_id // 8) + sV_view = sV_view[None, (None, s_col), None] + sV_new_view = sV_new_view[None, (None, s_col)] + + for chunk_id in range(num_chunks): + # wait for V to arrive + if warp_id == 0: + cute.arch.mbarrier_wait(tma_mbar + stage_id, parity) + cute.arch.barrier(barrier_id=2, number_of_threads=128) + + # unpack V BF16->FP32, then store to tmem for 1st MMA + # V smem layout: [BT, (64, V_dim/64)] / [BT, V_dim] + # each iteration, CTA loads [8, V_dim] tile + # (warp loads [8, 32] tile) + for i in cutlass.range_constexpr(BT // 8): + s_row = i * 8 + (lane_id % 8) + v_bf16 = cute.make_rmem_tensor(8, BFloat16) + cute.copy(ldsm_trans_atom, sV_view[s_row, None, stage_id], v_bf16) + v_fp32 = cvt.bf16x2_to_fp32x2(cute.recast_tensor(v_bf16, Uint32)) + v_fp32 = cute.logical_divide(v_fp32, 4) # (4, 2) + + tcol = wh_tmem + i * 8 + _tcgen05.st(warp_id * 32 + 0, tcol, "16x256b", 1, v_fp32[None, 0]) + _tcgen05.st(warp_id * 32 + 16, tcol, "16x256b", 1, v_fp32[None, 1]) + + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(wh_in_mbar + stage_id) + + # load g_cu for scaling + if tid < BT: + end_t = min(bos + (chunk_id + 1) * BT, eos) + last_idx = end_t - 1 + t = bos + chunk_id * BT + tid + val = Float32(0.0) + if t < eos: + val = cute.math.exp( + g_cu[last_idx, head_id] - g_cu[t, head_id], + fastmath=True, + ) + s_v_scale[tid] = val + + # wait for 1st MMA to finish + if warp_id == 2: + cute.arch.mbarrier_wait(wh_done_mbar + stage_id, parity) + elif warp_id == 3: + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_wait_group(0, read=True) + cute.arch.barrier(barrier_id=2, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + for i in cutlass.range_constexpr(BT // 8): + v_new = cute.make_rmem_tensor((4, 2), Float32) + tcol = wh_tmem + i * 8 + v_new[None, 0].store( + _tcgen05.ld(warp_id * 32 + 0, tcol, "16x256b", 1) + ) + v_new[None, 1].store( + _tcgen05.ld(warp_id * 32 + 16, tcol, "16x256b", 1) + ) + v_new_bf16 = cute.make_rmem_tensor(8, BFloat16) + v_new_bf16.store(v_new.load().to(BFloat16)) + + # scale V_new for 2nd MMA + scale0 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 0] + scale1 = s_v_scale[i * 8 + (lane_id % 4) * 2 + 1] + v_scaled = cute.make_rmem_tensor(8, Float32) + for k in cutlass.range_constexpr(4): + v_scaled[k * 2] = v_new[k * 2] * scale0 + v_scaled[k * 2 + 1] = v_new[k * 2 + 1] * scale1 + v_scaled_bf16 = v_scaled.load().to(BFloat16).reshape((4, 2)) + + # store V_new BF16 for O kernel + s_row = i * 8 + (lane_id % 8) + cute.copy(stsm_trans_atom, v_new_bf16, sV_new_view[s_row, None]) + + # store to tmem + tcol = v_tmem_base + i * 4 + _tcgen05.st( + warp_id * 32 + 0, tcol, "16x128b", 1, v_scaled_bf16[None, 0] + ) + _tcgen05.st( + warp_id * 32 + 16, tcol, "16x128b", 1, v_scaled_bf16[None, 1] + ) + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(vk_in_mbar + stage_id) + + # issue TMA store for V_new + cute.arch.barrier(barrier_id=2, number_of_threads=128) + fence_before_tma_store() + if warp_id == 3: + gV = gV_new_tiles[(None, chunk_offset + chunk_id), None] + simple_tma_copy(V_new_tma_atom, sV_new, gV) + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_commit_group() + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + @cache + @staticmethod + def compile( + H: int, + Hv: int, + K_dim: int, + V_dim: int, + h_dtype: cutlass.Numeric = Float32, + BT: int = 64, + num_stages: int = 2, + ): + total_t = cute.sym_int() + pad_t = cute.sym_int() + total_chunks_n = cute.sym_int() + num_sequences = cute.sym_int() + cu_entries = cute.sym_int() + + K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16) + V = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16) + W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16) + V_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16) + g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4) + h = make_fake_tensor( + BFloat16, (total_chunks_n, Hv, V_dim, K_dim), divisibility=16 + ) + h0 = make_fake_tensor( + h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16 + ) + ht = make_fake_tensor( + h_dtype, (num_sequences, Hv, V_dim, K_dim), divisibility=16 + ) + cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1) + chunk_offsets = make_fake_tensor(Int32, (cu_entries,), divisibility=1) + + kernel = Sm100ChunkHKernel(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages) + stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + return cute.compile( + kernel, + K, + V, + W, + V_new, + g_cu, + h, + h0, + ht, + cu_seqlens, + chunk_offsets, + stream, + options="--enable-tvm-ffi", + ) + + +def h_cutedsl( + K: torch.Tensor, + V: torch.Tensor, + W: torch.Tensor, + V_new: torch.Tensor, + g_cu: torch.Tensor, + h: torch.Tensor, + h0: torch.Tensor, + ht: torch.Tensor, + cu_seqlens: torch.Tensor, + chunk_offsets: torch.Tensor, + BT: int = 64, + num_stages: int = 2, +) -> None: + """Compute H/V_new with the same argument order as the CUDA wrapper.""" + + _, H, K_dim = K.shape + _, Hv, V_dim = V.shape + h_dtype = { + torch.bfloat16: BFloat16, + torch.float32: Float32, + }[h0.dtype] + Sm100ChunkHKernel.compile(H, Hv, K_dim, V_dim, h_dtype, BT, num_stages)( + K, + V, + W, + V_new, + g_cu, + h, + h0, + ht, + cu_seqlens, + chunk_offsets, + ) + + +h_v2b_cutedsl = h_cutedsl diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_kkt_inv_uw.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_kkt_inv_uw.py new file mode 100644 index 000000000000..21fcd509a1db --- /dev/null +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_kkt_inv_uw.py @@ -0,0 +1,823 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py +from functools import cache + +import cutlass +import torch +from cuda.bindings.driver import CUstream +from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute +from cutlass.cute.nvgpu import cpasync, warp +from quack.compile_utils import make_fake_tensor + +from sglang.srt.layers.attention.cute_utils import ( + EVICT_FIRST, + _tcgen05, + cvt, + fence_before_tma_store, + mma_bf16, + simple_tma_copy, +) + + +class Sm100ChunkUWKernel: + """Compute per-chunk KKT inverse preprocessing and U/W tiles. + + Gamma[i,j] = exp(g_cu[i] - g_cu[j]) + A = strictLower(beta * (K @ K.T) * Gamma) + Ai = inverse(I + A) + U = (Ai * beta) @ V + W = (Ai * beta * exp(g_cu)) @ K + """ + + def __init__( + self, + H: int, + Hv: int, + K_dim: int, + V_dim: int, + num_stages: int = 2, + ) -> None: + assert Hv % H == 0 + assert K_dim == V_dim == 128 + self.H = H + self.Hv = Hv + self.K_dim = K_dim + self.V_dim = V_dim + self.num_stages = num_stages + + # hard-code + self.BT = 64 + self.num_warps = 2 + 4 + 4 + + @cute.jit + def _make_tma_args( + self, + tensor: cute.Tensor, + dim: cutlass.Constexpr[int], + num_stages: int, + op: cpasync.TmaCopyOp, + ): + # logical layout: [BT, dim] + # permute for TMA: [dim/64, BT, 64] with swizzling + swizzle_128B = cute.make_swizzle(3, 4, 3) + slayout = cute.make_layout( + (self.BT, 1, (64, dim // 64), num_stages), + stride=(64, 0, (1, self.BT * 64), self.BT * dim), + ) + slayout = cute.make_composed_layout(swizzle_128B, 0, slayout) + + # we need to convert gmem layout to (T, H, (64, D/64)) for make_tiled_tma_atom() + # to emit a single 4D TMA. otherwise, it will emit (D/64)x 3D TMA. + atom, tma_tensor = cpasync.make_tiled_tma_atom( + op, + cute.logical_divide(tensor, (None, None, 64)), + slayout, + cta_tiler=(self.BT, 1, dim), + ) + return atom, tma_tensor, slayout + + @cute.jit + def __call__( + self, + K: cute.Tensor, + V: cute.Tensor, + U: cute.Tensor, + W: cute.Tensor, + g: cute.Tensor, + beta: cute.Tensor, + g_cu: cute.Tensor, + cu_seqlens: cute.Tensor, + chunk_indices: cute.Tensor, + total_chunks: cute.Tensor, + num_sms: Int32, + stream: CUstream, + ): + tma_g2s = cpasync.CopyBulkTensorTileG2SOp() + tma_s2g = cpasync.CopyBulkTensorTileS2GOp() + + K_args = self._make_tma_args(K, self.K_dim, self.num_stages, tma_g2s) + V_args = self._make_tma_args(V, self.V_dim, self.num_stages, tma_g2s) + U_args = self._make_tma_args(U, self.V_dim, 1, tma_s2g) + W_args = self._make_tma_args(W, self.K_dim, 1, tma_s2g) + + grid = (num_sms // self.Hv, self.Hv, 1) + block = (self.num_warps * 32, 1, 1) + self.kernel( + K_args, + V_args, + U_args, + W_args, + g, + beta, + g_cu, + cu_seqlens, + chunk_indices, + total_chunks, + ).launch(grid=grid, block=block, stream=stream) + + @cute.kernel + def kernel( + self, + K_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + V_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + U_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + W_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + g: cute.Tensor, + beta: cute.Tensor, + g_cu: cute.Tensor, + cu_seqlens: cute.Tensor, + chunk_indices: cute.Tensor, + total_chunks: cute.Tensor, + ): + tid, _, _ = cute.arch.thread_idx() + bid, head_id, _ = cute.arch.block_idx() + grid_x, _, _ = cute.arch.grid_dim() + + warp_id = cute.arch.make_warp_uniform(tid // 32) + lane_id = tid % 32 + k_head_id = head_id // (self.Hv // self.H) + + BT = self.BT + K_dim = self.K_dim + V_dim = self.V_dim + num_stages = self.num_stages + + K_tma_atom, tmaK, sK_layout = K_args + V_tma_atom, tmaV, sV_layout = V_args + U_tma_atom, tmaU, sU_layout = U_args + W_tma_atom, tmaW, sW_layout = W_args + + def allocate_tensor(smem, dtype, layout): + return smem.allocate_tensor( + dtype, layout.outer, byte_alignment=128, swizzle=layout.inner + ) + + smem = cutlass.utils.SmemAllocator() + sK = allocate_tensor(smem, BFloat16, sK_layout)[None, 0, None, None] + sV = allocate_tensor(smem, BFloat16, sV_layout)[None, 0, None, None] + sU = allocate_tensor(smem, BFloat16, sU_layout)[None, 0, None, 0] + sW = allocate_tensor(smem, BFloat16, sW_layout)[None, 0, None, 0] + + swizzle_128B = cute.make_swizzle(3, 4, 3) + sA_layout = cute.make_layout((BT, (64, 1)), stride=(64, (1, BT * 64))) + sA_layout = cute.make_composed_layout(swizzle_128B, 0, sA_layout) + sA = allocate_tensor(smem, BFloat16, sA_layout) + sAi = allocate_tensor(smem, BFloat16, sA_layout) + + s_beta = smem.allocate_array(Float32, BT) + s_g_cu_exp = smem.allocate_array(Float32, BT) + s_g_cu = smem.allocate_array(Float32, BT) + + tma_mbar = smem.allocate_array(Int64, num_stages) + mma_kkt_mbar = smem.allocate_array(Int64, num_stages) + inv_mbar = smem.allocate_array(Int64, num_stages) + mma_u_mbar = smem.allocate_array(Int64, num_stages) + mma_w_mbar = smem.allocate_array(Int64, num_stages) + epi_mbar = smem.allocate_array(Int64, num_stages) + taddr = smem.allocate(Int32, 4) + + kkt_tmem = 0 + U_tmem_base = kkt_tmem + BT + Ab_tmem_base = U_tmem_base + V_dim * num_stages + assert Ab_tmem_base + (BT // 2) * num_stages <= 512 + + # prepare ldmatrix/stmatrix ops + ldsm_op = warp.LdMatrix8x8x16bOp(num_matrices=4) + stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4) + ldsm_trans_op = warp.LdMatrix8x8x16bOp(num_matrices=4, transpose=True) + ldsm_atom = cute.make_copy_atom(ldsm_op, BFloat16) + stsm_atom = cute.make_copy_atom(stsm_op, BFloat16) + ldsm_trans_atom = cute.make_copy_atom(ldsm_trans_op, BFloat16) + + if warp_id == 0: + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_init(tma_mbar + i, 1) + cute.arch.mbarrier_init(mma_kkt_mbar + i, 1) + cute.arch.mbarrier_init(inv_mbar + i, 128) + cute.arch.mbarrier_init(mma_u_mbar + i, 1) + cute.arch.mbarrier_init(mma_w_mbar + i, 1) + cute.arch.mbarrier_init(epi_mbar + i, 128) + cute.arch.mbarrier_init_fence() + elif warp_id == 1: + cpasync.prefetch_descriptor(K_tma_atom) + cpasync.prefetch_descriptor(V_tma_atom) + cpasync.prefetch_descriptor(U_tma_atom) + cpasync.prefetch_descriptor(W_tma_atom) + cute.arch.sync_threads() + + num_global_chunks = total_chunks[0] + if warp_id == 9: + # TMA warp + stage_id = 0 + parity = 1 + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + seq_id = chunk_indices[global_chunk_id, 0] + chunk_id = chunk_indices[global_chunk_id, 1] + bos = cu_seqlens[seq_id] + + # since off_t is not a multiple of BT, we need to use + # domain_offset() to shift the pointer first. + mbar = tma_mbar + stage_id + gK = cute.local_tile( + cute.domain_offset((bos, 0), tmaK[None, k_head_id, None]), + tiler=(BT, K_dim), + coord=(chunk_id, 0), + ) + gV = cute.local_tile( + cute.domain_offset((bos, 0), tmaV[None, head_id, None]), + tiler=(BT, V_dim), + coord=(chunk_id, 0), + ) + + # when UW MMA is done, K and V TMA buffers are released + cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity) + + with cute.arch.elect_one(): + STAGE_SIZE = BT * (K_dim + V_dim) * 2 + cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE) + simple_tma_copy(K_tma_atom, gK, sK[None, None, stage_id], mbar) + simple_tma_copy( + V_tma_atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST + ) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + elif warp_id == 8: + # MMA warp + _tcgen05.alloc(taddr) + + stage_id = 0 + parity = 0 + + kkt_idesc = _tcgen05.make_bf16_idesc(BT, BT) + u_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True) + w_idesc = _tcgen05.make_bf16_idesc(BT, K_dim, transpose_B=True) + + # LBO=BT*128 is ignored for K-major + sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128) + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + U_tmem = U_tmem_base + V_dim * stage_id + W_tmem = U_tmem | (16 << 16) + Ab_tmem = Ab_tmem_base + (BT // 2) * stage_id + Abg_tmem = Ab_tmem | (16 << 16) + + ##### KKT MMA: KKT = K @ K.T ##### + kaddr = sK[None, None, stage_id].iterator.toint() + kdesc_base = sdesc_template | (kaddr >> 4) + + # wait for TMA data to arrive + # kkt tmem is guaranteed to be free as this is issued + # after the previous kkt's consumer (inv warps) + cute.arch.mbarrier_wait(tma_mbar + stage_id, parity) + _tcgen05.fence_after_thread_sync() + + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(K_dim // 64): + for j in cutlass.range_constexpr(64 // 16): + kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4) + _tcgen05.mma_f16( + kkt_tmem, + kdesc, + kdesc, + kkt_idesc, + (i > 0) or (j > 0), + ) + _tcgen05.commit(mma_kkt_mbar + stage_id) + + ##### U/W MMA: U = Ab @ V, W = Abg @ K ##### + vaddr = sV[None, None, stage_id].iterator.toint() + vdesc = sdesc_template | (vaddr >> 4) + kdesc = sdesc_template | (kaddr >> 4) + + # wait for epilogue to release tmem buffer + cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1) + cute.arch.mbarrier_wait(inv_mbar + stage_id, parity) + _tcgen05.fence_after_thread_sync() + + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(BT // 16): + _tcgen05.mma_ts_f16( + W_tmem, Abg_tmem + i * 8, kdesc, w_idesc, i > 0 + ) + kdesc += (16 * 128) >> 4 + _tcgen05.commit(mma_w_mbar + stage_id) + + for i in cutlass.range_constexpr(BT // 16): + _tcgen05.mma_ts_f16( + U_tmem, Ab_tmem + i * 8, vdesc, u_idesc, i > 0 + ) + vdesc += (16 * 128) >> 4 + _tcgen05.commit(mma_u_mbar + stage_id) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + cute.arch.mbarrier_wait(epi_mbar + stage_id, parity ^ 1) + _tcgen05.dealloc() + + elif warp_id >= 4: + # inv warps + tid_ = tid % 128 + warp_id_ = warp_id % 4 + + stage_id = 0 + parity = 0 + + # view into (16,16) sub-tiles, then ldmatrix layout + sA_ldsm = cute.logical_divide(sA, (16, cute.make_layout((8, 2)))) + sAi_ldsm = cute.logical_divide(sAi, (16, cute.make_layout((8, 2)))) + sA_ldsm = sA_ldsm[(lane_id % 16, None), ((None, lane_id // 16), None)] + sAi_ldsm = sAi_ldsm[(lane_id % 16, None), ((None, lane_id // 16), None)] + + # init Ai smem buffer with zeros (only the first 48 rows) + for i in cutlass.range_constexpr((BT // 4 * 3) * BT // 128): + idx = i * 128 + tid_ + sAi[idx // BT, idx % BT] = BFloat16(0.0) + + # indices for ldmatrix layout later + row_indices = cute.make_rmem_tensor((1, 2, 1), Int32) + row_indices[0, 0, 0] = warp_id_ * 16 + (lane_id // 4) + row_indices[0, 1, 0] = warp_id_ * 16 + (lane_id // 4) + 8 + row_indices = row_indices.load() + + col_indices = cute.make_rmem_tensor((2, 1, 2), Int32) + col_indices[0, 0, 0] = (lane_id % 4) * 2 + 0 + col_indices[1, 0, 0] = (lane_id % 4) * 2 + 1 + col_indices[0, 0, 1] = (lane_id % 4) * 2 + 8 + col_indices[1, 0, 1] = (lane_id % 4) * 2 + 9 + col_indices = col_indices.load() + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + seq_id = chunk_indices[global_chunk_id, 0] + chunk_id = chunk_indices[global_chunk_id, 1] + bos = cu_seqlens[seq_id] + eos = cu_seqlens[seq_id + 1] + off_t = bos + chunk_id * BT + + t = off_t + tid_ + + ##### Phase 1: load g and beta ##### + if tid_ < BT: + in_bounds = t < eos + beta_val = beta[t, head_id] if in_bounds else Float32(0.0) + g_val = g[t, head_id] if in_bounds else Float32(0.0) + + s_beta[tid_] = beta_val + + # compute cumsum(g) + # parallel scan within a warp + for i in cutlass.range_constexpr(5): + offset = cutlass.const_expr(1 << i) + lower = cute.arch.shuffle_sync_up( + g_val, offset, mask_and_clamp=0 + ) + if lane_id >= offset: + g_val += lower + + # store warp sum + if lane_id == 31: + s_g_cu[warp_id_] = g_val + cute.arch.barrier(barrier_id=3, number_of_threads=BT) + + # add warp sum from lower warps + for i in cutlass.range_constexpr(1, BT // 32): + if warp_id_ >= i: + g_val += s_g_cu[i - 1] + cute.arch.barrier(barrier_id=3, number_of_threads=BT) + + # store g_cu to gmem for H and O kernels + if in_bounds: + g_cu[t, head_id] = g_val + + # store g and g_cu to smem for later + s_g_cu[tid_] = g_val + s_g_cu_exp[tid_] = cute.math.exp(g_val) if in_bounds else 0.0 + + ##### Phase 2: A = strictLower(beta * kkt * Gamma) ##### + if warp_id_ == 0: + cute.arch.mbarrier_wait(mma_kkt_mbar + stage_id, parity) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + # tmem 16x256b layout / ldmatrix layout + # mode0 is 8 rows together + # mode1 is top and bottom 8 rows + # mode2 is groups of 16 rows + row_coord = (lane_id // 4, None, warp_id_) + s_beta_view = cute.make_tensor(s_beta, (8, 2, 4)) + beta_row = s_beta_view[row_coord].load().reshape((1, 2, 1)) + + s_g_cu_view = cute.make_tensor(s_g_cu, (8, 2, 4)) + g_cu_row = s_g_cu_view[row_coord].load().reshape((1, 2, 1)) + + # mode0 is 2 consecutive elems + # mode1 is top and bottom 8 rows + # mode2 is next 8 columns + # mode3 is repeating that 16x16 tile pattern + kkt = _tcgen05.ld(kkt_tmem, 0, "16x256b", BT // 8) + kkt = kkt.reshape((2, 2, 2, BT // 16)) + + for i in cutlass.range_constexpr(BT // 16): + # mode0 is 2 elems next to each other + # mode1 is 4 pairs of elems on 1 row + # mode2 is top and bottom 8 rows + # mode3 is next 16 columns + col_coord = (None, lane_id % 4, None, i) + s_g_cu_view = cute.make_tensor(s_g_cu, (2, 4, 2, BT // 16)) + g_cu_col = s_g_cu_view[col_coord].load().reshape((2, 1, 2)) + + Gamma = cute.math.exp(g_cu_row - g_cu_col, fastmath=True) + A = kkt[None, None, None, i] * beta_row * Gamma + + # strict lower mask + # NOTE: for OOB t position, s_beta is filled with zeros. + # hence, we don't need to apply bounds check for columns. + A_masked = cute.where(row_indices > col_indices + i * 16, A, 0.0) + + # pack to BF16 + # CuteDSL doesn't generate cvt.bf16x2.f32 here for some reasons + packed = cute.make_rmem_tensor(4, Uint32) + packed[0] = cvt.fp32x2_to_bf16x2( + A_masked[0, 0, 0], A_masked[1, 0, 0] + ) + packed[1] = cvt.fp32x2_to_bf16x2( + A_masked[0, 1, 0], A_masked[1, 1, 0] + ) + packed[2] = cvt.fp32x2_to_bf16x2( + A_masked[0, 0, 1], A_masked[1, 0, 1] + ) + packed[3] = cvt.fp32x2_to_bf16x2( + A_masked[0, 1, 1], A_masked[1, 1, 1] + ) + + # store to smem + cute.copy( + stsm_atom, + cute.recast_tensor(packed, BFloat16), + sA_ldsm[warp_id_, None, i], + ) + + cute.arch.barrier(barrier_id=1, number_of_threads=128) + + ##### Phase 3: matrix inverse ##### + # we use Newton-Schulz iterations to compute the inverse + # of the four 16x16 diagonal blocks. + # Ai_new = 2 Ai - Ai @ M @ Ai + # where M = I + A + # + # we do this with 2 MMAs: + # 1. -AiM = Ai @ (-M) + # 2. Ai_new = 2 Ai + (-AiM) @ Ai + zeros_f32 = cute.make_rmem_tensor(4, Float32) + zeros_f32.fill(0.0) + + def set_diagonal(A: cute.Tensor, lane_id: Int32): + "Set the diagonal to 1s" + if lane_id % 9 == 0: + A[0] = (A[0] & Uint32(0xFFFF0000)) | Uint32(0x00003F80) + A[3] = (A[3] & Uint32(0xFFFF0000)) | Uint32(0x00003F80) + elif lane_id % 9 == 4: + A[0] = (A[0] & Uint32(0x0000FFFF)) | Uint32(0x3F800000) + A[3] = (A[3] & Uint32(0x0000FFFF)) | Uint32(0x3F800000) + + Ai_bf16 = cute.make_rmem_tensor(8, BFloat16) + mma_B_bf16 = cute.make_rmem_tensor(8, BFloat16) + M_bf16 = cute.make_rmem_tensor(8, BFloat16) + acc = cute.make_rmem_tensor((4, 2), Float32) + + # share the same storage + Ai = cute.recast_tensor(Ai_bf16, Uint32) + mma_B = cute.logical_divide(cute.recast_tensor(mma_B_bf16, Uint32), 2) + M = cute.logical_divide(cute.recast_tensor(M_bf16, Uint32), 2) + + # initial guess: Ai = I-A + cute.copy(ldsm_atom, sA_ldsm[warp_id_, None, warp_id_], Ai_bf16) + for i in cutlass.range_constexpr(4): + Ai[i] ^= Uint32(0x80008000) # negate A + set_diagonal(Ai, lane_id) + + # (4, 2) + Ai_f32 = cute.logical_divide(cvt.bf16x2_to_fp32x2(Ai), 4) + + # M is holding -(I+A), stay constant throughout the iterations + cute.copy(ldsm_trans_atom, sA_ldsm[warp_id_, None, warp_id_], M_bf16) + set_diagonal(M, lane_id) + for i in cutlass.range_constexpr(4): + M[i] ^= Uint32(0x80008000) + + # 3 rounds of Newton-Schulz + for _ in cutlass.range_constexpr(3): + # First MMA: -AiM = Ai @ (-M) + cute.copy(stsm_atom, Ai_bf16, sA_ldsm[warp_id_, None, warp_id_]) + cute.arch.sync_warp() + acc[None, 0] = mma_bf16(Ai, M[None, 0], zeros_f32) + acc[None, 1] = mma_bf16(Ai, M[None, 1], zeros_f32) + Ai_bf16.store(acc.load().to(BFloat16)) + + # Second MMA: Ai_new = 2Ai + (-AiM) @ Ai + for j in cutlass.range_constexpr(8): + Ai_f32[j] *= 2.0 + cute.copy( + ldsm_trans_atom, + sA_ldsm[warp_id_, None, warp_id_], + mma_B_bf16, + ) + Ai_f32[None, 0] = mma_bf16(Ai, mma_B[None, 0], Ai_f32[None, 0]) + Ai_f32[None, 1] = mma_bf16(Ai, mma_B[None, 1], Ai_f32[None, 1]) + Ai_bf16.store(Ai_f32.load().to(BFloat16)) + + cute.copy(stsm_atom, Ai_bf16, sAi_ldsm[warp_id_, None, warp_id_]) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + + # off-diagonal by 1 + # Ai[i,i-1] = -Ai[i,i] @ A[i,i-1] @ Ai[i-1,i-1]. + if warp_id_ > 0: + neg_Ai = cute.make_rmem_tensor(4, Uint32) + for i in cutlass.range_constexpr(4): + neg_Ai[i] = Ai[i] ^ Uint32(0x80008000) + + cute.copy( + ldsm_trans_atom, + sA_ldsm[warp_id_, None, warp_id_ - 1], + mma_B_bf16, + ) + acc[None, 0] = mma_bf16(neg_Ai, mma_B[None, 0], zeros_f32) + acc[None, 1] = mma_bf16(neg_Ai, mma_B[None, 1], zeros_f32) + Ai_bf16.store(acc.load().to(BFloat16)) + + cute.copy( + ldsm_trans_atom, + sAi_ldsm[warp_id_ - 1, None, warp_id_ - 1], + mma_B_bf16, + ) + acc[None, 0] = mma_bf16(Ai, mma_B[None, 0], zeros_f32) + acc[None, 1] = mma_bf16(Ai, mma_B[None, 1], zeros_f32) + Ai_bf16.store(acc.load().to(BFloat16)) + cute.copy( + stsm_atom, + Ai_bf16, + sAi_ldsm[warp_id_, None, warp_id_ - 1], + ) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + + # off-diagonal by 2 + if warp_id_ < 2: + cute.copy( + ldsm_atom, + sA_ldsm[warp_id_ + 2, None, warp_id_], + Ai_bf16, + ) + cute.copy( + ldsm_trans_atom, + sAi_ldsm[warp_id_, None, warp_id_], + mma_B_bf16, + ) + acc[None, 0] = mma_bf16(Ai, mma_B[None, 0], zeros_f32) + acc[None, 1] = mma_bf16(Ai, mma_B[None, 1], zeros_f32) + + cute.copy( + ldsm_atom, + sA_ldsm[warp_id_ + 2, None, warp_id_ + 1], + Ai_bf16, + ) + cute.copy( + ldsm_trans_atom, + sAi_ldsm[warp_id_ + 1, None, warp_id_], + mma_B_bf16, + ) + acc[None, 0] = mma_bf16(Ai, mma_B[None, 0], acc[None, 0]) + acc[None, 1] = mma_bf16(Ai, mma_B[None, 1], acc[None, 1]) + + tmp = cute.make_rmem_tensor(8, BFloat16) + tmp.store(acc.load().to(BFloat16)) + cute.copy(stsm_atom, tmp, sAi_ldsm[warp_id_ + 2, None, warp_id_]) + cute.arch.sync_warp() + + cute.copy( + ldsm_atom, sAi_ldsm[warp_id_ + 2, None, warp_id_ + 2], Ai_bf16 + ) + for i in cutlass.range_constexpr(4): + Ai[i] ^= Uint32(0x80008000) + cute.copy( + ldsm_trans_atom, + sAi_ldsm[warp_id_ + 2, None, warp_id_], + mma_B_bf16, + ) + acc[None, 0] = mma_bf16(Ai, mma_B[None, 0], zeros_f32) + acc[None, 1] = mma_bf16(Ai, mma_B[None, 1], zeros_f32) + tmp.store(acc.load().to(BFloat16)) + cute.copy(stsm_atom, tmp, sAi_ldsm[warp_id_ + 2, None, warp_id_]) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + + # off-diagonal by 3 + if warp_id_ == 0: + cute.copy(ldsm_atom, sA_ldsm[3, None, 0], Ai_bf16) + cute.copy(ldsm_trans_atom, sAi_ldsm[0, None, 0], mma_B_bf16) + acc[None, 0] = mma_bf16(Ai, mma_B[None, 0], zeros_f32) + acc[None, 1] = mma_bf16(Ai, mma_B[None, 1], zeros_f32) + + for i in cutlass.range_constexpr(1, 3): + cute.copy(ldsm_atom, sA_ldsm[3, None, i], Ai_bf16) + cute.copy(ldsm_trans_atom, sAi_ldsm[i, None, 0], mma_B_bf16) + acc[None, 0] = mma_bf16(Ai, mma_B[None, 0], acc[None, 0]) + acc[None, 1] = mma_bf16(Ai, mma_B[None, 1], acc[None, 1]) + + tmp = cute.make_rmem_tensor(8, BFloat16) + tmp.store(acc.load().to(BFloat16)) + cute.copy(stsm_atom, tmp, sAi_ldsm[3, None, 0]) + cute.arch.sync_warp() + + cute.copy(ldsm_atom, sAi_ldsm[3, None, 3], Ai_bf16) + for i in cutlass.range_constexpr(4): + Ai[i] ^= Uint32(0x80008000) + cute.copy(ldsm_trans_atom, sAi_ldsm[3, None, 0], mma_B_bf16) + acc[None, 0] = mma_bf16(Ai, mma_B[None, 0], zeros_f32) + acc[None, 1] = mma_bf16(Ai, mma_B[None, 1], zeros_f32) + tmp.store(acc.load().to(BFloat16)) + cute.copy(stsm_atom, tmp, sAi_ldsm[3, None, 0]) + + ##### Phase 4: compute Ab, Abg ##### + if warp_id_ == 3: + cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity ^ 1) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + + for i in cutlass.range_constexpr(BT // 16): + cute.copy(ldsm_atom, sAi_ldsm[warp_id_, None, i], Ai_bf16) + + col_coord = (None, lane_id % 4, None, i) + s_beta_view = cute.make_tensor(s_beta, (2, 4, 2, BT // 16)) + beta_col = s_beta_view[col_coord].load().reshape((2, 1, 2)) + + s_g_cu_view = cute.make_tensor(s_g_cu_exp, (2, 4, 2, BT // 16)) + g_cu_col = s_g_cu_view[col_coord].load().reshape((2, 1, 2)) + + Ai_f32 = cvt.bf16x2_to_fp32x2(Ai).load().reshape((2, 2, 2)) + + Ab_f32 = Ai_f32 * beta_col + Ab = Ab_f32.to(BFloat16) + Ab_tmem = Ab_tmem_base + (BT // 2) * stage_id + i * 8 + _tcgen05.st(warp_id_ * 32, Ab_tmem, "16x128b", 2, Ab) + + Abg_f32 = Ab_f32 * g_cu_col + Abg = Abg_f32.to(BFloat16) + _tcgen05.st(warp_id_ * 32 + 16, Ab_tmem, "16x128b", 2, Abg) + + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(inv_mbar + stage_id) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + elif warp_id < 4: + # epi warps + stage_id = 0 + parity = 0 + + # ((BT, num_global_chunks), V_dim) + gU_tiles = cute.logical_divide(tmaU[None, head_id, None], (BT, None)) + gW_tiles = cute.logical_divide(tmaW[None, head_id, None], (BT, None)) + + # sW shape: [BT, (64, K_dim/64)] + # sW_view shape: [(8, 2), (4, K_dim/64)] + s_row = warp_id * 16 + lane_id % 16 # select the rows of [16,16] tile + sW_view = cute.zipped_divide( + sW[s_row, None], + tiler=cute.make_layout((8, 2)), + ) + sU_view = cute.zipped_divide( + sU[s_row, None], + tiler=cute.make_layout((8, 2)), + ) + + # select the 8 columns within [16,16] tile + sW_view = sW_view[(None, lane_id // 16), None] + sU_view = sU_view[(None, lane_id // 16), None] + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + # wait for W MMA + previous TMA store to finish + U_tmem = U_tmem_base + V_dim * stage_id + if warp_id == 0: + cute.arch.mbarrier_wait(mma_w_mbar + stage_id, parity) + elif warp_id == 1: + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_wait_group(0, read=True) + cute.arch.barrier(barrier_id=2, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + w_f32 = _tcgen05.ld(warp_id * 32 + 16, U_tmem, "16x256b", K_dim // 8) + _tcgen05.wait_ld() + w_bf16 = cute.make_rmem_tensor((8, K_dim // 16), BFloat16) + w_bf16.store(w_f32.to(BFloat16)) + cute.copy(stsm_atom, w_bf16, sW_view) + + # wait for U MMA + issue W TMA store + cute.arch.barrier(barrier_id=2, number_of_threads=128) + fence_before_tma_store() + if warp_id == 0: + cute.arch.mbarrier_wait(mma_u_mbar + stage_id, parity) + elif warp_id == 1: + # don't need to commit + simple_tma_copy( + W_tma_atom, sW, gW_tiles[(None, global_chunk_id), None] + ) + cute.arch.barrier(barrier_id=2, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + u_f32 = _tcgen05.ld(warp_id * 32, U_tmem, "16x256b", V_dim // 8) + _tcgen05.wait_ld() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(epi_mbar + stage_id) + u_bf16 = cute.make_rmem_tensor((8, V_dim // 16), BFloat16) + u_bf16.store(u_f32.to(BFloat16)) + cute.copy(stsm_atom, u_bf16, sU_view) + + cute.arch.barrier(barrier_id=2, number_of_threads=128) + fence_before_tma_store() + if warp_id == 1: + simple_tma_copy( + U_tma_atom, sU, gU_tiles[(None, global_chunk_id), None] + ) + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_commit_group() + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + @cache + @staticmethod + def compile(H: int, Hv: int, K_dim: int, V_dim: int, num_stages: int = 2): + total_t = cute.sym_int() + pad_t = cute.sym_int() + total_chunks_n = cute.sym_int() + num_sequences = cute.sym_int() + + K = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16) + V = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16) + U = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16) + W = make_fake_tensor(BFloat16, (pad_t, Hv, K_dim), divisibility=16) + g = make_fake_tensor(Float32, (total_t, Hv), divisibility=4) + beta = make_fake_tensor(Float32, (total_t, Hv), divisibility=4) + g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4) + cu_seqlens = make_fake_tensor(Int32, (num_sequences,), divisibility=1) + chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2) + total_chunks = make_fake_tensor(Int32, (1,), divisibility=1) + + kernel = Sm100ChunkUWKernel(H, Hv, K_dim, V_dim, num_stages) + stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + return cute.compile( + kernel, + K, + V, + U, + W, + g, + beta, + g_cu, + cu_seqlens, + chunk_indices, + total_chunks, + Int32(148), + stream, + options="--enable-tvm-ffi", + ) + + +def kkt_inv_uw_cutedsl( + K: torch.Tensor, + V: torch.Tensor, + U: torch.Tensor, + W: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + g_cu: torch.Tensor, + cu_seqlens: torch.Tensor, + chunk_indices: torch.Tensor, + total_chunks: torch.Tensor, + num_sms: int = 148, +) -> None: + _, Hv, V_dim = V.shape + _, H, K_dim = K.shape + + Sm100ChunkUWKernel.compile(H, Hv, K_dim, V_dim)( + K, + V, + U, + W, + g, + beta, + g_cu, + cu_seqlens, + chunk_indices, + total_chunks, + num_sms, + ) diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_o.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_o.py new file mode 100644 index 000000000000..3665cc19e105 --- /dev/null +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_blackwell/kernel_o.py @@ -0,0 +1,631 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_o.py +from functools import cache + +import cutlass +import torch +from cuda.bindings.driver import CUstream +from cutlass import BFloat16, Float32, Int32, Int64, Uint32, cute +from cutlass.cute.nvgpu import cpasync, warp +from quack.compile_utils import make_fake_tensor + +from sglang.srt.layers.attention.cute_utils import ( + EVICT_FIRST, + _tcgen05, + cvt, + fence_before_tma_store, + simple_tma_copy, +) + + +class Sm100ChunkOKernel: + """Compute per-token output from recurrent and intra-chunk terms. + + Gamma[i,j] = exp(g_cu[i] - g_cu[j]) + P = mask((Q @ K.T) * Gamma) + O = scale * (exp(g_cu) * (Q @ H.T) + P @ V) + """ + + def __init__( + self, + H: int, + Hv: int, + K_dim: int, + V_dim: int, + BT: int = 64, + num_stages: int = 2, + ) -> None: + assert Hv % H == 0 + assert K_dim == 128 + assert V_dim == 128 + assert BT == 64 + self.H = H + self.Hv = Hv + self.K_dim = K_dim + self.V_dim = V_dim + self.BT = BT + self.num_stages = num_stages + self.num_warps = 10 + + @cute.jit + def _make_bf16_tma_args( + self, + tensor: cute.Tensor, + dim: cutlass.Constexpr[int], + op: cpasync.TmaCopyOp, + stages: cutlass.Constexpr[int], + ): + swizzle_128B = cute.make_swizzle(3, 4, 3) + slayout = cute.make_layout( + (self.BT, 1, (64, dim // 64), stages), + stride=(64, 0, (1, self.BT * 64), self.BT * dim), + ) + slayout = cute.make_composed_layout(swizzle_128B, 0, slayout) + atom, tma_tensor = cpasync.make_tiled_tma_atom( + op, + cute.logical_divide(tensor, (None, None, 64)), + slayout, + cta_tiler=(self.BT, 1, dim), + ) + return atom, tma_tensor, slayout + + @cute.jit + def _make_h_tma_args( + self, + tensor: cute.Tensor, + op: cpasync.TmaCopyOp, + stages: cutlass.Constexpr[int], + ): + num_elems = 128 // (tensor.element_type.width // 8) + swizzle_128B = cute.make_swizzle(3, 4, 3) + slayout = cute.make_layout( + (1, self.V_dim, (num_elems, self.K_dim // num_elems), stages), + stride=(0, num_elems, (1, self.V_dim * num_elems), self.V_dim * self.K_dim), + ) + slayout = cute.make_composed_layout(swizzle_128B, 0, slayout) + atom, tma_tensor = cpasync.make_tiled_tma_atom( + op, + cute.logical_divide(tensor, (None, None, num_elems)), + slayout, + cta_tiler=(1, self.V_dim, self.K_dim), + ) + return atom, tma_tensor, slayout + + @cute.jit + def __call__( + self, + q: cute.Tensor, + k: cute.Tensor, + v_new_chunks: cute.Tensor, + h: cute.Tensor, + g_cu: cute.Tensor, + o: cute.Tensor, + cu_seqlens: cute.Tensor, + chunk_indices: cute.Tensor, + total_chunks: cute.Tensor, + scale: Float32, + num_sms: Int32, + stream: CUstream, + ): + grid = (num_sms // self.Hv, self.Hv, 1) + block = (self.num_warps * 32, 1, 1) + tma_g2s = cpasync.CopyBulkTensorTileG2SOp() + tma_s2g = cpasync.CopyBulkTensorTileS2GOp() + Q_args = self._make_bf16_tma_args(q, self.K_dim, tma_g2s, self.num_stages) + K_args = self._make_bf16_tma_args(k, self.K_dim, tma_g2s, self.num_stages) + V_args = self._make_bf16_tma_args( + v_new_chunks, self.V_dim, tma_g2s, self.num_stages + ) + H_args = self._make_h_tma_args(h, tma_g2s, self.num_stages) + O_args = self._make_bf16_tma_args(o, self.V_dim, tma_s2g, 1) + self.kernel( + Q_args, + K_args, + V_args, + H_args, + O_args, + g_cu, + o, + cu_seqlens, + chunk_indices, + total_chunks, + scale, + ).launch(grid=grid, block=block, stream=stream) + + @cute.kernel + def kernel( + self, + Q_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + K_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + V_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + H_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + O_args: tuple[cute.CopyAtom, cute.Tensor, cute.ComposedLayout], + g_cu: cute.Tensor, + o: cute.Tensor, + cu_seqlens: cute.Tensor, + chunk_indices: cute.Tensor, + total_chunks: cute.Tensor, + scale: Float32, + ): + tid, _, _ = cute.arch.thread_idx() + bid, v_head_id, _ = cute.arch.block_idx() + grid_x, _, _ = cute.arch.grid_dim() + warp_id = cute.arch.make_warp_uniform(tid // 32) + lane_id = tid % 32 + + BT = self.BT + K_dim = self.K_dim + V_dim = self.V_dim + num_stages = self.num_stages + + heads_per_qk = self.Hv // self.H + k_head_id = v_head_id // heads_per_qk + num_global_chunks = total_chunks[0] + + Q_tma_atom, tmaQ, sQ_layout = Q_args + K_tma_atom, tmaK, sK_layout = K_args + V_tma_atom, tmaV, sV_layout = V_args + H_tma_atom, tmaH, sH_layout = H_args + O_tma_atom, tmaO, sO_layout = O_args + + def allocate_tensor(smem, dtype, layout): + return smem.allocate_tensor( + dtype, layout.outer, byte_alignment=128, swizzle=layout.inner + ) + + smem = cutlass.utils.SmemAllocator() + sQ = allocate_tensor(smem, BFloat16, sQ_layout)[None, 0, None, None] + sK = allocate_tensor(smem, BFloat16, sK_layout)[None, 0, None, None] + sV = allocate_tensor(smem, BFloat16, sV_layout)[None, 0, None, None] + sH = allocate_tensor(smem, BFloat16, sH_layout)[0, None, None, None] + sO = allocate_tensor(smem, BFloat16, sO_layout)[None, 0, None, 0] + + s_g_cu = smem.allocate_array(Float32, BT) + qk_full_mbar = smem.allocate_array(Int64, num_stages) + hv_full_mbar = smem.allocate_array(Int64, num_stages) + qk_empty_mbar = smem.allocate_array(Int64, num_stages) + pv_mma_mbar = smem.allocate_array(Int64, num_stages) + qk_mbar = smem.allocate_array(Int64, 1) + mask_mbar = smem.allocate_array(Int64, 1) + epi_mbar = smem.allocate_array(Int64, 1) + taddr = smem.allocate(Int32, 4) + + qk_tmem = 0 + p_tmem = 64 + out_tmem = 128 + qh_tmem = 256 + + if warp_id == 0: + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_init(qk_full_mbar + i, 1) + cute.arch.mbarrier_init(qk_empty_mbar + i, 1) + cute.arch.mbarrier_init(hv_full_mbar + i, 1) + cute.arch.mbarrier_init(pv_mma_mbar + i, 1) + cute.arch.mbarrier_init(qk_mbar, 1) + cute.arch.mbarrier_init(mask_mbar, 128) + cute.arch.mbarrier_init(epi_mbar, 128) + cute.arch.mbarrier_init_fence() + elif warp_id == 9: + cpasync.prefetch_descriptor(Q_tma_atom) + cpasync.prefetch_descriptor(K_tma_atom) + cpasync.prefetch_descriptor(V_tma_atom) + cpasync.prefetch_descriptor(H_tma_atom) + cute.arch.sync_threads() + + if warp_id == 9: + # TMA warp + stage_id = 0 + parity = 1 + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + seq_id = chunk_indices[global_chunk_id, 0] + chunk_id = chunk_indices[global_chunk_id, 1] + bos = cu_seqlens[seq_id] + + # copy Q and K + q_tile = cute.local_tile( + cute.domain_offset((bos, 0), tmaQ[None, k_head_id, None]), + tiler=(BT, K_dim), + coord=(chunk_id, 0), + ) + k_tile = cute.local_tile( + cute.domain_offset((bos, 0), tmaK[None, k_head_id, None]), + tiler=(BT, K_dim), + coord=(chunk_id, 0), + ) + mbar = qk_full_mbar + stage_id + + cute.arch.mbarrier_wait(qk_empty_mbar + stage_id, parity) + + with cute.arch.elect_one(): + STAGE_SIZE = BT * (K_dim + K_dim) * 2 + cute.arch.mbarrier_arrive_and_expect_tx(mbar, STAGE_SIZE) + simple_tma_copy(Q_tma_atom, q_tile, sQ[None, None, stage_id], mbar) + simple_tma_copy(K_tma_atom, k_tile, sK[None, None, stage_id], mbar) + + # copy H and V + gH = tmaH[global_chunk_id * self.Hv + v_head_id, None, None] + gV = cute.local_tile( + tmaV[None, v_head_id, None], + tiler=(BT, V_dim), + coord=(global_chunk_id, 0), + ) + mbar = hv_full_mbar + stage_id + + cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, parity) + + with cute.arch.elect_one(): + H_STAGE_SIZE = V_dim * K_dim * 2 + V_STAGE_SIZE = BT * V_dim * 2 + cute.arch.mbarrier_arrive_and_expect_tx( + mbar, H_STAGE_SIZE + V_STAGE_SIZE + ) + simple_tma_copy( + H_tma_atom, gH, sH[None, None, stage_id], mbar, EVICT_FIRST + ) + simple_tma_copy( + V_tma_atom, gV, sV[None, None, stage_id], mbar, EVICT_FIRST + ) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + parity ^= 1 + + elif warp_id == 8: + # MMA warp + _tcgen05.alloc(taddr) + + # LBO=BT*128 is ignored for K-major + sdesc_template = _tcgen05.make_sdesc_128B_swizzle(BT * 128) + qk_idesc = _tcgen05.make_bf16_idesc(BT, BT) + qh_idesc = _tcgen05.make_bf16_idesc(BT, V_dim) + pv_idesc = _tcgen05.make_bf16_idesc(BT, V_dim, transpose_B=True) + + stage_id = 0 + tma_parity = 0 + mask_parity = 0 + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + qaddr = sQ[None, None, stage_id].iterator.toint() + kaddr = sK[None, None, stage_id].iterator.toint() + haddr = sH[None, None, stage_id].iterator.toint() + vaddr = sV[None, None, stage_id].iterator.toint() + qdesc_base = sdesc_template | (qaddr >> 4) + kdesc_base = sdesc_template | (kaddr >> 4) + hdesc_base = sdesc_template | (haddr >> 4) + vdesc_base = sdesc_template | (vaddr >> 4) + + ##### 1st MMA: Q @ K.T ##### + # do this first to unblock mask(QK) + cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1) + cute.arch.mbarrier_wait(qk_full_mbar + stage_id, tma_parity) + _tcgen05.fence_after_thread_sync() + + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(K_dim // BT): + for j in cutlass.range_constexpr(BT // 16): + qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4) + kdesc = kdesc_base | ((i * BT * 128 + j * 32) >> 4) + _tcgen05.mma_f16( + qk_tmem, qdesc, kdesc, qk_idesc, (i > 0) or (j > 0) + ) + _tcgen05.commit(qk_mbar) + + ##### 2nd MMA: Q @ H.T ##### + cute.arch.mbarrier_wait(hv_full_mbar + stage_id, tma_parity) + _tcgen05.fence_after_thread_sync() + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(K_dim // BT): + for j in cutlass.range_constexpr(BT // 16): + qdesc = qdesc_base | ((i * BT * 128 + j * 32) >> 4) + hdesc = hdesc_base | ((i * V_dim * 128 + j * 32) >> 4) + _tcgen05.mma_f16( + qh_tmem, qdesc, hdesc, qh_idesc, (i > 0) or (j > 0) + ) + _tcgen05.commit(qk_empty_mbar + stage_id) + + ##### 3rd MMA: P @ V ##### + # stalled by mask(QK) + cute.arch.mbarrier_wait(mask_mbar, mask_parity) + _tcgen05.fence_after_thread_sync() + with cute.arch.elect_one(): + for i in cutlass.range_constexpr(BT // 16): + vdesc = vdesc_base | ((i * 16 * 128) >> 4) + _tcgen05.mma_ts_f16( + out_tmem, p_tmem + i * 8, vdesc, pv_idesc, i > 0 + ) + _tcgen05.commit(pv_mma_mbar + stage_id) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + tma_parity ^= 1 + mask_parity ^= 1 + + # wait for epilogue to finish for deallocation + cute.arch.mbarrier_wait(epi_mbar, mask_parity ^ 1) + _tcgen05.dealloc() + + elif warp_id >= 4: + # masking warps + warp_id_ = warp_id % 4 + tid_ = tid % 128 + row0 = warp_id_ * 16 + lane_id // 4 + row1 = row0 + 8 + + parity = 0 + + # for ldmatrix layout later + row_indices = cute.make_rmem_tensor(2, Int32) + row_indices[0] = warp_id_ * 16 + lane_id // 4 + row_indices[1] = warp_id_ * 16 + lane_id // 4 + 8 + row_indices = row_indices.load().reshape((1, 2)) + + col_indices = cute.make_rmem_tensor(2, Int32) + col_indices[0] = (lane_id % 4) * 2 + col_indices[1] = (lane_id % 4) * 2 + 1 + col_indices = col_indices.load().reshape((2, 1)) + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + if tid_ < BT: + seq_id = chunk_indices[global_chunk_id, 0] + chunk_id = chunk_indices[global_chunk_id, 1] + bos = cu_seqlens[seq_id] + eos = cu_seqlens[seq_id + 1] + + t_ = bos + chunk_id * BT + tid_ + s_g_cu[tid_] = g_cu[t_, v_head_id] if t_ < eos else Float32(0.0) + + # wait for QK MMA + if warp_id_ == 0: + cute.arch.mbarrier_wait(qk_mbar, parity) + cute.arch.barrier(barrier_id=1, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + qk = _tcgen05.ld(warp_id_ * 32, qk_tmem, "16x256b", BT // 8) + qk = qk.reshape((2, 2, BT // 8)) + _tcgen05.wait_ld() + + g_cu_rows = cute.make_rmem_tensor(2, Float32) + g_cu_rows[0] = s_g_cu[row0] + g_cu_rows[1] = s_g_cu[row1] + g_cu_rows = g_cu_rows.load().reshape((1, 2)) + + for i in cutlass.range_constexpr(BT // 8): + col = i * 8 + (lane_id % 4) * 2 + g_cu_cols = cute.make_rmem_tensor(2, Float32) + g_cu_cols[0] = s_g_cu[col] + g_cu_cols[1] = s_g_cu[col + 1] + g_cu_cols = g_cu_cols.load().reshape((2, 1)) + + # apply gamma and causal mask + Gamma = cute.math.exp(g_cu_rows - g_cu_cols, fastmath=True) + tmp = qk[None, None, i] * Gamma + tmp = cute.where(row_indices >= col_indices + i * 8, tmp, 0.0) + + # CuteDSL can't emit cvt.bf16x2.f32 here + attn_lo = cute.make_rmem_tensor(2, Uint32) + attn_lo[0] = cvt.fp32x2_to_bf16x2(tmp[0, 0], tmp[1, 0]) + attn_lo[1] = cvt.fp32x2_to_bf16x2(tmp[0, 1], tmp[1, 1]) + _tcgen05.st(warp_id_ * 32, p_tmem + i * 4, "16x128b", 1, attn_lo) + + _tcgen05.wait_st() + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(mask_mbar) + + parity ^= 1 + + else: + # epilogue warps + # for ldmatrix layout later + row0 = warp_id * 16 + lane_id // 4 + row1 = row0 + 8 + + stage_id = 0 + mma_parity = 0 + + op = cute.nvgpu.CopyUniversalOp() + cp_4B = cute.make_copy_atom(op, BFloat16, num_bits_per_copy=32) + stsm_op = warp.StMatrix8x8x16bOp(num_matrices=4, transpose=False) + stsm_atom = cute.make_copy_atom(stsm_op, BFloat16) + + # ldmatrix layout + # [total_seq_len, ((2, 4, WIDTH/8), V_DIM/WIDTH)] + WIDTH = 64 + o_view = cute.logical_divide( + o[None, v_head_id, None], + (None, cute.make_layout((2, 4, WIDTH // 8))), + ) + # select lane: [total_seq_len, 2, WIDTH/8, V_DIM/WIDTH] + o_view = o_view[None, ((None, lane_id % 4, None), None)] + + for global_chunk_id in range(bid, num_global_chunks, grid_x): + seq_id = chunk_indices[global_chunk_id, 0] + chunk_id = chunk_indices[global_chunk_id, 1] + bos = cu_seqlens[seq_id] + eos = cu_seqlens[seq_id + 1] + chunk_start = bos + chunk_id * BT + full_chunk = chunk_start + BT <= eos + + g_cu_rows = cute.make_rmem_tensor(2, Float32) + g_cu_rows.fill(0.0) + + # load g_cu + if chunk_start + row0 < eos: + g_cu_rows[0] = cute.math.exp( + g_cu[chunk_start + row0, v_head_id], fastmath=True + ) + if chunk_start + row1 < eos: + g_cu_rows[1] = cute.math.exp( + g_cu[chunk_start + row1, v_head_id], fastmath=True + ) + g_cu_rows = g_cu_rows.load().reshape((1, 2, 1)) + + if warp_id == 0: + cute.arch.mbarrier_wait(pv_mma_mbar + stage_id, mma_parity) + elif warp_id == 3 and full_chunk: + cute.arch.cp_async_bulk_wait_group(0, read=True) + cute.arch.barrier(barrier_id=2, number_of_threads=128) + _tcgen05.fence_after_thread_sync() + + if full_chunk: + # use TMA store: tmem->rmem->smem->gmem + for i in cutlass.range_constexpr(V_dim // WIDTH): + qh = _tcgen05.ld( + warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8 + ) + pv = _tcgen05.ld( + warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8 + ) + _tcgen05.wait_ld() + if i == V_dim // WIDTH - 1: + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(epi_mbar) + + qh = qh.reshape((2, 2, WIDTH // 8)) + pv = pv.reshape((2, 2, WIDTH // 8)) + + out_f32 = scale * (g_cu_rows * qh + pv) + out_bf16 = cute.make_rmem_tensor((8, WIDTH // 16), BFloat16) + out_bf16.store(out_f32.to(BFloat16).reshape((8, WIDTH // 16))) + + # TODO: issue single cute.copy() + for j in cutlass.range_constexpr(WIDTH // 16): + s_row = warp_id * 16 + lane_id % 16 + s_col = i * (WIDTH // 8) + j * 2 + lane_id // 16 + sO_tile = cute.local_tile(sO[s_row, None], (8,), (s_col,)) + cute.copy(stsm_atom, out_bf16[None, j], sO_tile) + + cute.arch.barrier(barrier_id=2, number_of_threads=128) + fence_before_tma_store() + if warp_id == 3: + gO = cute.local_tile( + cute.domain_offset((bos, 0), tmaO[None, v_head_id, None]), + tiler=(BT, V_dim), + coord=(chunk_id, 0), + ) + simple_tma_copy(O_tma_atom, sO, gO) + with cute.arch.elect_one(): + cute.arch.cp_async_bulk_commit_group() + + else: + # direct gmem store + # TODO: explore doing multiple 1D TMAs + for i in cutlass.range_constexpr(V_dim // WIDTH): + qh = _tcgen05.ld( + warp_id * 32, qh_tmem + i * WIDTH, "16x256b", WIDTH // 8 + ) + pv = _tcgen05.ld( + warp_id * 32, out_tmem + i * WIDTH, "16x256b", WIDTH // 8 + ) + _tcgen05.wait_ld() + if i == V_dim // WIDTH - 1: + _tcgen05.fence_before_thread_sync() + cute.arch.mbarrier_arrive(epi_mbar) + + qh = qh.reshape((2, 2, WIDTH // 8)) + pv = pv.reshape((2, 2, WIDTH // 8)) + + out_f32 = scale * (g_cu_rows * qh + pv) + out_bf16 = cute.make_rmem_tensor((2, 2, WIDTH // 8), BFloat16) + out_bf16.store(out_f32.to(BFloat16)) + + if chunk_start + row0 < eos: + cute.copy( + cp_4B, + out_bf16[None, 0, None], + o_view[chunk_start + row0, None, None, i], + ) + if chunk_start + row1 < eos: + cute.copy( + cp_4B, + out_bf16[None, 1, None], + o_view[chunk_start + row1, None, None, i], + ) + + stage_id = (stage_id + 1) % num_stages + if stage_id == 0: + mma_parity ^= 1 + + @cache + @staticmethod + def compile( + H: int, + Hv: int, + K_dim: int, + V_dim: int, + BT: int = 64, + num_stages: int = 2, + ): + total_t = cute.sym_int() + pad_t = cute.sym_int() + total_chunks_n = cute.sym_int() + h_outer_n = cute.sym_int() + cu_entries = cute.sym_int() + + q = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16) + k = make_fake_tensor(BFloat16, (total_t, H, K_dim), divisibility=16) + v_new = make_fake_tensor(BFloat16, (pad_t, Hv, V_dim), divisibility=16) + h_flat = make_fake_tensor(BFloat16, (h_outer_n, V_dim, K_dim), divisibility=16) + g_cu = make_fake_tensor(Float32, (total_t, Hv), divisibility=4) + o = make_fake_tensor(BFloat16, (total_t, Hv, V_dim), divisibility=16) + cu_seqlens = make_fake_tensor(Int32, (cu_entries,), divisibility=1) + chunk_indices = make_fake_tensor(Int32, (total_chunks_n, 2), divisibility=2) + total_chunks = make_fake_tensor(Int32, (1,), divisibility=1) + + kernel = Sm100ChunkOKernel( + H, + Hv, + K_dim, + V_dim, + BT, + num_stages, + ) + stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + return cute.compile( + kernel, + q, + k, + v_new, + h_flat, + g_cu, + o, + cu_seqlens, + chunk_indices, + total_chunks, + Float32(1.0), + Int32(148), + stream, + options="--enable-tvm-ffi", + ) + + +def o_cutedsl( + q: torch.Tensor, + k: torch.Tensor, + v_new_chunks: torch.Tensor, + h: torch.Tensor, + g_cu: torch.Tensor, + o: torch.Tensor, + cu_seqlens: torch.Tensor, + chunk_indices: torch.Tensor, + total_chunks: torch.Tensor, + scale: float, + num_sms: int = 148, +) -> None: + _, H, K_dim = q.shape + _, Hv, V_dim = o.shape + + Sm100ChunkOKernel.compile(H, Hv, K_dim, V_dim)( + q, + k, + v_new_chunks.view(-1, Hv, V_dim), + h.view(-1, V_dim, K_dim), + g_cu, + o, + cu_seqlens, + chunk_indices, + total_chunks, + float(scale), + num_sms, + ) diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py index fff4ef9015d6..311f40093a9a 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py @@ -1,3 +1,15 @@ +"""CuTe DSL kernels for GDN (Gated Delta Network) linear attention. + +Decode path uses the existing ``cutedsl_fused_sigmoid_gating_delta_rule_update`` +(works on SM90+). + +Prefill (extend) path uses the ported vLLM SM100 chunkwise kernel +(``chunk_gated_delta_rule_cutedsl``). Requires SM100+ and ``head_k_dim == 128``. +""" + +import logging +from typing import Optional + import torch from sglang.jit_kernel.cutedsl_gdn import cutedsl_fused_sigmoid_gating_delta_rule_update @@ -5,9 +17,64 @@ LinearAttnKernelBase, ) +logger = logging.getLogger(__name__) + + +def _is_blackwell() -> bool: + """True iff running on SM100+ (Blackwell) where the ported kernel is valid.""" + if not torch.cuda.is_available(): + return False + major, _ = torch.cuda.get_device_capability() + return major >= 10 + class CuteDSLGDNKernel(LinearAttnKernelBase): - """CuTe DSL kernel for GDN decode (CUDA only).""" + """CuTe DSL kernel for GDN. + + Decode: ``cutedsl_fused_sigmoid_gating_delta_rule_update`` (SM90+). + Extend (prefill): chunkwise ``chunk_gated_delta_rule_cutedsl`` + (SM100+ only, ``head_k_dim`` must be 128). On SM90 the prefill path is + unsupported; callers should query :attr:`supports_prefill` and fall back + to another backend (e.g. Triton). + """ + + def __init__(self): + # The Blackwell extend kernel uses tcgen05/TMA-bulk-swizzle features + # that don't exist on SM90. The decode kernel does work on SM90+. + self.supports_prefill = _is_blackwell() + + # Heavy CuteDSL imports are deferred to extend() so SM90 boxes can + # still construct the kernel just for decode. + self._extend_fn: Optional[callable] = None + self._prepare_meta_fn: Optional[callable] = None + self._l2norm_fn: Optional[callable] = None + + def _ensure_extend_loaded(self, head_k_dim: int) -> None: + if self._extend_fn is not None: + return + if not self.supports_prefill: + major = ( + torch.cuda.get_device_capability()[0] + if torch.cuda.is_available() + else -1 + ) + raise RuntimeError( + f"CuTe DSL GDN prefill requires SM100+ (Blackwell); got SM{major}." + ) + if head_k_dim != 128: + raise RuntimeError( + f"CuTe DSL GDN prefill requires head_k_dim=128, got {head_k_dim}." + ) + from sglang.srt.layers.attention.fla.l2norm import l2norm_fwd + from sglang.srt.layers.attention.linear.kernels.gdn_blackwell import ( + chunk_gated_delta_rule_cutedsl, + prepare_metadata_cutedsl, + ) + + self._extend_fn = chunk_gated_delta_rule_cutedsl + self._prepare_meta_fn = prepare_metadata_cutedsl + self._l2norm_fn = l2norm_fwd + logger.info("Using CuTe DSL GDN prefill (Blackwell)") def decode( self, @@ -40,8 +107,69 @@ def decode( softplus_threshold=20.0, ) - def extend(self, *args, **kwargs): - raise NotImplementedError("CuteDSLGDNKernel only supports decode") + def extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + *, + ssm_states: torch.Tensor, + cache_indices: torch.Tensor, + query_start_loc: torch.Tensor, + **kwargs, + ) -> tuple: + head_k_dim = k.shape[-1] + self._ensure_extend_loaded(head_k_dim) + + total_seq_len = q.shape[1] + num_v_heads = v.shape[2] + head_v_dim = v.shape[3] + + # L2 norm Q/K outside the kernel (same as flashinfer path). + q_norm = self._l2norm_fn(q[0].contiguous()).unsqueeze(0) + k_norm = self._l2norm_fn(k[0].contiguous()).unsqueeze(0) + v_in = v[0].contiguous().unsqueeze(0) + # Kernel expects log-space float32 gate per (token, v-head). + g_in = g[0].to(torch.float32).unsqueeze(0) + beta_in = beta[0].to(torch.float32).unsqueeze(0) + + cu_seqlens = query_start_loc.to(torch.int32) + + # Pool gather: remap padding (-1) to the last (sentinel) slot. + ssm_cache_indices = torch.where( + cache_indices >= 0, + cache_indices, + ssm_states.shape[0] - 1, + ).to(torch.long) + initial_state = ssm_states[ssm_cache_indices].contiguous() + + chunk_indices, chunk_offsets = self._prepare_meta_fn( + cu_seqlens, total_seq_len, chunk_size=64 + ) + + output, final_state = self._extend_fn( + q=q_norm, + k=k_norm, + v=v_in, + g=g_in, + beta=beta_in, + initial_state=initial_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + ) + + ssm_states.index_copy_( + 0, + ssm_cache_indices, + final_state.to(ssm_states.dtype), + ) + + # Match Triton extend interface: (output, last_recurrent_state, h). + # We've already written state back, so no need to return it. + return output, None, None def target_verify(self, *args, **kwargs): - raise NotImplementedError("CuteDSLGDNKernel only supports decode") + raise NotImplementedError("CuteDSLGDNKernel does not support target_verify") diff --git a/python/sglang/srt/layers/attention/mamba/mamba.py b/python/sglang/srt/layers/attention/mamba/mamba.py index 1d48809caa4e..1c46a35d4f81 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba.py +++ b/python/sglang/srt/layers/attention/mamba/mamba.py @@ -26,6 +26,7 @@ ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.mem_cache.memory_pool import MambaPool +from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import ( composed_weight_loader, sharded_weight_loader, @@ -410,6 +411,7 @@ def forward( output: torch.Tensor, layer_cache: MambaPool.State, metadata: Mamba2Metadata, + forward_batch: ForwardBatch, mup_vector: Optional[torch.Tensor] = None, use_triton_causal_conv: bool = False, ): @@ -420,6 +422,7 @@ def forward( state_indices_tensor = metadata.mamba_cache_indices conv_state = layer_cache.conv[0] ssm_state = layer_cache.temporal + intermediate_states = None query_start_loc = metadata.query_start_loc @@ -517,6 +520,14 @@ def forward( x = hidden_states_B_C_p.transpose( 0, 1 ) # this is the form that causal-conv see + if ( + forward_batch.mamba_track_mask is not None + and forward_batch.mamba_track_mask.any() + and metadata.track_conv_indices is not None + ): + x_to_track = x[:, metadata.track_conv_indices].transpose(0, 1) + mask_indices = forward_batch.mamba_track_mask.nonzero(as_tuple=True)[0] + conv_state[forward_batch.mamba_track_indices[mask_indices]] = x_to_track ccfn = ( causal_conv1d_fn if not use_triton_causal_conv @@ -546,7 +557,7 @@ def forward( ) # NOTE: final output is an in-place update of out tensor - varlen_state = mamba_chunk_scan_combined( + intermediate_states, varlen_state = mamba_chunk_scan_combined( hidden_states_p.view( 1, num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim ), @@ -565,6 +576,7 @@ def forward( initial_states=initial_states, return_varlen_states=True, return_final_states=False, + return_intermediate_states=True, dt_softplus=True, dt_limit=(0.0, float("inf")), out=preallocated_ssm_out_p.view( @@ -708,6 +720,8 @@ def forward( # 5. Final linear projection output[:num_actual_tokens], _ = self.out_proj(hidden_states) + return intermediate_states + @property def mamba_type(self) -> str: return "mamba2" diff --git a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py index c009a1016f7b..e2e04fd7e890 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py +++ b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py @@ -171,6 +171,11 @@ def prepare_decode( retrieve_next_token=forward_metadata.retrieve_next_token, retrieve_next_sibling=forward_metadata.retrieve_next_sibling, retrieve_parent_token=forward_metadata.retrieve_parent_token, + track_conv_indices=forward_metadata.track_conv_indices, + track_ssm_h_src=forward_metadata.track_ssm_h_src, + track_ssm_h_dst=forward_metadata.track_ssm_h_dst, + track_ssm_final_src=forward_metadata.track_ssm_final_src, + track_ssm_final_dst=forward_metadata.track_ssm_final_dst, num_decodes=len(seq_lens), num_prefills=0, num_prefill_tokens=0, @@ -248,6 +253,11 @@ def prepare_mixed( retrieve_next_token=forward_metadata.retrieve_next_token, retrieve_next_sibling=forward_metadata.retrieve_next_sibling, retrieve_parent_token=forward_metadata.retrieve_parent_token, + track_conv_indices=forward_metadata.track_conv_indices, + track_ssm_h_src=forward_metadata.track_ssm_h_src, + track_ssm_h_dst=forward_metadata.track_ssm_h_dst, + track_ssm_final_src=forward_metadata.track_ssm_final_src, + track_ssm_final_dst=forward_metadata.track_ssm_final_dst, num_prefills=num_prefills, num_prefill_tokens=num_prefill_tokens, num_decodes=num_decodes, diff --git a/python/sglang/srt/layers/attention/nsa/triton_decode/__init__.py b/python/sglang/srt/layers/attention/nsa/triton_decode/__init__.py new file mode 100644 index 000000000000..7762b8bd2cd0 --- /dev/null +++ b/python/sglang/srt/layers/attention/nsa/triton_decode/__init__.py @@ -0,0 +1,98 @@ +""" +Triton-based sparse attention decode kernels for DeepSeek V4. + +This package provides an alternative to the tilelang implementation, +controlled by the environment variable SGLANG_HACK_FLASHMLA_BACKEND=triton. +""" + +from typing import Optional, Tuple + +import torch + +from sglang.srt.layers.attention.nsa.triton_decode.triton_mla_kernels_decode_optimized import ( + triton_sparse_attn_decode, +) + + +class _KVScopeAdapter: + """Lightweight adapter providing the kv_scope interface expected by + ``triton_sparse_attn_decode``. + + The Triton kernels access four fields: + * ``blocked_k_quantized`` – the raw FP8 KV cache tensor. + * ``blocked_k`` – only ``blocked_k.shape[1]`` (block size) + is read, so we reuse the same tensor. + * ``indices_in_kvcache`` – sparse top-k page indices. + * ``topk_length`` – valid length per batch element. + """ + + __slots__ = [ + "blocked_k", + "blocked_k_quantized", + "indices_in_kvcache", + "topk_length", + ] + + def __init__( + self, + k_cache: torch.Tensor, + indices: torch.Tensor, + topk_length: Optional[torch.Tensor], + ): + self.blocked_k_quantized = k_cache + self.blocked_k = k_cache + self.indices_in_kvcache = indices + self.topk_length = topk_length + + +def triton_fp8_attention_fwd( + q: torch.Tensor, + k_cache: torch.Tensor, + head_dim_v: int, + softmax_scale: float, + indices: torch.Tensor, + attn_sink: Optional[torch.Tensor] = None, + extra_k_cache: Optional[torch.Tensor] = None, + extra_indices_in_kvcache: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + extra_topk_length: Optional[torch.Tensor] = None, + **_unused, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Sparse MLA decode via Triton kernels. + + Accepts the same ``**kwargs`` dict that the caller builds for + ``flash_mla_with_kvcache`` / ``dpsk_v4_fp8_attention_fwd``, but only + uses the subset of arguments relevant to the Triton implementation. + Unused keys (``block_table``, ``cache_seqlens``, + ``tile_scheduler_metadata``, ``num_splits``, ``causal``, + ``is_fp8_kvcache``) are silently ignored via ``**_unused``. + + Returns: + ``(output, lse)`` where *output* has shape + ``[batch, seq_len, num_heads, head_dim_v]`` and *lse* has shape + ``[batch, seq_len, num_heads]``. + """ + kv_scope = _KVScopeAdapter(k_cache, indices, topk_length) + + extra_kv_scope = None + if extra_k_cache is not None: + extra_kv_scope = _KVScopeAdapter( + extra_k_cache, + extra_indices_in_kvcache, + extra_topk_length, + ) + + output, lse = triton_sparse_attn_decode( + q=q, + kv_scope=kv_scope, + extra_kv_scope=extra_kv_scope, + sm_scale=softmax_scale, + d_v=head_dim_v, + attn_sink=attn_sink, + ) + + # Triton kernel returns lse as (b, h_q, s_q); transpose to + # (b, s_q, h_q) to match the tilelang / flash_mla convention. + lse = lse.transpose(1, 2) + + return output, lse diff --git a/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_common.py b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_common.py new file mode 100644 index 000000000000..ac9c1cced658 --- /dev/null +++ b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_common.py @@ -0,0 +1,585 @@ +""" +Common utilities and attention kernels for Triton MLA Decode. + +This module contains shared code for the DeepSeek V4 Triton decode implementation: +- Attention kernels (unified sparse decode) +- Helper functions for chunked attention +- Token range computation for memory-based chunking +""" + +from typing import List, Tuple + +import torch +import triton +import triton.language as tl + +LOG2E = tl.constexpr(1.4426950408889634) + + +# ============================================================================ +# Bucketing for autotune keys to avoid recompilation per unique batch size +# ============================================================================ +def _bucket_total_tokens(total_tokens: int) -> int: + """Round total_tokens up to the nearest power of 2 for autotune key stability. + + In serving, total_tokens (= batch_size * seq_len) varies with every batch. + Using the exact value as an autotune key causes recompilation for each unique + value. Bucketing to powers of 2 limits the number of unique keys to ~15, + dramatically reducing autotuning overhead. + + Returns: + Power-of-2 bucket: 1, 2, 4, 8, ..., up to the next power of 2. + """ + if total_tokens <= 0: + return 1 + # Round up to next power of 2 + n = 1 + while n < total_tokens: + n <<= 1 + return n + + +# ============================================================================ +# Helper function to compute workload size category for autotune +# ============================================================================ +def _get_workload_size_category(total_tokens: int, topk: int) -> int: + """ + Compute workload size category for autotune key. + Returns: + 0: small (< 10K elements) + 1: medium (10K - 100K elements) + 2: large (100K - 1M elements) + 3: very large (> 1M elements) + """ + total_elements = total_tokens * topk + if total_elements < 10000: + return 0 + elif total_elements < 100000: + return 1 + elif total_elements < 1000000: + return 2 + else: + return 3 + + +# ============================================================================ +# Unified Attention Kernels +# ============================================================================ + + +# ============================================================================ +# CDNA4 (gfx950) Optimized: Added high-performance configs for MI355X +# Best config for h_q=128, large topk: BLOCK_H=64, BLOCK_N=256, num_warps=8 +# ============================================================================ +@triton.autotune( + configs=[ + # Selected based on CDNA4 architecture analysis: + # - BLOCK_D=128 is fixed (matches KV tile structure for d_qk=512). + # - BLOCK_N=256: best for amortizing memory access over topk dimension. + # (decode attention is memory-bound; larger BLOCK_N = fewer iterations) + # - num_warps=8: memory-bound decode benefits from more warps for latency hiding. + # - BLOCK_H varies to cover different batch sizes: + # * BLOCK_H=16: cdiv(128,16)=8 H-blocks, best for small batches (bs=1-8) + # * BLOCK_H=32: cdiv(128,32)=4 H-blocks, good for medium batches (bs=8-32) + # * BLOCK_H=64: cdiv(128,64)=2 H-blocks, best for large batches (bs=32+) + # (original comment: "Best for h_q=128, large topk") + # * BLOCK_H=128: cdiv(128,128)=1 H-block, for very large batches (bs=128+) + triton.Config( + {"BLOCK_H": 16, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=1 + ), + triton.Config( + {"BLOCK_H": 32, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=1 + ), + triton.Config( + {"BLOCK_H": 64, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=1 + ), + triton.Config( + {"BLOCK_H": 128, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=1 + ), + ], + key=["total_tokens_bucket", "h_q", "total_topk", "d_qk"], +) +@triton.jit +def _unified_sparse_decode_kernel( + Q, + KV, + Mask, + AttnSink, + Output, + LSE, + sm_scale, + total_tokens, + total_tokens_bucket, + h_q, + total_topk, + d_qk, + d_v, + stride_q_t, + stride_q_h, + stride_q_d, + stride_kv_t, + stride_kv_k, + stride_kv_d, + stride_mask_t, + stride_mask_k, + stride_o_t, + stride_o_h, + stride_o_d, + stride_lse_t, + stride_lse_h, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Unified attention kernel with single KV buffer (int64 safe).""" + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + pid_t_64 = pid_t.to(tl.int64) + + NEG_INF = float("-inf") + POS_INF = float("+inf") + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + + m_i = tl.full([BLOCK_H], NEG_INF, dtype=tl.float32) + l_i = tl.zeros([BLOCK_H], dtype=tl.float32) + + acc_0 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_1 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_2 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_3 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + + stride_q_t_64 = tl.cast(stride_q_t, tl.int64) + stride_kv_t_64 = tl.cast(stride_kv_t, tl.int64) + stride_mask_t_64 = tl.cast(stride_mask_t, tl.int64) + q_base = Q + pid_t_64 * stride_q_t_64 + kv_base = KV + pid_t_64 * stride_kv_t_64 + mask_base = Mask + pid_t_64 * stride_mask_t_64 + + for n_start in range(0, total_topk, BLOCK_N): + offs_n = n_start + tl.arange(0, BLOCK_N) + mask_n = offs_n < total_topk + + mask_ptrs = mask_base + offs_n * stride_mask_k + invalid = tl.load(mask_ptrs, mask=mask_n, other=True) + valid = mask_n & ~invalid + + qk = tl.zeros([BLOCK_H, BLOCK_N], dtype=tl.float32) + + for d_start in range(0, d_qk, BLOCK_D): + offs_d = d_start + tl.arange(0, BLOCK_D) + mask_d = offs_d < d_qk + + q_ptrs = ( + q_base + offs_h[:, None] * stride_q_h + offs_d[None, :] * stride_q_d + ) + q_chunk = tl.load( + q_ptrs, mask=mask_h[:, None] & mask_d[None, :], other=0.0 + ).to(tl.bfloat16) + + k_ptrs = ( + kv_base + offs_n[:, None] * stride_kv_k + offs_d[None, :] * stride_kv_d + ) + k_chunk = tl.load( + k_ptrs, mask=valid[:, None] & mask_d[None, :], other=0.0 + ).to(tl.bfloat16) + + qk += tl.dot(q_chunk, tl.trans(k_chunk)) + + qk = qk * sm_scale + qk = tl.where(valid[None, :], qk, NEG_INF) + + m_ij = tl.max(qk, axis=1) + m_new = tl.maximum(m_i, m_ij) + alpha = tl.where(m_i == NEG_INF, 0.0, tl.math.exp2((m_i - m_new) * LOG2E)) + p = tl.where(qk == NEG_INF, 0.0, tl.math.exp2((qk - m_new[:, None]) * LOG2E)) + l_new = alpha * l_i + tl.sum(p, axis=1) + p_bf16 = p.to(tl.bfloat16) + + offs_v = tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load(v_ptrs, mask=valid[:, None], other=0.0).to(tl.bfloat16) + acc_0 = acc_0 * alpha[:, None] + tl.dot(p_bf16, v) + + offs_v = BLOCK_D + tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load( + v_ptrs, mask=valid[:, None] & (offs_v[None, :] < d_v), other=0.0 + ).to(tl.bfloat16) + acc_1 = acc_1 * alpha[:, None] + tl.dot(p_bf16, v) + + offs_v = 2 * BLOCK_D + tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load( + v_ptrs, mask=valid[:, None] & (offs_v[None, :] < d_v), other=0.0 + ).to(tl.bfloat16) + acc_2 = acc_2 * alpha[:, None] + tl.dot(p_bf16, v) + + offs_v = 3 * BLOCK_D + tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load( + v_ptrs, mask=valid[:, None] & (offs_v[None, :] < d_v), other=0.0 + ).to(tl.bfloat16) + acc_3 = acc_3 * alpha[:, None] + tl.dot(p_bf16, v) + + m_i = m_new + l_i = l_new + + lse = m_i + tl.math.log2(tl.where(l_i == 0.0, 1.0, l_i)) / LOG2E + is_lonely_q = l_i == 0.0 + + if HAS_ATTN_SINK: + attn_sink_vals = tl.load(AttnSink + offs_h, mask=mask_h, other=0.0) + exp_attn_sink_minus_m = tl.math.exp2((attn_sink_vals - m_i) * LOG2E) + denominator = l_i + exp_attn_sink_minus_m + denominator = tl.where(denominator == 0.0, 1.0, denominator) + output_scale = 1.0 / denominator + else: + output_scale = tl.where(l_i == 0.0, 0.0, 1.0 / l_i) + + # Pre-compute 2D versions for efficiency + is_lonely_q_2d = is_lonely_q[:, None] + output_scale_2d = output_scale[:, None] + acc_0 = tl.where(is_lonely_q_2d, 0.0, acc_0 * output_scale_2d) + acc_1 = tl.where(is_lonely_q_2d, 0.0, acc_1 * output_scale_2d) + acc_2 = tl.where(is_lonely_q_2d, 0.0, acc_2 * output_scale_2d) + acc_3 = tl.where(is_lonely_q_2d, 0.0, acc_3 * output_scale_2d) + lse = tl.where(is_lonely_q, POS_INF, lse) + + stride_lse_t_64 = tl.cast(stride_lse_t, tl.int64) + tl.store(LSE + pid_t_64 * stride_lse_t_64 + offs_h * stride_lse_h, lse, mask=mask_h) + + stride_o_t_64 = tl.cast(stride_o_t, tl.int64) + o_base = Output + pid_t_64 * stride_o_t_64 + # Pre-compute 2D versions + offs_h_2d = offs_h[:, None] + mask_h_2d = mask_h[:, None] + offs_v_0 = tl.arange(0, BLOCK_D) + offs_v_1 = BLOCK_D + tl.arange(0, BLOCK_D) + offs_v_2 = 2 * BLOCK_D + tl.arange(0, BLOCK_D) + offs_v_3 = 3 * BLOCK_D + tl.arange(0, BLOCK_D) + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_0[None, :] * stride_o_d, + acc_0.to(tl.bfloat16), + mask=mask_h_2d, + ) + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_1[None, :] * stride_o_d, + acc_1.to(tl.bfloat16), + mask=mask_h_2d & (offs_v_1[None, :] < d_v), + ) + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_2[None, :] * stride_o_d, + acc_2.to(tl.bfloat16), + mask=mask_h_2d & (offs_v_2[None, :] < d_v), + ) + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_3[None, :] * stride_o_d, + acc_3.to(tl.bfloat16), + mask=mask_h_2d & (offs_v_3[None, :] < d_v), + ) + + +# ============================================================================ +# Attention Runner Functions +# ============================================================================ + + +def run_unified_attention( + q_reshaped, + gathered_kv, + invalid_mask, + d_v, + sm_scale, + total_tokens, + h_q, + total_topk, + d_qk, + attn_sink=None, +): + """Run unified attention with single KV buffer. + + Run unified sparse decode attention kernel. + """ + output = torch.empty( + (total_tokens, h_q, d_v), dtype=torch.bfloat16, device=q_reshaped.device + ) + lse = torch.empty( + (total_tokens, h_q), dtype=torch.float32, device=q_reshaped.device + ) + + HAS_ATTN_SINK = attn_sink is not None + attn_sink_tensor = attn_sink if HAS_ATTN_SINK else lse[:1] + + grid = lambda meta: (total_tokens, triton.cdiv(h_q, meta["BLOCK_H"])) + _unified_sparse_decode_kernel[grid]( + q_reshaped, + gathered_kv, + invalid_mask, + attn_sink_tensor, + output, + lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + total_topk, + d_qk, + d_v, + q_reshaped.stride(0), + q_reshaped.stride(1), + q_reshaped.stride(2), + gathered_kv.stride(0), + gathered_kv.stride(1), + gathered_kv.stride(2), + invalid_mask.stride(0), + invalid_mask.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_ATTN_SINK=HAS_ATTN_SINK, + ) + return output, lse + + +def run_chunked_attention_triton( + q_reshaped, + gathered_kv, + invalid_mask, + d_v, + sm_scale, + total_tokens, + h_q, + total_topk, + d_qk, + attn_sink=None, + chunk_size=8192, +): + """Chunked attention using Triton kernels with cross-chunk softmax merging.""" + device = q_reshaped.device + + num_chunks = (total_topk + chunk_size - 1) // chunk_size + + kv_chunks = [] + mask_chunks = [] + chunk_sizes = [] + + for chunk_idx in range(num_chunks): + start_k = chunk_idx * chunk_size + end_k = min(start_k + chunk_size, total_topk) + chunk_topk = end_k - start_k + chunk_sizes.append(chunk_topk) + kv_chunks.append(gathered_kv[:, start_k:end_k, :].contiguous()) + mask_chunks.append(invalid_mask[:, start_k:end_k].contiguous()) + + lse_acc = torch.full( + (total_tokens, h_q), float("-inf"), dtype=torch.float32, device=device + ) + acc = torch.zeros((total_tokens, h_q, d_v), dtype=torch.float32, device=device) + + for chunk_idx in range(num_chunks): + kv_chunk = kv_chunks[chunk_idx] + mask_chunk = mask_chunks[chunk_idx] + chunk_topk = chunk_sizes[chunk_idx] + + chunk_output, chunk_lse = run_unified_attention( + q_reshaped, + kv_chunk, + mask_chunk, + d_v, + sm_scale, + total_tokens, + h_q, + chunk_topk, + d_qk, + attn_sink=None, + ) + + is_chunk_lonely = torch.isinf(chunk_lse) & (chunk_lse > 0) + + chunk_lse_for_merge = torch.where( + is_chunk_lonely, torch.full_like(chunk_lse, float("-inf")), chunk_lse + ) + + lse_max = torch.maximum(lse_acc, chunk_lse_for_merge) + + exp_acc = torch.exp(lse_acc - lse_max) + exp_acc = torch.where(torch.isnan(exp_acc), torch.zeros_like(exp_acc), exp_acc) + + exp_chunk = torch.exp(chunk_lse_for_merge - lse_max) + exp_chunk = torch.where( + torch.isnan(exp_chunk) | is_chunk_lonely, + torch.zeros_like(exp_chunk), + exp_chunk, + ) + + sum_exp = exp_acc + exp_chunk + lse_new = lse_max + torch.log( + torch.where(sum_exp == 0, torch.ones_like(sum_exp), sum_exp) + ) + + both_empty = (lse_acc == float("-inf")) & (chunk_lse_for_merge == float("-inf")) + lse_new = torch.where( + both_empty, torch.full_like(lse_new, float("-inf")), lse_new + ) + + weight_acc = torch.exp(lse_acc - lse_new) + weight_acc = torch.where( + torch.isnan(weight_acc) | torch.isinf(weight_acc), + torch.zeros_like(weight_acc), + weight_acc, + ) + + weight_chunk = torch.exp(chunk_lse_for_merge - lse_new) + weight_chunk = torch.where( + torch.isnan(weight_chunk) | torch.isinf(weight_chunk) | is_chunk_lonely, + torch.zeros_like(weight_chunk), + weight_chunk, + ) + + acc = ( + weight_acc.unsqueeze(-1) * acc + + weight_chunk.unsqueeze(-1) * chunk_output.float() + ) + + lse_acc = lse_new + + output = acc + lse = lse_acc + + is_lonely_final = lse == float("-inf") + + lse = torch.where(is_lonely_final, torch.full_like(lse, float("+inf")), lse) + + if attn_sink is not None: + attn_sink_expanded = attn_sink.view(1, h_q) + exp_diff = torch.exp(attn_sink_expanded - lse) + exp_diff = torch.where( + is_lonely_final, torch.full_like(exp_diff, float("inf")), exp_diff + ) + scale = 1.0 / (1.0 + exp_diff) + output = output * scale.unsqueeze(-1) + + output = torch.where( + is_lonely_final.unsqueeze(-1), torch.zeros_like(output), output + ) + + return output.to(torch.bfloat16), lse + + +# ============================================================================ +# Helper class and functions for token-range based chunking +# ============================================================================ + + +class SlicedKVScope: + """A sliced view of KV scope for a specific token range.""" + + __slots__ = [ + "blocked_k", + "blocked_k_quantized", + "indices_in_kvcache", + "topk_length", + ] + + def __init__(self, blocked_k, blocked_k_quantized, indices_in_kvcache, topk_length): + self.blocked_k = blocked_k + self.blocked_k_quantized = blocked_k_quantized + self.indices_in_kvcache = indices_in_kvcache + self.topk_length = topk_length + + +def slice_kv_scope_for_tokens(orig_scope, start_t: int, end_t: int, s_q: int): + """Slice a KV scope to only include tokens in range [start_t, end_t).""" + if orig_scope is None: + return None + + orig_indices = orig_scope.indices_in_kvcache.reshape( + -1, orig_scope.indices_in_kvcache.size(-1) + ) + sliced_indices = orig_indices[start_t:end_t] + + sliced_topk_length = None + if orig_scope.topk_length is not None: + batch_start = start_t // s_q + batch_end = (end_t + s_q - 1) // s_q + batch_topk_length = orig_scope.topk_length[batch_start:batch_end] + if s_q > 1: + chunk_tokens = end_t - start_t + expanded = batch_topk_length.unsqueeze(1).expand(-1, s_q).reshape(-1) + offset_in_first_batch = start_t % s_q + sliced_topk_length = expanded[ + offset_in_first_batch : offset_in_first_batch + chunk_tokens + ] + else: + sliced_topk_length = batch_topk_length + + return SlicedKVScope( + blocked_k=orig_scope.blocked_k, + blocked_k_quantized=orig_scope.blocked_k_quantized, + indices_in_kvcache=sliced_indices, + topk_length=sliced_topk_length, + ) + + +def compute_token_ranges( + total_tokens: int, + total_topk: int, + d_qk: int, + max_buffer_bytes: int = 2 * 1024 * 1024 * 1024, +) -> List[Tuple[int, int]]: + """Compute token ranges for processing, chunking if buffer would exceed limit.""" + buffer_size_bytes = total_tokens * total_topk * d_qk * 2 + + if buffer_size_bytes <= max_buffer_bytes: + return [(0, total_tokens)] + + max_tokens_per_chunk = max_buffer_bytes // (total_topk * d_qk * 2) + chunk_size = max(1, max_tokens_per_chunk) + + token_ranges = [] + start_t = 0 + while start_t < total_tokens: + end_t = min(start_t + chunk_size, total_tokens) + token_ranges.append((start_t, end_t)) + start_t = end_t + + return token_ranges + + +# ============================================================================ +# Split-K Attention for Large TopK +# ============================================================================ +def run_splitk_unified_attention( + q_reshaped, + gathered_kv, + invalid_mask, + d_v, + sm_scale, + total_tokens, + h_q, + total_topk, + d_qk, + attn_sink=None, + split_k=4, +): + """Run split-K attention for large topk cases.""" + from .triton_mla_kernels_decode_splitk import run_splitk_attention + + return run_splitk_attention( + q_reshaped, + gathered_kv, + invalid_mask, + d_v, + sm_scale, + total_tokens, + h_q, + total_topk, + d_qk, + attn_sink=attn_sink, + split_k=split_k, + ) diff --git a/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_dsv4.py b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_dsv4.py new file mode 100644 index 000000000000..429a6e02fd91 --- /dev/null +++ b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_dsv4.py @@ -0,0 +1,1355 @@ +""" +Triton MLA Decode Kernels for DSV4 (d_qk=512). + +This module contains DSV4-specific gather+dequant kernels and the main +sparse attention decode entry point for DSV4. +""" + +import os +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from .triton_mla_kernels_decode_common import ( + _bucket_total_tokens, + _get_workload_size_category, + compute_token_ranges, + run_chunked_attention_triton, + run_splitk_unified_attention, + run_unified_attention, + slice_kv_scope_for_tokens, +) + +# Enable Triton autotune cache persistence +TRITON_CACHE_DIR = os.path.join(os.path.dirname(__file__), ".triton_cache") +os.makedirs(TRITON_CACHE_DIR, exist_ok=True) +os.environ.setdefault("TRITON_CACHE_DIR", TRITON_CACHE_DIR) + +# Constants for DSV4 layout +DSV4_D_QK = 512 +DSV4_D_NOPE = 448 +DSV4_D_ROPE = 64 +DSV4_TILE_SIZE = 64 +DSV4_NUM_TILES = 7 +DSV4_BYTES_PER_TOKEN_DATA = 576 # 448 nope + 128 rope +DSV4_BYTES_PER_TOKEN_SCALE = 8 # 7 scales + 1 padding + +# Performance tuning thresholds (empirically determined) +# These thresholds balance kernel launch overhead vs. computation efficiency +# +# DSV4_USE_FUSED_THRESHOLD: Use 1D fused kernel below this element count +# Rationale: Single kernel launch reduces overhead for small/medium workloads +# Value 150K determined by benchmarking on typical production workloads +DSV4_USE_FUSED_THRESHOLD = 150000 +# +# DSV4_USE_FIXED_KERNEL_THRESHOLD: Use fixed BLOCK_TK=128 kernel below this +# Rationale: Avoids autotune overhead for small workloads where fixed config +# performs well. Value 32K balances autotune benefit vs. overhead +DSV4_USE_FIXED_KERNEL_THRESHOLD = 32768 + + +# ============================================================================ +# DSV4 Gather+Dequant Kernels - Optimized with Batched Scale Loading +# ============================================================================ + + +@triton.autotune( + configs=[ + # This is a pure memory-copy + FP8→BF16 dequant kernel. + # - BLOCK_TK controls how many (token×topk) pairs per block. + # - Larger BLOCK_TK amortizes launch overhead but needs more warps. + # - BLOCK_TK=128 is already validated as the fixed config for small workloads + # (below DSV4_USE_FIXED_KERNEL_THRESHOLD = 32K elements). + # - BLOCK_TK=64/128: good for small/medium workloads (fewer warps, less overhead). + # - BLOCK_TK=256: better bandwidth utilization for large workloads. + triton.Config({"BLOCK_TK": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_TK": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_TK": 256}, num_warps=8, num_stages=1), + ], + key=["total_tokens_bucket", "topk", "workload_size_cat"], +) +@triton.jit +def _gather_dequant_dsv4_kernel( + KV_Cache, + Indices, + TopkLength, + OutputKV, + OutputMask, + total_tokens, + total_tokens_bucket, + topk, + num_blocks, + block_size, + workload_size_cat, + k_offset, + s_q, + stride_kv_block, + stride_idx_t, + stride_idx_k, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + BLOCK_TK: tl.constexpr, + D_NOPE: tl.constexpr, + D_ROPE: tl.constexpr, + BYTES_PER_TOKEN_DATA: tl.constexpr, + BYTES_PER_TOKEN_SCALE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HAS_TOPK_LENGTH: tl.constexpr, +): + """Optimized gather + dequant kernel with batched scale loading.""" + pid = tl.program_id(0) + num_tk = total_tokens * topk + + offs_tk = pid * BLOCK_TK + tl.arange(0, BLOCK_TK) + mask_tk = offs_tk < num_tk + + t_idx = offs_tk // topk + k_idx = offs_tk % topk + + idx_ptrs = Indices + t_idx * stride_idx_t + k_idx * stride_idx_k + indices = tl.load(idx_ptrs, mask=mask_tk, other=-1) + + is_invalid = indices == -1 + + if HAS_TOPK_LENGTH: + batch_idx = t_idx // s_q + topk_len = tl.load(TopkLength + batch_idx, mask=mask_tk, other=topk) + is_invalid = is_invalid | (k_idx >= topk_len) + + mask_out_ptrs = ( + OutputMask + t_idx * stride_mask_t + (k_idx + k_offset) * stride_mask_k + ) + tl.store(mask_out_ptrs, is_invalid, mask=mask_tk) + + valid_mask = mask_tk & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size + offset_in_block = indices_clamped % block_size + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + kv_block_base = KV_Cache + block_idx_64 * stride_kv_block + + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size * BYTES_PER_TOKEN_DATA + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + t_idx_64 = t_idx.to(tl.int64) + k_idx_64 = k_idx.to(tl.int64) + stride_out_t_64 = tl.cast(stride_out_t, tl.int64) + stride_out_k_64 = tl.cast(stride_out_k, tl.int64) + out_base_ptrs = ( + OutputKV + t_idx_64 * stride_out_t_64 + (k_idx_64 + k_offset) * stride_out_k_64 + ) + + # Load all 7 scales at once - each scale is at scale_base_offset + tile_idx + scale_ptrs_0 = kv_block_base + scale_base_offset + scale_ptrs_1 = kv_block_base + scale_base_offset + 1 + scale_ptrs_2 = kv_block_base + scale_base_offset + 2 + scale_ptrs_3 = kv_block_base + scale_base_offset + 3 + scale_ptrs_4 = kv_block_base + scale_base_offset + 4 + scale_ptrs_5 = kv_block_base + scale_base_offset + 5 + scale_ptrs_6 = kv_block_base + scale_base_offset + 6 + + scale_uint8_0 = tl.load(scale_ptrs_0, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_1 = tl.load(scale_ptrs_1, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_2 = tl.load(scale_ptrs_2, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_3 = tl.load(scale_ptrs_3, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_4 = tl.load(scale_ptrs_4, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_5 = tl.load(scale_ptrs_5, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_6 = tl.load(scale_ptrs_6, mask=valid_mask, other=127).to(tl.uint8) + + # Convert all scales to bf16 and pre-compute 2D versions + scale_bf16_0 = tl.math.exp2(scale_uint8_0.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_1 = tl.math.exp2(scale_uint8_1.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_2 = tl.math.exp2(scale_uint8_2.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_3 = tl.math.exp2(scale_uint8_3.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_4 = tl.math.exp2(scale_uint8_4.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_5 = tl.math.exp2(scale_uint8_5.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_6 = tl.math.exp2(scale_uint8_6.to(tl.float32) - 127.0).to(tl.bfloat16) + # Pre-compute 2D versions for tile processing + scale_2d_0 = scale_bf16_0[:, None] + scale_2d_1 = scale_bf16_1[:, None] + scale_2d_2 = scale_bf16_2[:, None] + scale_2d_3 = scale_bf16_3[:, None] + scale_2d_4 = scale_bf16_4[:, None] + scale_2d_5 = scale_bf16_5[:, None] + scale_2d_6 = scale_bf16_6[:, None] + + offs_d = tl.arange(0, TILE_SIZE) + + # Pre-compute base pointers for optimization + tile_base = kv_block_base[:, None] + nope_rope_offset[:, None] + out_base = out_base_ptrs[:, None] + valid_mask_2d = valid_mask[:, None] + is_invalid_2d = is_invalid[:, None] + mask_tk_2d = mask_tk[:, None] + + # Process tile 0 + nope_ptrs = tile_base + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_0 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + offs_d[None, :] * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 1 + tile_start_1 = TILE_SIZE + nope_ptrs = tile_base + tile_start_1 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_1 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_1 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 2 + tile_start_2 = 2 * TILE_SIZE + nope_ptrs = tile_base + tile_start_2 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_2 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_2 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 3 + tile_start_3 = 3 * TILE_SIZE + nope_ptrs = tile_base + tile_start_3 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_3 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_3 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 4 + tile_start_4 = 4 * TILE_SIZE + nope_ptrs = tile_base + tile_start_4 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_4 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_4 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 5 + tile_start_5 = 5 * TILE_SIZE + nope_ptrs = tile_base + tile_start_5 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_5 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_5 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 6 + tile_start_6 = 6 * TILE_SIZE + nope_ptrs = tile_base + tile_start_6 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_6 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_6 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process rope + offs_rope = tl.arange(0, D_ROPE) + rope_byte_start = D_NOPE + + rope_lo_ptrs = tile_base + rope_byte_start + offs_rope[None, :] * 2 + rope_hi_ptrs = tile_base + rope_byte_start + offs_rope[None, :] * 2 + 1 + + rope_lo = tl.load(rope_lo_ptrs, mask=valid_mask_2d, other=0).to(tl.uint16) + rope_hi = tl.load(rope_hi_ptrs, mask=valid_mask_2d, other=0).to(tl.uint16) + + rope_uint16 = rope_lo | (rope_hi << 8) + rope_bf16 = rope_uint16.to(tl.bfloat16, bitcast=True) + rope_bf16 = tl.where(is_invalid_2d, 0.0, rope_bf16) + + out_ptrs = out_base + (D_NOPE + offs_rope[None, :]) * stride_out_d + tl.store(out_ptrs, rope_bf16, mask=mask_tk_2d) + + +@triton.jit +def _gather_dequant_dsv4_kernel_fixed_128( + KV_Cache, + Indices, + TopkLength, + OutputKV, + OutputMask, + total_tokens, + total_tokens_bucket, + topk, + num_blocks, + block_size, + k_offset, + s_q, + stride_kv_block, + stride_idx_t, + stride_idx_k, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + D_NOPE: tl.constexpr, + D_ROPE: tl.constexpr, + BYTES_PER_TOKEN_DATA: tl.constexpr, + BYTES_PER_TOKEN_SCALE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HAS_TOPK_LENGTH: tl.constexpr, +): + """Fixed-config gather kernel with BLOCK_TK=128 and batched scale loading.""" + BLOCK_TK: tl.constexpr = 128 + pid = tl.program_id(0) + num_tk = total_tokens * topk + + offs_tk = pid * BLOCK_TK + tl.arange(0, BLOCK_TK) + mask_tk = offs_tk < num_tk + + t_idx = offs_tk // topk + k_idx = offs_tk % topk + + idx_ptrs = Indices + t_idx * stride_idx_t + k_idx * stride_idx_k + indices = tl.load(idx_ptrs, mask=mask_tk, other=-1) + + is_invalid = indices == -1 + + if HAS_TOPK_LENGTH: + batch_idx = t_idx // s_q + topk_len = tl.load(TopkLength + batch_idx, mask=mask_tk, other=topk) + is_invalid = is_invalid | (k_idx >= topk_len) + + mask_out_ptrs = ( + OutputMask + t_idx * stride_mask_t + (k_idx + k_offset) * stride_mask_k + ) + tl.store(mask_out_ptrs, is_invalid, mask=mask_tk) + + valid_mask = mask_tk & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size + offset_in_block = indices_clamped % block_size + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + kv_block_base = KV_Cache + block_idx_64 * stride_kv_block + + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size * BYTES_PER_TOKEN_DATA + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + t_idx_64 = t_idx.to(tl.int64) + k_idx_64 = k_idx.to(tl.int64) + stride_out_t_64 = tl.cast(stride_out_t, tl.int64) + stride_out_k_64 = tl.cast(stride_out_k, tl.int64) + out_base_ptrs = ( + OutputKV + t_idx_64 * stride_out_t_64 + (k_idx_64 + k_offset) * stride_out_k_64 + ) + + # Load all 7 scales at once + scale_ptrs_0 = kv_block_base + scale_base_offset + scale_ptrs_1 = kv_block_base + scale_base_offset + 1 + scale_ptrs_2 = kv_block_base + scale_base_offset + 2 + scale_ptrs_3 = kv_block_base + scale_base_offset + 3 + scale_ptrs_4 = kv_block_base + scale_base_offset + 4 + scale_ptrs_5 = kv_block_base + scale_base_offset + 5 + scale_ptrs_6 = kv_block_base + scale_base_offset + 6 + + scale_uint8_0 = tl.load(scale_ptrs_0, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_1 = tl.load(scale_ptrs_1, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_2 = tl.load(scale_ptrs_2, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_3 = tl.load(scale_ptrs_3, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_4 = tl.load(scale_ptrs_4, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_5 = tl.load(scale_ptrs_5, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_6 = tl.load(scale_ptrs_6, mask=valid_mask, other=127).to(tl.uint8) + + # Convert all scales to bf16 and pre-compute 2D versions + scale_bf16_0 = tl.math.exp2(scale_uint8_0.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_1 = tl.math.exp2(scale_uint8_1.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_2 = tl.math.exp2(scale_uint8_2.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_3 = tl.math.exp2(scale_uint8_3.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_4 = tl.math.exp2(scale_uint8_4.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_5 = tl.math.exp2(scale_uint8_5.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_6 = tl.math.exp2(scale_uint8_6.to(tl.float32) - 127.0).to(tl.bfloat16) + # Pre-compute 2D versions for tile processing + scale_2d_0 = scale_bf16_0[:, None] + scale_2d_1 = scale_bf16_1[:, None] + scale_2d_2 = scale_bf16_2[:, None] + scale_2d_3 = scale_bf16_3[:, None] + scale_2d_4 = scale_bf16_4[:, None] + scale_2d_5 = scale_bf16_5[:, None] + scale_2d_6 = scale_bf16_6[:, None] + + offs_d = tl.arange(0, TILE_SIZE) + + # Pre-compute base pointers for optimization + tile_base = kv_block_base[:, None] + nope_rope_offset[:, None] + out_base = out_base_ptrs[:, None] + valid_mask_2d = valid_mask[:, None] + is_invalid_2d = is_invalid[:, None] + mask_tk_2d = mask_tk[:, None] + + # Process tile 0 + nope_ptrs = tile_base + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_0 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + offs_d[None, :] * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 1 + tile_start_1 = TILE_SIZE + nope_ptrs = tile_base + tile_start_1 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_1 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_1 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 2 + tile_start_2 = 2 * TILE_SIZE + nope_ptrs = tile_base + tile_start_2 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_2 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_2 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 3 + tile_start_3 = 3 * TILE_SIZE + nope_ptrs = tile_base + tile_start_3 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_3 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_3 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 4 + tile_start_4 = 4 * TILE_SIZE + nope_ptrs = tile_base + tile_start_4 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_4 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_4 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 5 + tile_start_5 = 5 * TILE_SIZE + nope_ptrs = tile_base + tile_start_5 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_5 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_5 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 6 + tile_start_6 = 6 * TILE_SIZE + nope_ptrs = tile_base + tile_start_6 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_6 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_6 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process rope + offs_rope = tl.arange(0, D_ROPE) + rope_byte_start = D_NOPE + + rope_lo_ptrs = tile_base + rope_byte_start + offs_rope[None, :] * 2 + rope_hi_ptrs = tile_base + rope_byte_start + offs_rope[None, :] * 2 + 1 + + rope_lo = tl.load(rope_lo_ptrs, mask=valid_mask_2d, other=0).to(tl.uint16) + rope_hi = tl.load(rope_hi_ptrs, mask=valid_mask_2d, other=0).to(tl.uint16) + + rope_uint16 = rope_lo | (rope_hi << 8) + rope_bf16 = rope_uint16.to(tl.bfloat16, bitcast=True) + rope_bf16 = tl.where(is_invalid_2d, 0.0, rope_bf16) + + out_ptrs = out_base + (D_NOPE + offs_rope[None, :]) * stride_out_d + tl.store(out_ptrs, rope_bf16, mask=mask_tk_2d) + + +# ============================================================================ +# DSV4 Wrapper Functions +# ============================================================================ + + +def gather_dequant_fp8_dsv4( + kv_cache_quantized: torch.Tensor, + indices: torch.Tensor, + block_size: int, + output_kv: torch.Tensor, + output_mask: torch.Tensor, + k_offset: int = 0, + topk_length: Optional[torch.Tensor] = None, + s_q: int = 1, +) -> bool: + """Unified DSV4 gather+dequant with optional topk_length mask.""" + total_tokens, topk = indices.shape + num_blocks = kv_cache_quantized.shape[0] + + kv_uint8 = kv_cache_quantized.view(torch.uint8) + bytes_per_block = kv_uint8.shape[1] * kv_uint8.shape[2] * kv_uint8.shape[3] + kv_flat = kv_uint8.reshape(num_blocks, bytes_per_block) + + stride_kv_block = kv_uint8.stride(0) + workload_size_cat = _get_workload_size_category(total_tokens, topk) + + grid = lambda meta: (triton.cdiv(total_tokens * topk, meta["BLOCK_TK"]),) + + topk_length_tensor = topk_length if topk_length is not None else output_mask[:1, 0] + has_topk_length = topk_length is not None + + _gather_dequant_dsv4_kernel[grid]( + kv_flat, + indices, + topk_length_tensor, + output_kv, + output_mask, + total_tokens, + _bucket_total_tokens(total_tokens), + topk, + num_blocks, + block_size, + workload_size_cat, + k_offset, + s_q, + stride_kv_block, + indices.stride(0), + indices.stride(1), + output_kv.stride(0), + output_kv.stride(1), + output_kv.stride(2), + output_mask.stride(0), + output_mask.stride(1), + D_NOPE=DSV4_D_NOPE, + D_ROPE=DSV4_D_ROPE, + BYTES_PER_TOKEN_DATA=DSV4_BYTES_PER_TOKEN_DATA, + BYTES_PER_TOKEN_SCALE=DSV4_BYTES_PER_TOKEN_SCALE, + TILE_SIZE=DSV4_TILE_SIZE, + HAS_TOPK_LENGTH=has_topk_length, + ) + return True + + +# ============================================================================ +# DSV4 1D Grid Fused Gather+Dequant Kernel (Optimized - No Empty Blocks) +# Single kernel launch with 1D grid: (num_main_pids + num_extra_pids,) +# ============================================================================ + + +@triton.jit +def _gather_dequant_dsv4_1d_fused_kernel( + # Main KV cache + KV_Cache_Main, + Indices_Main, + TopkLength_Main, + # Extra KV cache + KV_Cache_Extra, + Indices_Extra, + TopkLength_Extra, + # Output + OutputKV, + OutputMask, + # Dimensions + total_tokens, + topk_main, + topk_extra, + num_blocks_main, + num_blocks_extra, + block_size_main, + block_size_extra, + s_q, + # Strides for main + stride_kv_block_main, + stride_idx_t_main, + stride_idx_k_main, + # Strides for extra + stride_kv_block_extra, + stride_idx_t_extra, + stride_idx_k_extra, + # Output strides + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + # Grid info + num_main_pids, + # Constexpr + BLOCK_TK: tl.constexpr, + D_NOPE: tl.constexpr, + D_ROPE: tl.constexpr, + BYTES_PER_TOKEN_DATA: tl.constexpr, + BYTES_PER_TOKEN_SCALE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HAS_TOPK_LENGTH_MAIN: tl.constexpr, + HAS_TOPK_LENGTH_EXTRA: tl.constexpr, +): + """1D fused gather kernel - single launch, no empty blocks. + + Grid: (num_main_pids + num_extra_pids,) + - pid < num_main_pids: process main cache + - pid >= num_main_pids: process extra cache + + This eliminates empty blocks when main/extra topk differ significantly. + """ + pid = tl.program_id(0) + + # Determine if this is main or extra processing + is_main_pid = pid < num_main_pids + + # Select parameters based on pid + if is_main_pid: + local_pid = pid + topk = topk_main + k_offset = 0 + num_tk = total_tokens * topk_main + KV_Cache = KV_Cache_Main + Indices = Indices_Main + TopkLength = TopkLength_Main + block_size = block_size_main + stride_kv_block = stride_kv_block_main + stride_idx_t = stride_idx_t_main + stride_idx_k = stride_idx_k_main + else: + local_pid = pid - num_main_pids + topk = topk_extra + k_offset = topk_main + num_tk = total_tokens * topk_extra + KV_Cache = KV_Cache_Extra + Indices = Indices_Extra + TopkLength = TopkLength_Extra + block_size = block_size_extra + stride_kv_block = stride_kv_block_extra + stride_idx_t = stride_idx_t_extra + stride_idx_k = stride_idx_k_extra + + # Compute element indices for this block + offs_tk = local_pid * BLOCK_TK + tl.arange(0, BLOCK_TK) + mask_tk = offs_tk < num_tk + + t_idx = offs_tk // topk + k_idx = offs_tk % topk + + # Load indices + idx_ptrs = Indices + t_idx * stride_idx_t + k_idx * stride_idx_k + indices = tl.load(idx_ptrs, mask=mask_tk, other=-1) + + is_invalid = indices == -1 + + # Handle topk_length - need to handle both cases + batch_idx = t_idx // s_q + if is_main_pid: + if HAS_TOPK_LENGTH_MAIN: + topk_len = tl.load(TopkLength + batch_idx, mask=mask_tk, other=topk) + is_invalid = is_invalid | (k_idx >= topk_len) + else: + if HAS_TOPK_LENGTH_EXTRA: + topk_len = tl.load(TopkLength + batch_idx, mask=mask_tk, other=topk) + is_invalid = is_invalid | (k_idx >= topk_len) + + # Store mask + mask_out_ptrs = ( + OutputMask + t_idx * stride_mask_t + (k_idx + k_offset) * stride_mask_k + ) + tl.store(mask_out_ptrs, is_invalid, mask=mask_tk) + + valid_mask = mask_tk & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size + offset_in_block = indices_clamped % block_size + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + kv_block_base = KV_Cache + block_idx_64 * stride_kv_block + + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size * BYTES_PER_TOKEN_DATA + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + t_idx_64 = t_idx.to(tl.int64) + k_idx_64 = k_idx.to(tl.int64) + stride_out_t_64 = tl.cast(stride_out_t, tl.int64) + stride_out_k_64 = tl.cast(stride_out_k, tl.int64) + out_base_ptrs = ( + OutputKV + t_idx_64 * stride_out_t_64 + (k_idx_64 + k_offset) * stride_out_k_64 + ) + + # Load all 7 scales + scale_ptrs_0 = kv_block_base + scale_base_offset + scale_uint8_0 = tl.load(scale_ptrs_0, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_1 = tl.load(scale_ptrs_0 + 1, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_2 = tl.load(scale_ptrs_0 + 2, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_3 = tl.load(scale_ptrs_0 + 3, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_4 = tl.load(scale_ptrs_0 + 4, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_5 = tl.load(scale_ptrs_0 + 5, mask=valid_mask, other=127).to(tl.uint8) + scale_uint8_6 = tl.load(scale_ptrs_0 + 6, mask=valid_mask, other=127).to(tl.uint8) + + scale_bf16_0 = tl.math.exp2(scale_uint8_0.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_1 = tl.math.exp2(scale_uint8_1.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_2 = tl.math.exp2(scale_uint8_2.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_3 = tl.math.exp2(scale_uint8_3.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_4 = tl.math.exp2(scale_uint8_4.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_5 = tl.math.exp2(scale_uint8_5.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_6 = tl.math.exp2(scale_uint8_6.to(tl.float32) - 127.0).to(tl.bfloat16) + # Pre-compute 2D versions for tile processing + scale_2d_0 = scale_bf16_0[:, None] + scale_2d_1 = scale_bf16_1[:, None] + scale_2d_2 = scale_bf16_2[:, None] + scale_2d_3 = scale_bf16_3[:, None] + scale_2d_4 = scale_bf16_4[:, None] + scale_2d_5 = scale_bf16_5[:, None] + scale_2d_6 = scale_bf16_6[:, None] + + offs_d = tl.arange(0, TILE_SIZE) + + # Pre-compute base pointers for optimization + tile_base = kv_block_base[:, None] + nope_rope_offset[:, None] + out_base = out_base_ptrs[:, None] + valid_mask_2d = valid_mask[:, None] + is_invalid_2d = is_invalid[:, None] + mask_tk_2d = mask_tk[:, None] + + # Process tile 0 + nope_ptrs = tile_base + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_0 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + offs_d[None, :] * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 1 + tile_start_1 = TILE_SIZE + nope_ptrs = tile_base + tile_start_1 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_1 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_1 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 2 + tile_start_2 = 2 * TILE_SIZE + nope_ptrs = tile_base + tile_start_2 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_2 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_2 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 3 + tile_start_3 = 3 * TILE_SIZE + nope_ptrs = tile_base + tile_start_3 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_3 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_3 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 4 + tile_start_4 = 4 * TILE_SIZE + nope_ptrs = tile_base + tile_start_4 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_4 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_4 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 5 + tile_start_5 = 5 * TILE_SIZE + nope_ptrs = tile_base + tile_start_5 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_5 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_5 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process tile 6 + tile_start_6 = 6 * TILE_SIZE + nope_ptrs = tile_base + tile_start_6 + offs_d[None, :] + nope_uint8 = tl.load(nope_ptrs, mask=valid_mask_2d, other=0) + nope_fp8 = nope_uint8.to(tl.float8e4nv, bitcast=True) + nope_bf16 = nope_fp8.to(tl.bfloat16) + dequant = nope_bf16 * scale_2d_6 + dequant = tl.where(is_invalid_2d, 0.0, dequant) + out_ptrs = out_base + (tile_start_6 + offs_d[None, :]) * stride_out_d + tl.store(out_ptrs, dequant, mask=mask_tk_2d) + + # Process rope + offs_rope = tl.arange(0, D_ROPE) + rope_byte_start = D_NOPE + rope_lo_ptrs = tile_base + rope_byte_start + offs_rope[None, :] * 2 + rope_hi_ptrs = tile_base + rope_byte_start + offs_rope[None, :] * 2 + 1 + rope_lo = tl.load(rope_lo_ptrs, mask=valid_mask_2d, other=0).to(tl.uint16) + rope_hi = tl.load(rope_hi_ptrs, mask=valid_mask_2d, other=0).to(tl.uint16) + rope_uint16 = rope_lo | (rope_hi << 8) + rope_bf16 = rope_uint16.to(tl.bfloat16, bitcast=True) + rope_bf16 = tl.where(is_invalid_2d, 0.0, rope_bf16) + out_ptrs = out_base + (D_NOPE + offs_rope[None, :]) * stride_out_d + tl.store(out_ptrs, rope_bf16, mask=mask_tk_2d) + + +def _prepare_kv_cache_flat(kv_cache): + """Helper to prepare KV cache for gather operations. + + Returns: (kv_flat, num_blocks, stride_kv_block) + """ + kv_uint8 = kv_cache.view(torch.uint8) + num_blocks = kv_cache.shape[0] + bytes_per_block = kv_uint8.shape[1] * kv_uint8.shape[2] * kv_uint8.shape[3] + kv_flat = kv_uint8.reshape(num_blocks, bytes_per_block) + stride_kv_block = kv_uint8.stride(0) + return kv_flat, num_blocks, stride_kv_block + + +def _launch_gather_dequant_one_dsv4( + kv_flat, + indices, + topk_length_tensor, + output_kv, + output_mask, + total_tokens, + topk, + num_blocks, + block_size, + k_offset, + s_q, + stride_kv_block, + stride_idx_t, + stride_idx_k, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + has_topk_length, +): + """Helper to launch gather+dequant kernel for one KV cache (main or extra). + + This eliminates code duplication between main and extra kernel launches + in the two-kernel path of fused_gather_dequant_fp8_dsv4. + """ + total_elements = total_tokens * topk + + if total_elements < DSV4_USE_FIXED_KERNEL_THRESHOLD: + grid = (triton.cdiv(total_elements, 128),) + _gather_dequant_dsv4_kernel_fixed_128[grid]( + kv_flat, + indices, + topk_length_tensor, + output_kv, + output_mask, + total_tokens, + _bucket_total_tokens(total_tokens), + topk, + num_blocks, + block_size, + k_offset, + s_q, + stride_kv_block, + stride_idx_t, + stride_idx_k, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + D_NOPE=DSV4_D_NOPE, + D_ROPE=DSV4_D_ROPE, + BYTES_PER_TOKEN_DATA=DSV4_BYTES_PER_TOKEN_DATA, + BYTES_PER_TOKEN_SCALE=DSV4_BYTES_PER_TOKEN_SCALE, + TILE_SIZE=DSV4_TILE_SIZE, + HAS_TOPK_LENGTH=has_topk_length, + num_warps=8, + num_stages=2, + ) + else: + workload_cat = _get_workload_size_category(total_tokens, topk) + grid = lambda meta: (triton.cdiv(total_elements, meta["BLOCK_TK"]),) + _gather_dequant_dsv4_kernel[grid]( + kv_flat, + indices, + topk_length_tensor, + output_kv, + output_mask, + total_tokens, + _bucket_total_tokens(total_tokens), + topk, + num_blocks, + block_size, + workload_cat, + k_offset, + s_q, + stride_kv_block, + stride_idx_t, + stride_idx_k, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + D_NOPE=DSV4_D_NOPE, + D_ROPE=DSV4_D_ROPE, + BYTES_PER_TOKEN_DATA=DSV4_BYTES_PER_TOKEN_DATA, + BYTES_PER_TOKEN_SCALE=DSV4_BYTES_PER_TOKEN_SCALE, + TILE_SIZE=DSV4_TILE_SIZE, + HAS_TOPK_LENGTH=has_topk_length, + ) + + +def truly_fused_gather_dequant_fp8_dsv4( + kv_cache_main, + indices_main, + block_size_main, + topk_length_main, + kv_cache_extra, + indices_extra, + block_size_extra, + topk_length_extra, + output_kv, + output_mask, + s_q=1, +): + """Truly fused DSV4 gather - single kernel launch with 1D grid (no empty blocks).""" + total_tokens, topk_main = indices_main.shape + topk_extra = indices_extra.shape[1] + b = total_tokens // s_q # batch size + + kv_flat_main, num_blocks_main, stride_kv_block_main = _prepare_kv_cache_flat( + kv_cache_main + ) + kv_flat_extra, num_blocks_extra, stride_kv_block_extra = _prepare_kv_cache_flat( + kv_cache_extra + ) + + has_topk_length_main = topk_length_main is not None + has_topk_length_extra = topk_length_extra is not None + + # Always use int32 tensors for topk_length to avoid type mismatch in Triton + if has_topk_length_main: + topk_length_main_tensor = topk_length_main + else: + topk_length_main_tensor = torch.full( + (b,), topk_main, dtype=torch.int32, device=indices_main.device + ) + + if has_topk_length_extra: + topk_length_extra_tensor = topk_length_extra + else: + topk_length_extra_tensor = torch.full( + (b,), topk_extra, dtype=torch.int32, device=indices_extra.device + ) + + stride_idx_t_main, stride_idx_k_main = indices_main.stride(0), indices_main.stride( + 1 + ) + stride_idx_t_extra, stride_idx_k_extra = indices_extra.stride( + 0 + ), indices_extra.stride(1) + stride_out_t, stride_out_k, stride_out_d = ( + output_kv.stride(0), + output_kv.stride(1), + output_kv.stride(2), + ) + stride_mask_t, stride_mask_k = output_mask.stride(0), output_mask.stride(1) + + BLOCK_TK = 128 + + # Calculate grid sizes - 1D grid with exact number of needed blocks + num_elements_main = total_tokens * topk_main + num_elements_extra = total_tokens * topk_extra + num_main_pids = triton.cdiv(num_elements_main, BLOCK_TK) + num_extra_pids = triton.cdiv(num_elements_extra, BLOCK_TK) + + # 1D grid: (num_main_pids + num_extra_pids,) - no empty blocks! + grid = (num_main_pids + num_extra_pids,) + + _gather_dequant_dsv4_1d_fused_kernel[grid]( + kv_flat_main, + indices_main, + topk_length_main_tensor, + kv_flat_extra, + indices_extra, + topk_length_extra_tensor, + output_kv, + output_mask, + total_tokens, + topk_main, + topk_extra, + num_blocks_main, + num_blocks_extra, + block_size_main, + block_size_extra, + s_q, + stride_kv_block_main, + stride_idx_t_main, + stride_idx_k_main, + stride_kv_block_extra, + stride_idx_t_extra, + stride_idx_k_extra, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + num_main_pids, + BLOCK_TK=BLOCK_TK, + D_NOPE=DSV4_D_NOPE, + D_ROPE=DSV4_D_ROPE, + BYTES_PER_TOKEN_DATA=DSV4_BYTES_PER_TOKEN_DATA, + BYTES_PER_TOKEN_SCALE=DSV4_BYTES_PER_TOKEN_SCALE, + TILE_SIZE=DSV4_TILE_SIZE, + HAS_TOPK_LENGTH_MAIN=has_topk_length_main, + HAS_TOPK_LENGTH_EXTRA=has_topk_length_extra, + num_warps=8, + num_stages=2, + ) + return True + + +def fused_gather_dequant_fp8_dsv4( + kv_cache_main, + indices_main, + block_size_main, + topk_length_main, + kv_cache_extra, + indices_extra, + block_size_extra, + topk_length_extra, + output_kv, + output_mask, + s_q=1, +): + """Fused DSV4 gather - uses 1D fused kernel for small workloads, two kernels for large.""" + has_topk_length_main = topk_length_main is not None + has_topk_length_extra = topk_length_extra is not None + + total_tokens, topk_main = indices_main.shape + topk_extra = indices_extra.shape[1] + total_elements = total_tokens * (topk_main + topk_extra) + + # Use fused 2D grid kernel only for small workloads where kernel launch overhead matters + # For large workloads, the two-kernel approach is more efficient + USE_FUSED_THRESHOLD = DSV4_USE_FUSED_THRESHOLD + + # IMPORTANT: Disable fused kernel when topk_length settings differ between main and extra + # The 1D fused kernel has issues with runtime conditional handling when + # HAS_TOPK_LENGTH_MAIN != HAS_TOPK_LENGTH_EXTRA, causing incorrect results in extra part. + # Only use fused kernel when both have same topk_length setting. + topk_length_settings_match = has_topk_length_main == has_topk_length_extra + use_fused = total_elements < USE_FUSED_THRESHOLD and topk_length_settings_match + + if use_fused: + return truly_fused_gather_dequant_fp8_dsv4( + kv_cache_main, + indices_main, + block_size_main, + topk_length_main, + kv_cache_extra, + indices_extra, + block_size_extra, + topk_length_extra, + output_kv, + output_mask, + s_q, + ) + + # Use original two-kernel approach for large workloads + kv_flat_main, num_blocks_main, stride_kv_block_main = _prepare_kv_cache_flat( + kv_cache_main + ) + kv_flat_extra, num_blocks_extra, stride_kv_block_extra = _prepare_kv_cache_flat( + kv_cache_extra + ) + + topk_length_main_tensor = ( + topk_length_main if has_topk_length_main else output_mask[:1, 0] + ) + topk_length_extra_tensor = ( + topk_length_extra if has_topk_length_extra else output_mask[:1, 0] + ) + + stride_idx_t_main, stride_idx_k_main = indices_main.stride(0), indices_main.stride( + 1 + ) + stride_idx_t_extra, stride_idx_k_extra = indices_extra.stride( + 0 + ), indices_extra.stride(1) + stride_out_t, stride_out_k, stride_out_d = ( + output_kv.stride(0), + output_kv.stride(1), + output_kv.stride(2), + ) + stride_mask_t, stride_mask_k = output_mask.stride(0), output_mask.stride(1) + + # Launch main kernel + _launch_gather_dequant_one_dsv4( + kv_flat_main, + indices_main, + topk_length_main_tensor, + output_kv, + output_mask, + total_tokens, + topk_main, + num_blocks_main, + block_size_main, + 0, + s_q, + stride_kv_block_main, + stride_idx_t_main, + stride_idx_k_main, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + has_topk_length_main, + ) + + # Launch extra kernel + _launch_gather_dequant_one_dsv4( + kv_flat_extra, + indices_extra, + topk_length_extra_tensor, + output_kv, + output_mask, + total_tokens, + topk_extra, + num_blocks_extra, + block_size_extra, + topk_main, + s_q, + stride_kv_block_extra, + stride_idx_t_extra, + stride_idx_k_extra, + stride_out_t, + stride_out_k, + stride_out_d, + stride_mask_t, + stride_mask_k, + has_topk_length_extra, + ) + + return True + + +def triton_sparse_attn_decode_dsv4( + q: torch.Tensor, + kv_scope, + extra_kv_scope, + sm_scale: float, + d_v: int = 512, + attn_sink: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Sparse attention decode for DSV4 (d_qk=512).""" + assert kv_scope is not None + b, s_q, h_q, d_qk = q.shape + assert d_qk == DSV4_D_QK, f"Expected d_qk={DSV4_D_QK} for DSV4, got {d_qk}" + total_tokens = b * s_q + + topk_main = kv_scope.indices_in_kvcache.size(-1) + topk_extra = ( + extra_kv_scope.indices_in_kvcache.size(-1) if extra_kv_scope is not None else 0 + ) + total_topk = topk_main + topk_extra + + token_ranges = compute_token_ranges(total_tokens, total_topk, d_qk) + + if len(token_ranges) == 1: + return _triton_sparse_attn_decode_dsv4_impl( + q, kv_scope, extra_kv_scope, sm_scale, d_v, attn_sink + ) + + outputs = [] + lses = [] + + for start_t, end_t in token_ranges: + chunk_tokens = end_t - start_t + q_chunk = q.reshape(total_tokens, h_q, d_qk)[start_t:end_t] + q_input = q_chunk.reshape(chunk_tokens, 1, h_q, d_qk) + chunk_kv_scope = slice_kv_scope_for_tokens(kv_scope, start_t, end_t, s_q) + chunk_extra_kv_scope = slice_kv_scope_for_tokens( + extra_kv_scope, start_t, end_t, s_q + ) + + chunk_out, chunk_lse = _triton_sparse_attn_decode_dsv4_impl( + q_input, chunk_kv_scope, chunk_extra_kv_scope, sm_scale, d_v, attn_sink + ) + + outputs.append(chunk_out.reshape(chunk_tokens, h_q, d_v)) + lses.append(chunk_lse.reshape(chunk_tokens, h_q)) + + output = torch.cat(outputs, dim=0).reshape(b, s_q, h_q, d_v) + lse = torch.cat(lses, dim=0).reshape(b, s_q, h_q).transpose(1, 2) + + return output, lse + + +def _triton_sparse_attn_decode_dsv4_impl( + q: torch.Tensor, + kv_scope, + extra_kv_scope, + sm_scale: float, + d_v: int = 512, + attn_sink: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Internal implementation of sparse attention decode for DSV4. + + Assumes KV cache is always FP8 quantized (blocked_k_quantized is not None). + """ + assert kv_scope is not None + b, s_q, h_q, d_qk = q.shape + total_tokens = b * s_q + + topk_main = kv_scope.indices_in_kvcache.size(-1) + topk_extra = ( + extra_kv_scope.indices_in_kvcache.size(-1) if extra_kv_scope is not None else 0 + ) + total_topk = topk_main + topk_extra + + gathered_kv = torch.empty( + total_tokens, total_topk, d_qk, dtype=torch.bfloat16, device=q.device + ) + invalid_mask = torch.empty( + total_tokens, total_topk, dtype=torch.bool, device=q.device + ) + + block_size_main = kv_scope.blocked_k.shape[1] + indices_main = kv_scope.indices_in_kvcache.reshape(total_tokens, topk_main) + + if extra_kv_scope is not None: + # Fused gather for both main and extra scope + block_size_extra = extra_kv_scope.blocked_k.shape[1] + indices_extra = extra_kv_scope.indices_in_kvcache.reshape( + total_tokens, topk_extra + ) + fused_gather_dequant_fp8_dsv4( + kv_scope.blocked_k_quantized, + indices_main, + block_size_main, + kv_scope.topk_length, + extra_kv_scope.blocked_k_quantized, + indices_extra, + block_size_extra, + extra_kv_scope.topk_length, + gathered_kv, + invalid_mask, + s_q, + ) + else: + # Single gather for main scope only + gather_dequant_fp8_dsv4( + kv_scope.blocked_k_quantized, + indices_main, + block_size_main, + gathered_kv, + invalid_mask, + 0, + kv_scope.topk_length, + s_q, + ) + + q_reshaped = q.to(torch.bfloat16).reshape(total_tokens, h_q, d_qk) + + if not q_reshaped.is_contiguous(): + q_reshaped = q_reshaped.contiguous() + + # Use splitk for large topk to reduce register pressure + if total_topk >= 8192: + # Adaptive split_k selection for optimal performance + # split_k=3 is optimal for topk >= 16384 based on benchmarking + if total_topk >= 16384: + split_k = 3 + else: + split_k = 2 + output, lse = run_splitk_unified_attention( + q_reshaped, + gathered_kv, + invalid_mask, + d_v, + sm_scale, + total_tokens, + h_q, + total_topk, + d_qk, + attn_sink=attn_sink, + split_k=split_k, + ) + elif total_topk <= 65536: + output, lse = run_unified_attention( + q_reshaped, + gathered_kv, + invalid_mask, + d_v, + sm_scale, + total_tokens, + h_q, + total_topk, + d_qk, + attn_sink=attn_sink, + ) + else: + output, lse = run_chunked_attention_triton( + q_reshaped, + gathered_kv, + invalid_mask, + d_v, + sm_scale, + total_tokens, + h_q, + total_topk, + d_qk, + attn_sink=attn_sink, + chunk_size=32768, + ) + + return output.view(b, s_q, h_q, d_v), lse.view(b, s_q, h_q).transpose(1, 2) diff --git a/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_fused.py b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_fused.py new file mode 100644 index 000000000000..6167f58bff3d --- /dev/null +++ b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_fused.py @@ -0,0 +1,3089 @@ +""" +Fused Gather+Dequant+Attention Kernel for DSV4 (d_qk=512) + +This module implements a fused kernel that combines: +1. Gather: Load KV from sparse indices +2. Dequant: FP8 to BF16 dequantization +3. Attention: Compute attention scores and output + +Benefits for workloads without extra scope: +- Eliminates intermediate buffer (gathered_kv) write/read +- Reduces kernel launch overhead (1 kernel instead of 2) +- Better cache utilization + +Supports: +- DSV4 (d_qk=512): 7 tiles of 64, uint8 scales +- All configs: with/without topk_length, with/without attn_sink + +OPTIMIZED VERSION: Reduced code duplication in dual-scope kernel by using +a helper function for KV block processing. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from .triton_mla_kernels_decode_common import _bucket_total_tokens + +# ============================================================================ +# Constants for DSV4 layout +# ============================================================================ +DSV4_D_QK = 512 +DSV4_D_NOPE = 448 +DSV4_D_ROPE = 64 +DSV4_D_V = 512 +DSV4_TILE_SIZE = 64 +DSV4_NUM_TILES = 7 +DSV4_BYTES_PER_TOKEN_DATA = 576 # 448 nope + 128 rope +DSV4_BYTES_PER_TOKEN_SCALE = 8 # 7 scales + 1 padding + + +# ============================================================================ +# Helper: Process KV block and compute QK scores + accumulator update +# This is the core computation shared by both single and dual scope kernels +# ============================================================================ +@triton.jit +def _process_kv_block_aggressive( + # KV cache parameters + kv_block_base, + nope_rope_offset, + scale_base_offset, + valid, + valid_2d, + # Query tiles + q_0, + q_1, + q_2, + q_3, + q_4, + q_5, + q_6, + q_7, + # Accumulators (passed by reference via return) + acc_0, + acc_1, + acc_2, + acc_3, + acc_4, + acc_5, + acc_6, + acc_7, + # Softmax state + m_i, + l_i, + # Other parameters + offs_tile, + sm_scale, + # Constants + TILE_SIZE: tl.constexpr, + D_NOPE: tl.constexpr, + LOG2E: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """ + Process one block of KV tokens with batch loading. + Key optimization: Load all KV tiles first, then process them. + """ + NEG_INF = float("-inf") + + scale_ptrs = kv_block_base + scale_base_offset + scale_uint8_0 = tl.load(scale_ptrs, mask=valid, other=127).to(tl.uint8) + scale_uint8_1 = tl.load(scale_ptrs + 1, mask=valid, other=127).to(tl.uint8) + scale_uint8_2 = tl.load(scale_ptrs + 2, mask=valid, other=127).to(tl.uint8) + scale_uint8_3 = tl.load(scale_ptrs + 3, mask=valid, other=127).to(tl.uint8) + scale_uint8_4 = tl.load(scale_ptrs + 4, mask=valid, other=127).to(tl.uint8) + scale_uint8_5 = tl.load(scale_ptrs + 5, mask=valid, other=127).to(tl.uint8) + scale_uint8_6 = tl.load(scale_ptrs + 6, mask=valid, other=127).to(tl.uint8) + + tile_base = kv_block_base[:, None] + nope_rope_offset[:, None] + + # Batch load all tiles + nope_uint8_0 = tl.load(tile_base + offs_tile[None, :], mask=valid_2d, other=0) + nope_uint8_1 = tl.load( + tile_base + TILE_SIZE + offs_tile[None, :], mask=valid_2d, other=0 + ) + nope_uint8_2 = tl.load( + tile_base + 2 * TILE_SIZE + offs_tile[None, :], mask=valid_2d, other=0 + ) + nope_uint8_3 = tl.load( + tile_base + 3 * TILE_SIZE + offs_tile[None, :], mask=valid_2d, other=0 + ) + nope_uint8_4 = tl.load( + tile_base + 4 * TILE_SIZE + offs_tile[None, :], mask=valid_2d, other=0 + ) + nope_uint8_5 = tl.load( + tile_base + 5 * TILE_SIZE + offs_tile[None, :], mask=valid_2d, other=0 + ) + nope_uint8_6 = tl.load( + tile_base + 6 * TILE_SIZE + offs_tile[None, :], mask=valid_2d, other=0 + ) + rope_ptrs = tile_base + D_NOPE + offs_tile[None, :] * 2 + rope_lo = tl.load(rope_ptrs, mask=valid_2d, other=0).to(tl.uint16) + rope_hi = tl.load(rope_ptrs + 1, mask=valid_2d, other=0).to(tl.uint16) + + scale_bf16_0 = tl.math.exp2(scale_uint8_0.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_1 = tl.math.exp2(scale_uint8_1.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_2 = tl.math.exp2(scale_uint8_2.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_3 = tl.math.exp2(scale_uint8_3.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_4 = tl.math.exp2(scale_uint8_4.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_5 = tl.math.exp2(scale_uint8_5.to(tl.float32) - 127.0).to(tl.bfloat16) + scale_bf16_6 = tl.math.exp2(scale_uint8_6.to(tl.float32) - 127.0).to(tl.bfloat16) + + qk = tl.zeros([BLOCK_H, BLOCK_N], dtype=tl.float32) + + nope_fp8_0 = nope_uint8_0.to(tl.float8e4nv, bitcast=True) + kv_0 = (nope_fp8_0.to(tl.bfloat16) * scale_bf16_0[:, None]).to(tl.bfloat16) + kv_0 = tl.where(valid_2d, kv_0, 0.0) + qk += tl.dot(q_0, tl.trans(kv_0)).to(tl.float32) + + nope_fp8_1 = nope_uint8_1.to(tl.float8e4nv, bitcast=True) + kv_1 = (nope_fp8_1.to(tl.bfloat16) * scale_bf16_1[:, None]).to(tl.bfloat16) + kv_1 = tl.where(valid_2d, kv_1, 0.0) + qk += tl.dot(q_1, tl.trans(kv_1)).to(tl.float32) + + nope_fp8_2 = nope_uint8_2.to(tl.float8e4nv, bitcast=True) + kv_2 = (nope_fp8_2.to(tl.bfloat16) * scale_bf16_2[:, None]).to(tl.bfloat16) + kv_2 = tl.where(valid_2d, kv_2, 0.0) + qk += tl.dot(q_2, tl.trans(kv_2)).to(tl.float32) + + nope_fp8_3 = nope_uint8_3.to(tl.float8e4nv, bitcast=True) + kv_3 = (nope_fp8_3.to(tl.bfloat16) * scale_bf16_3[:, None]).to(tl.bfloat16) + kv_3 = tl.where(valid_2d, kv_3, 0.0) + qk += tl.dot(q_3, tl.trans(kv_3)).to(tl.float32) + + nope_fp8_4 = nope_uint8_4.to(tl.float8e4nv, bitcast=True) + kv_4 = (nope_fp8_4.to(tl.bfloat16) * scale_bf16_4[:, None]).to(tl.bfloat16) + kv_4 = tl.where(valid_2d, kv_4, 0.0) + qk += tl.dot(q_4, tl.trans(kv_4)).to(tl.float32) + + nope_fp8_5 = nope_uint8_5.to(tl.float8e4nv, bitcast=True) + kv_5 = (nope_fp8_5.to(tl.bfloat16) * scale_bf16_5[:, None]).to(tl.bfloat16) + kv_5 = tl.where(valid_2d, kv_5, 0.0) + qk += tl.dot(q_5, tl.trans(kv_5)).to(tl.float32) + + nope_fp8_6 = nope_uint8_6.to(tl.float8e4nv, bitcast=True) + kv_6 = (nope_fp8_6.to(tl.bfloat16) * scale_bf16_6[:, None]).to(tl.bfloat16) + kv_6 = tl.where(valid_2d, kv_6, 0.0) + qk += tl.dot(q_6, tl.trans(kv_6)).to(tl.float32) + + kv_7 = (rope_lo | (rope_hi << 8)).to(tl.bfloat16, bitcast=True) + kv_7 = tl.where(valid_2d, kv_7, 0.0) + qk += tl.dot(q_7, tl.trans(kv_7)).to(tl.float32) + + qk = qk * sm_scale + qk = tl.where(valid[None, :], qk, NEG_INF) + + m_ij = tl.max(qk, axis=1) + m_new = tl.maximum(m_i, m_ij) + alpha = tl.where(m_i == NEG_INF, 0.0, tl.math.exp2((m_i - m_new) * LOG2E)) + p = tl.where(qk == NEG_INF, 0.0, tl.math.exp2((qk - m_new[:, None]) * LOG2E)) + l_new = alpha * l_i + tl.sum(p, axis=1) + p_bf16 = p.to(tl.bfloat16) + + acc_0 = acc_0 * alpha[:, None] + tl.dot(p_bf16, kv_0).to(tl.float32) + acc_1 = acc_1 * alpha[:, None] + tl.dot(p_bf16, kv_1).to(tl.float32) + acc_2 = acc_2 * alpha[:, None] + tl.dot(p_bf16, kv_2).to(tl.float32) + acc_3 = acc_3 * alpha[:, None] + tl.dot(p_bf16, kv_3).to(tl.float32) + acc_4 = acc_4 * alpha[:, None] + tl.dot(p_bf16, kv_4).to(tl.float32) + acc_5 = acc_5 * alpha[:, None] + tl.dot(p_bf16, kv_5).to(tl.float32) + acc_6 = acc_6 * alpha[:, None] + tl.dot(p_bf16, kv_6).to(tl.float32) + acc_7 = acc_7 * alpha[:, None] + tl.dot(p_bf16, kv_7).to(tl.float32) + + return acc_0, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, m_new, l_new + + +# ============================================================================ +# DSV4 Fused Gather+Dequant+Attention Kernel (Single Scope) +# ============================================================================ +@triton.autotune( + configs=[ + # Fused gather+dequant+attention kernel. + # Two axes: BLOCK_H × BLOCK_N, with BLOCK_N being the key perf knob + # for h_q=64 where fewer BLOCK_H values affect the grid. + # BLOCK_N=64: better for large topk (less register pressure per iter). + # BLOCK_N=128: better for small topk (fewer iterations). + # num_warps=4: fused kernel is compute-bound. + triton.Config({"BLOCK_H": 16, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 16, "BLOCK_N": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 128}, num_warps=4, num_stages=1), + ], + key=["total_tokens_bucket", "h_q", "topk"], +) +@triton.jit +def _fused_gather_attn_dsv4_kernel( + Q, + KV_Cache, + Indices, + TopkLength, + AttnSink, + Output, + LSE, + sm_scale, + total_tokens, + total_tokens_bucket, + h_q, + topk, + num_blocks, + block_size, + s_q, + stride_q_t, + stride_q_h, + stride_q_d, + stride_kv_block, + stride_idx_t, + stride_idx_k, + stride_o_t, + stride_o_h, + stride_o_d, + stride_lse_t, + stride_lse_h, + HAS_TOPK_LENGTH: tl.constexpr, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """Fused gather+dequant+attention kernel for DSV4.""" + LOG2E: tl.constexpr = 1.4426950408889634 + D_NOPE: tl.constexpr = 448 + D_ROPE: tl.constexpr = 64 + TILE_SIZE: tl.constexpr = 64 + BYTES_PER_TOKEN_DATA: tl.constexpr = 576 + BYTES_PER_TOKEN_SCALE: tl.constexpr = 8 + + # OPTIMIZED: Swapped grid - pid_h first for better cache locality + pid_h = tl.program_id(0) + pid_t = tl.program_id(1) + pid_t_64 = pid_t.to(tl.int64) + + NEG_INF = float("-inf") + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + + m_i = tl.full([BLOCK_H], NEG_INF, dtype=tl.float32) + l_i = tl.zeros([BLOCK_H], dtype=tl.float32) + + acc_0 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_1 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_2 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_3 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_4 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_5 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_6 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_7 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + + stride_q_t_64 = tl.cast(stride_q_t, tl.int64) + q_base = Q + pid_t_64 * stride_q_t_64 + + batch_idx = pid_t // s_q + offs_tile = tl.arange(0, TILE_SIZE) + + q_0 = tl.load( + q_base + offs_h[:, None] * stride_q_h + offs_tile[None, :] * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_1 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_2 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (2 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_3 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (3 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_4 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (4 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_5 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (5 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_6 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (6 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_7 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (7 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + + # Early-exit: pre-load topk_len and skip invalid blocks + if HAS_TOPK_LENGTH: + topk_len = tl.load(TopkLength + batch_idx) + + for n_start in range(0, topk, BLOCK_N): + # Skip entire block if beyond valid topk range + should_compute = not HAS_TOPK_LENGTH or n_start < topk_len + if should_compute: + offs_n = n_start + tl.arange(0, BLOCK_N) + mask_n = offs_n < topk + + idx_ptrs = Indices + pid_t * stride_idx_t + offs_n * stride_idx_k + indices = tl.load(idx_ptrs, mask=mask_n, other=-1) + + is_invalid = indices == -1 + if HAS_TOPK_LENGTH: + is_invalid = is_invalid | (offs_n >= topk_len) + + valid = mask_n & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size + offset_in_block = indices_clamped % block_size + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + stride_kv_block_64 = tl.cast(stride_kv_block, tl.int64) + kv_block_base = KV_Cache + block_idx_64 * stride_kv_block_64 + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size * BYTES_PER_TOKEN_DATA + + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + valid_2d = valid[:, None] + + # Use helper function for KV processing + acc_0, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, m_i, l_i = ( + _process_kv_block_aggressive( + kv_block_base, + nope_rope_offset, + scale_base_offset, + valid, + valid_2d, + q_0, + q_1, + q_2, + q_3, + q_4, + q_5, + q_6, + q_7, + acc_0, + acc_1, + acc_2, + acc_3, + acc_4, + acc_5, + acc_6, + acc_7, + m_i, + l_i, + offs_tile, + sm_scale, + TILE_SIZE, + D_NOPE, + LOG2E, + BLOCK_H, + BLOCK_N, + ) + ) + + # Finalize + lse = m_i + tl.math.log2(tl.where(l_i == 0.0, 1.0, l_i)) / LOG2E + is_lonely_q = l_i == 0.0 + + if HAS_ATTN_SINK: + attn_sink_vals = tl.load(AttnSink + offs_h, mask=mask_h, other=0.0) + exp_attn_sink_minus_m = tl.math.exp2((attn_sink_vals - m_i) * LOG2E) + denominator = l_i + exp_attn_sink_minus_m + denominator = tl.where(denominator == 0.0, 1.0, denominator) + output_scale = 1.0 / denominator + else: + output_scale = tl.where(l_i == 0.0, 0.0, 1.0 / l_i) + + acc_0 = tl.where(is_lonely_q[:, None], 0.0, acc_0 * output_scale[:, None]) + acc_1 = tl.where(is_lonely_q[:, None], 0.0, acc_1 * output_scale[:, None]) + acc_2 = tl.where(is_lonely_q[:, None], 0.0, acc_2 * output_scale[:, None]) + acc_3 = tl.where(is_lonely_q[:, None], 0.0, acc_3 * output_scale[:, None]) + acc_4 = tl.where(is_lonely_q[:, None], 0.0, acc_4 * output_scale[:, None]) + acc_5 = tl.where(is_lonely_q[:, None], 0.0, acc_5 * output_scale[:, None]) + acc_6 = tl.where(is_lonely_q[:, None], 0.0, acc_6 * output_scale[:, None]) + acc_7 = tl.where(is_lonely_q[:, None], 0.0, acc_7 * output_scale[:, None]) + lse = tl.where(is_lonely_q, float("+inf"), lse) + + stride_o_t_64 = tl.cast(stride_o_t, tl.int64) + o_base = Output + pid_t_64 * stride_o_t_64 + + # Optimized output stores with pre-computed row base pointers + # Convert to bfloat16 first (batch conversion) + o_0 = acc_0.to(tl.bfloat16) + o_1 = acc_1.to(tl.bfloat16) + o_2 = acc_2.to(tl.bfloat16) + o_3 = acc_3.to(tl.bfloat16) + o_4 = acc_4.to(tl.bfloat16) + o_5 = acc_5.to(tl.bfloat16) + o_6 = acc_6.to(tl.bfloat16) + o_7 = acc_7.to(tl.bfloat16) + + # Pre-compute row base pointers (shared across all 8 stores) + row_ptrs = o_base + offs_h[:, None] * stride_o_h + + # Store all 8 tiles with optimized pointer arithmetic + tl.store(row_ptrs + offs_tile[None, :] * stride_o_d, o_0, mask=mask_h[:, None]) + tl.store( + row_ptrs + (TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_1, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (2 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_2, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (3 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_3, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (4 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_4, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (5 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_5, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (6 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_6, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (7 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_7, + mask=mask_h[:, None], + ) + + lse_ptrs = LSE + pid_t * stride_lse_t + offs_h * stride_lse_h + tl.store(lse_ptrs, lse, mask=mask_h) + + +# Threshold for disabling AMD buffer_ops optimization +# When KV cache size exceeds INT32_MAX, buffer_ops can cause int32 overflow +# INT32_MAX = 2^31 - 1 = 2,147,483,647 bytes (~2GB) +BUFFER_OPS_DISABLE_THRESHOLD = 2 * 1024 * 1024 * 1024 # 2GB + + +def fused_gather_attn_decode_dsv4( + q: torch.Tensor, + kv_cache: torch.Tensor, + indices: torch.Tensor, + block_size: int, + sm_scale: float, + topk_length: Optional[torch.Tensor] = None, + attn_sink: Optional[torch.Tensor] = None, + s_q: int = 1, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Fused gather+dequant+attention for DSV4. + Uses Split-K optimization for large topk (>= 8192). + + Args: + q: Query tensor [total_tokens, h_q, d_qk] + kv_cache: Quantized KV cache + indices: KV indices [total_tokens, topk] + block_size: Block size for KV cache + sm_scale: Softmax scale + topk_length: Optional per-batch topk length [b] + attn_sink: Optional attention sink values [h_q] + s_q: Sequence length per batch + + Returns: + output: Attention output [total_tokens, h_q, d_v] + lse: Log-sum-exp values [total_tokens, h_q] + """ + total_tokens, h_q, d_qk = q.shape + topk = indices.shape[1] + d_v = DSV4_D_V + device = q.device + + kv_uint8 = kv_cache.view(torch.uint8) + num_blocks = kv_cache.shape[0] + stride_kv_block = kv_uint8.stride(0) + kv_flat = kv_uint8.reshape(num_blocks, -1) + + if q.dtype != torch.bfloat16 or not q.is_contiguous(): + q = q.to(torch.bfloat16).contiguous() + + if not indices.is_contiguous(): + indices = indices.contiguous() + + kv_cache_size = stride_kv_block * num_blocks + disable_buffer_ops = kv_cache_size > BUFFER_OPS_DISABLE_THRESHOLD + + # Use Split-K for large topk + if topk >= SPLITK_TOPK_THRESHOLD: + split_k = _select_split_k(topk, h_q, total_tokens) + topk_per_split = (topk + split_k - 1) // split_k + + partial_output = torch.empty( + split_k, total_tokens, h_q, d_v, dtype=torch.float32, device=device + ) + partial_lse = torch.empty( + split_k, total_tokens, h_q, dtype=torch.float32, device=device + ) + output = torch.empty( + total_tokens, h_q, d_v, dtype=torch.bfloat16, device=device + ) + lse = torch.empty(total_tokens, h_q, dtype=torch.float32, device=device) + + topk_length_tensor = topk_length if topk_length is not None else lse[:1, 0] + attn_sink_tensor = attn_sink if attn_sink is not None else lse[0, :] + + # Use autotuned grid + grid_splitk = lambda meta: ( + triton.cdiv(h_q, meta["BLOCK_H"]), + total_tokens, + split_k, + ) + + def run_splitk_kernel(): + _fused_gather_attn_dsv4_splitk_kernel[grid_splitk]( + q, + kv_flat, + indices, + topk_length_tensor, + partial_output, + partial_lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + topk, + num_blocks, + block_size, + s_q, + topk_per_split, + q.stride(0), + q.stride(1), + q.stride(2), + stride_kv_block, + indices.stride(0), + indices.stride(1), + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + HAS_TOPK_LENGTH=topk_length is not None, + ) + + if disable_buffer_ops: + with triton.knobs.amd.scope(): + triton.knobs.amd.use_buffer_ops = False + run_splitk_kernel() + else: + run_splitk_kernel() + + # Use autotuned combine kernel for split_k=8 + if split_k == 8: + # Autotuned kernel - grid is determined by autotune + grid_combine = lambda meta: ( + total_tokens, + triton.cdiv(h_q, meta["BLOCK_H"]), + ) + _combine_splitk_kernel_8_optimized[grid_combine]( + partial_output, + partial_lse, + attn_sink_tensor, + output, + lse, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + d_v, + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_ATTN_SINK=attn_sink is not None, + ) + else: + BLOCK_H_COMBINE = 16 + BLOCK_D_COMBINE = 128 + grid_combine = (total_tokens, triton.cdiv(h_q, BLOCK_H_COMBINE)) + + # Select appropriate combine kernel based on split_k + if split_k == 2: + combine_kernel = _combine_splitk_kernel_2 + elif split_k == 4: + combine_kernel = _combine_splitk_kernel + else: + raise ValueError(f"Unsupported split_k: {split_k}") + + combine_kernel[grid_combine]( + partial_output, + partial_lse, + attn_sink_tensor, + output, + lse, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + d_v, + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_ATTN_SINK=attn_sink is not None, + BLOCK_H=BLOCK_H_COMBINE, + BLOCK_D=BLOCK_D_COMBINE, + num_warps=4, + num_stages=1, + ) + + return output, lse + + # Use original kernel for smaller topk + output = torch.empty(total_tokens, h_q, d_v, dtype=torch.bfloat16, device=device) + lse = torch.empty(total_tokens, h_q, dtype=torch.float32, device=device) + + topk_length_tensor = topk_length if topk_length is not None else lse[:1, 0] + attn_sink_tensor = attn_sink if attn_sink is not None else lse[0, :] + + grid = lambda meta: (triton.cdiv(h_q, meta["BLOCK_H"]), total_tokens) + + def run_kernel(): + _fused_gather_attn_dsv4_kernel[grid]( + q, + kv_flat, + indices, + topk_length_tensor, + attn_sink_tensor, + output, + lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + topk, + num_blocks, + block_size, + s_q, + q.stride(0), + q.stride(1), + q.stride(2), + stride_kv_block, + indices.stride(0), + indices.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_TOPK_LENGTH=topk_length is not None, + HAS_ATTN_SINK=attn_sink is not None, + ) + + if disable_buffer_ops: + with triton.knobs.amd.scope(): + triton.knobs.amd.use_buffer_ops = False + run_kernel() + else: + run_kernel() + + return output, lse + + +# Uses helper function to eliminate code duplication +# ============================================================================ + + +def _prune_dual_scope_configs(configs, named_args, **kwargs): + """Prune configs where BLOCK_H > h_q for the dual-scope kernel. + + When BLOCK_H > h_q, cdiv(h_q, BLOCK_H) = 1 regardless of BLOCK_H value, + so larger BLOCK_H gives the same grid but may have worse register allocation. + Keep only the smallest BLOCK_H that gives cdiv(h_q, BLOCK_H) = 1, plus + any BLOCK_H <= h_q configs. + + For h_q=64: keep BLOCK_H <= 64 (removes BLOCK_H=128 which gives same grid) + For h_q=128: keep all (all give different grid sizes) + """ + h_q = named_args.get("h_q", 128) + pruned = [c for c in configs if c.kwargs.get("BLOCK_H", 16) <= h_q] + return pruned if pruned else configs + + +@triton.autotune( + configs=[ + # Dual-scope fused gather+dequant+attention. + # Three axes: BLOCK_H × BLOCK_N × (warps, stages). + # - BLOCK_H: {16, 32, 64, 128} covers h_q=64 and h_q=128. + # - BLOCK_N: {64, 128}. BLOCK_N=64 better for large topk, 128 for small topk. + # - _prune_dual_scope_configs removes BLOCK_H > h_q configs (e.g. BLOCK_H=128 + # is pruned when h_q=64 since it gives the same grid as BLOCK_H=64). + # warps=4: baseline configs + triton.Config({"BLOCK_H": 16, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 16, "BLOCK_N": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 32, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 32, "BLOCK_N": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 128}, num_warps=4, num_stages=1), + # warps=8: for memory-bound scenarios + triton.Config({"BLOCK_H": 16, "BLOCK_N": 64}, num_warps=8, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 64}, num_warps=8, num_stages=1), + ], + key=["total_tokens_bucket", "h_q", "topk_main", "topk_extra"], + prune_configs_by={"early_config_prune": _prune_dual_scope_configs}, +) +@triton.jit +def _fused_gather_attn_dsv4_dual_scope_kernel( + Q, + KV_Cache_Main, + Indices_Main, + TopkLength_Main, + KV_Cache_Extra, + Indices_Extra, + TopkLength_Extra, + AttnSink, + Output, + LSE, + sm_scale, + total_tokens, + total_tokens_bucket, + h_q, + topk_main, + num_blocks_main, + block_size_main, + topk_extra, + num_blocks_extra, + block_size_extra, + s_q, + stride_q_t, + stride_q_h, + stride_q_d, + stride_kv_block_main, + stride_kv_block_extra, + stride_idx_main_t, + stride_idx_main_k, + stride_idx_extra_t, + stride_idx_extra_k, + stride_o_t, + stride_o_h, + stride_o_d, + stride_lse_t, + stride_lse_h, + HAS_TOPK_LENGTH_MAIN: tl.constexpr, + HAS_TOPK_LENGTH_EXTRA: tl.constexpr, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """ + OPTIMIZED fused gather+dequant+attention kernel for DSV4 with dual scope. + + This version uses a helper function (_process_kv_block_aggressive) to + eliminate the ~200 lines of duplicated code between MAIN and EXTRA scope + processing loops. + + The kernel processes: + 1. MAIN scope: topk_main tokens from KV_Cache_Main + 2. EXTRA scope: topk_extra tokens from KV_Cache_Extra + + Both scopes contribute to the same online softmax accumulator. + """ + LOG2E: tl.constexpr = 1.4426950408889634 + D_NOPE: tl.constexpr = 448 + D_ROPE: tl.constexpr = 64 + TILE_SIZE: tl.constexpr = 64 + BYTES_PER_TOKEN_DATA: tl.constexpr = 576 + BYTES_PER_TOKEN_SCALE: tl.constexpr = 8 + + # OPTIMIZED: Swapped grid - pid_h first for better cache locality + pid_h = tl.program_id(0) + pid_t = tl.program_id(1) + pid_t_64 = pid_t.to(tl.int64) + + NEG_INF = float("-inf") + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + + # Initialize accumulators + m_i = tl.full([BLOCK_H], NEG_INF, dtype=tl.float32) + l_i = tl.zeros([BLOCK_H], dtype=tl.float32) + + acc_0 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_1 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_2 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_3 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_4 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_5 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_6 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_7 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + + stride_q_t_64 = tl.cast(stride_q_t, tl.int64) + q_base = Q + pid_t_64 * stride_q_t_64 + + batch_idx = pid_t // s_q + offs_tile = tl.arange(0, TILE_SIZE) + + # Load Q tiles (shared by both scopes) + q_0 = tl.load( + q_base + offs_h[:, None] * stride_q_h + offs_tile[None, :] * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_1 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_2 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (2 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_3 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (3 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_4 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (4 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_5 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (5 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_6 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (6 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_7 = tl.load( + q_base + + offs_h[:, None] * stride_q_h + + (7 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + + # ======================================================================== + # Process MAIN scope + # ======================================================================== + # Early-exit: pre-load topk_len and skip invalid blocks + if HAS_TOPK_LENGTH_MAIN: + topk_len = tl.load(TopkLength_Main + batch_idx) + + for n_start in range(0, topk_main, BLOCK_N): + # Skip entire block if beyond valid topk range + should_compute = not HAS_TOPK_LENGTH_MAIN or n_start < topk_len + if should_compute: + offs_n = n_start + tl.arange(0, BLOCK_N) + mask_n = offs_n < topk_main + + idx_ptrs = ( + Indices_Main + pid_t * stride_idx_main_t + offs_n * stride_idx_main_k + ) + indices = tl.load(idx_ptrs, mask=mask_n, other=-1) + + is_invalid = indices == -1 + if HAS_TOPK_LENGTH_MAIN: + is_invalid = is_invalid | (offs_n >= topk_len) + + valid = mask_n & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size_main + offset_in_block = indices_clamped % block_size_main + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + stride_kv_block_main_64 = tl.cast(stride_kv_block_main, tl.int64) + kv_block_base = KV_Cache_Main + block_idx_64 * stride_kv_block_main_64 + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size_main * BYTES_PER_TOKEN_DATA + + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + valid_2d = valid[:, None] + + # Use helper function for KV processing + acc_0, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, m_i, l_i = ( + _process_kv_block_aggressive( + kv_block_base, + nope_rope_offset, + scale_base_offset, + valid, + valid_2d, + q_0, + q_1, + q_2, + q_3, + q_4, + q_5, + q_6, + q_7, + acc_0, + acc_1, + acc_2, + acc_3, + acc_4, + acc_5, + acc_6, + acc_7, + m_i, + l_i, + offs_tile, + sm_scale, + TILE_SIZE, + D_NOPE, + LOG2E, + BLOCK_H, + BLOCK_N, + ) + ) + + # ======================================================================== + # Process EXTRA scope + # ======================================================================== + # Early-exit: pre-load topk_len and skip invalid blocks + if HAS_TOPK_LENGTH_EXTRA: + topk_len = tl.load(TopkLength_Extra + batch_idx) + + for n_start in range(0, topk_extra, BLOCK_N): + # Skip entire block if beyond valid topk range + should_compute = not HAS_TOPK_LENGTH_EXTRA or n_start < topk_len + if should_compute: + offs_n = n_start + tl.arange(0, BLOCK_N) + mask_n = offs_n < topk_extra + + idx_ptrs = ( + Indices_Extra + pid_t * stride_idx_extra_t + offs_n * stride_idx_extra_k + ) + indices = tl.load(idx_ptrs, mask=mask_n, other=-1) + + is_invalid = indices == -1 + if HAS_TOPK_LENGTH_EXTRA: + is_invalid = is_invalid | (offs_n >= topk_len) + + valid = mask_n & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size_extra + offset_in_block = indices_clamped % block_size_extra + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + stride_kv_block_extra_64 = tl.cast(stride_kv_block_extra, tl.int64) + kv_block_base = KV_Cache_Extra + block_idx_64 * stride_kv_block_extra_64 + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size_extra * BYTES_PER_TOKEN_DATA + + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + valid_2d = valid[:, None] + + # Use helper function for KV processing + acc_0, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, m_i, l_i = ( + _process_kv_block_aggressive( + kv_block_base, + nope_rope_offset, + scale_base_offset, + valid, + valid_2d, + q_0, + q_1, + q_2, + q_3, + q_4, + q_5, + q_6, + q_7, + acc_0, + acc_1, + acc_2, + acc_3, + acc_4, + acc_5, + acc_6, + acc_7, + m_i, + l_i, + offs_tile, + sm_scale, + TILE_SIZE, + D_NOPE, + LOG2E, + BLOCK_H, + BLOCK_N, + ) + ) + + # ======================================================================== + # Finalize: compute LSE and output + # ======================================================================== + lse = m_i + tl.math.log2(tl.where(l_i == 0.0, 1.0, l_i)) / LOG2E + is_lonely_q = l_i == 0.0 + + # Compute output scale + if HAS_ATTN_SINK: + attn_sink_vals = tl.load(AttnSink + offs_h, mask=mask_h, other=0.0) + exp_attn_sink_minus_m = tl.math.exp2((attn_sink_vals - m_i) * LOG2E) + denominator = l_i + exp_attn_sink_minus_m + denominator = tl.where(denominator == 0.0, 1.0, denominator) + output_scale = 1.0 / denominator + else: + output_scale = tl.where(l_i == 0.0, 0.0, 1.0 / l_i) + + # Apply output scaling and handle lonely queries + acc_0 = tl.where(is_lonely_q[:, None], 0.0, acc_0 * output_scale[:, None]) + acc_1 = tl.where(is_lonely_q[:, None], 0.0, acc_1 * output_scale[:, None]) + acc_2 = tl.where(is_lonely_q[:, None], 0.0, acc_2 * output_scale[:, None]) + acc_3 = tl.where(is_lonely_q[:, None], 0.0, acc_3 * output_scale[:, None]) + acc_4 = tl.where(is_lonely_q[:, None], 0.0, acc_4 * output_scale[:, None]) + acc_5 = tl.where(is_lonely_q[:, None], 0.0, acc_5 * output_scale[:, None]) + acc_6 = tl.where(is_lonely_q[:, None], 0.0, acc_6 * output_scale[:, None]) + acc_7 = tl.where(is_lonely_q[:, None], 0.0, acc_7 * output_scale[:, None]) + lse = tl.where(is_lonely_q, float("+inf"), lse) + + stride_o_t_64 = tl.cast(stride_o_t, tl.int64) + o_base = Output + pid_t_64 * stride_o_t_64 + + # Optimized output stores with pre-computed row base pointers + # Convert to bfloat16 first (batch conversion) + o_0 = acc_0.to(tl.bfloat16) + o_1 = acc_1.to(tl.bfloat16) + o_2 = acc_2.to(tl.bfloat16) + o_3 = acc_3.to(tl.bfloat16) + o_4 = acc_4.to(tl.bfloat16) + o_5 = acc_5.to(tl.bfloat16) + o_6 = acc_6.to(tl.bfloat16) + o_7 = acc_7.to(tl.bfloat16) + + # Pre-compute row base pointers (shared across all 8 stores) + row_ptrs = o_base + offs_h[:, None] * stride_o_h + + # Store all 8 tiles with optimized pointer arithmetic + tl.store(row_ptrs + offs_tile[None, :] * stride_o_d, o_0, mask=mask_h[:, None]) + tl.store( + row_ptrs + (TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_1, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (2 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_2, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (3 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_3, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (4 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_4, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (5 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_5, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (6 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_6, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (7 * TILE_SIZE + offs_tile[None, :]) * stride_o_d, + o_7, + mask=mask_h[:, None], + ) + + lse_ptrs = LSE + pid_t * stride_lse_t + offs_h * stride_lse_h + tl.store(lse_ptrs, lse, mask=mask_h) + + +def _prune_splitk_configs(configs, named_args, **kwargs): + """Prune BLOCK_H=16 configs for large batch sizes to avoid CU oversubscription. + + With h_q=128 and BLOCK_H=16, the grid has cdiv(128,16)=8 H-blocks. + At bs=32 with split_k=2, this creates 8*32*2=512 blocks (200% CU), + causing performance regression from oversubscription. + + For small batch sizes (bucket <= 8), BLOCK_H=16 provides better + parallelism and is ~10% faster in CUDA graph replay. + """ + total_tokens_bucket = named_args.get("total_tokens_bucket", 32) + if total_tokens_bucket > 8: + # Remove BLOCK_H=16 configs for large batch sizes + pruned = [c for c in configs if c.kwargs.get("BLOCK_H", 32) > 16] + if pruned: + return pruned + return configs + + +# ============================================================================ +# Split-K Kernel for Dual Scope +# ============================================================================ +@triton.autotune( + configs=[ + # Split-K dual-scope fused kernel. + # - Split-K adds parallelism in K dim (2-8 splits). + # - BLOCK_N={64,128}: BLOCK_N=64 better for large topk_per_split. + # - num_warps=4: compute-bound fused kernel. + # - BLOCK_H={16,64}: covers h_q=64 and h_q=128. + triton.Config({"BLOCK_H": 16, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 16, "BLOCK_N": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 64}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 128}, num_warps=4, num_stages=1), + ], + key=["total_tokens_bucket", "h_q", "topk_per_split"], + prune_configs_by={"early_config_prune": _prune_splitk_configs}, +) +@triton.jit +def _fused_gather_attn_dsv4_dual_scope_splitk_kernel( + Q, + KV_Cache_Main, + Indices_Main, + TopkLength_Main, + KV_Cache_Extra, + Indices_Extra, + TopkLength_Extra, + PartialOutput, + PartialLSE, + sm_scale, + total_tokens, + total_tokens_bucket, + h_q, + topk_main, + num_blocks_main, + block_size_main, + topk_extra, + num_blocks_extra, + block_size_extra, + s_q, + topk_per_split, + stride_q_t, + stride_q_h, + stride_q_d, + stride_kv_block_main, + stride_kv_block_extra, + stride_idx_main_t, + stride_idx_main_k, + stride_idx_extra_t, + stride_idx_extra_k, + stride_po_s, + stride_po_t, + stride_po_h, + stride_po_d, + stride_plse_s, + stride_plse_t, + stride_plse_h, + HAS_TOPK_LENGTH_MAIN: tl.constexpr, + HAS_TOPK_LENGTH_EXTRA: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """ + Split-K fused gather+dequant+attention kernel for DSV4 with dual scope. + + This kernel processes a portion of the combined topk range (main + extra). + Each split handles topk_per_split tokens from the combined range. + """ + LOG2E: tl.constexpr = 1.4426950408889634 + D_NOPE: tl.constexpr = 448 + TILE_SIZE: tl.constexpr = 64 + BYTES_PER_TOKEN_DATA: tl.constexpr = 576 + BYTES_PER_TOKEN_SCALE: tl.constexpr = 8 + + pid_h = tl.program_id(0) + pid_t = tl.program_id(1) + pid_k = tl.program_id(2) + pid_t_64 = pid_t.to(tl.int64) + + NEG_INF = float("-inf") + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + + # Calculate the range for this split + total_topk = topk_main + topk_extra + k_start = pid_k * topk_per_split + k_end = tl.minimum(k_start + topk_per_split, total_topk) + + # Initialize accumulators + m_i = tl.full([BLOCK_H], NEG_INF, dtype=tl.float32) + l_i = tl.zeros([BLOCK_H], dtype=tl.float32) + + acc_0 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_1 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_2 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_3 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_4 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_5 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_6 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_7 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + + stride_q_t_64 = tl.cast(stride_q_t, tl.int64) + q_base = Q + pid_t_64 * stride_q_t_64 + + batch_idx = pid_t // s_q + offs_tile = tl.arange(0, TILE_SIZE) + + # Load Q tiles (shared by both scopes) + q_row_base = q_base + offs_h[:, None] * stride_q_h + q_0 = tl.load( + q_row_base + offs_tile[None, :] * stride_q_d, mask=mask_h[:, None], other=0.0 + ).to(tl.bfloat16) + q_1 = tl.load( + q_row_base + (TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_2 = tl.load( + q_row_base + (2 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_3 = tl.load( + q_row_base + (3 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_4 = tl.load( + q_row_base + (4 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_5 = tl.load( + q_row_base + (5 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_6 = tl.load( + q_row_base + (6 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_7 = tl.load( + q_row_base + (7 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + + stride_kv_block_main_64 = tl.cast(stride_kv_block_main, tl.int64) + stride_kv_block_extra_64 = tl.cast(stride_kv_block_extra, tl.int64) + + # Process the combined range [k_start, k_end) + # First, process MAIN scope portion (indices 0 to topk_main-1) + main_start = k_start + main_end = tl.minimum(k_end, topk_main) + + # Early-exit: pre-load topk_len and skip invalid blocks + if HAS_TOPK_LENGTH_MAIN: + topk_len = tl.load(TopkLength_Main + batch_idx) + + for n_start in range(main_start, main_end, BLOCK_N): + # Skip entire block if beyond valid topk range + should_compute = not HAS_TOPK_LENGTH_MAIN or n_start < topk_len + if should_compute: + offs_n = n_start + tl.arange(0, BLOCK_N) + mask_n = offs_n < main_end + + idx_ptrs = ( + Indices_Main + pid_t * stride_idx_main_t + offs_n * stride_idx_main_k + ) + indices = tl.load(idx_ptrs, mask=mask_n, other=-1) + + is_invalid = indices == -1 + if HAS_TOPK_LENGTH_MAIN: + is_invalid = is_invalid | (offs_n >= topk_len) + + valid = mask_n & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size_main + offset_in_block = indices_clamped % block_size_main + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + kv_block_base = KV_Cache_Main + block_idx_64 * stride_kv_block_main_64 + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size_main * BYTES_PER_TOKEN_DATA + + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + valid_2d = valid[:, None] + + acc_0, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, m_i, l_i = ( + _process_kv_block_aggressive( + kv_block_base, + nope_rope_offset, + scale_base_offset, + valid, + valid_2d, + q_0, + q_1, + q_2, + q_3, + q_4, + q_5, + q_6, + q_7, + acc_0, + acc_1, + acc_2, + acc_3, + acc_4, + acc_5, + acc_6, + acc_7, + m_i, + l_i, + offs_tile, + sm_scale, + TILE_SIZE, + D_NOPE, + LOG2E, + BLOCK_H, + BLOCK_N, + ) + ) + + # Process EXTRA scope portion (indices topk_main to topk_main+topk_extra-1) + extra_global_start = tl.maximum(k_start, topk_main) + extra_global_end = k_end + + # Early-exit: pre-load topk_len and skip invalid blocks + if HAS_TOPK_LENGTH_EXTRA: + topk_len = tl.load(TopkLength_Extra + batch_idx) + + for n_global in range(extra_global_start, extra_global_end, BLOCK_N): + # Skip entire block if beyond valid topk range + should_compute = not HAS_TOPK_LENGTH_EXTRA or (n_global - topk_main) < topk_len + if should_compute: + offs_n_local = (n_global - topk_main) + tl.arange(0, BLOCK_N) + offs_n_global = n_global + tl.arange(0, BLOCK_N) + mask_n = offs_n_global < extra_global_end + + idx_ptrs = ( + Indices_Extra + + pid_t * stride_idx_extra_t + + offs_n_local * stride_idx_extra_k + ) + indices = tl.load(idx_ptrs, mask=mask_n, other=-1) + + is_invalid = indices == -1 + if HAS_TOPK_LENGTH_EXTRA: + is_invalid = is_invalid | (offs_n_local >= topk_len) + + valid = mask_n & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size_extra + offset_in_block = indices_clamped % block_size_extra + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + kv_block_base = KV_Cache_Extra + block_idx_64 * stride_kv_block_extra_64 + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size_extra * BYTES_PER_TOKEN_DATA + + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + valid_2d = valid[:, None] + + acc_0, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, m_i, l_i = ( + _process_kv_block_aggressive( + kv_block_base, + nope_rope_offset, + scale_base_offset, + valid, + valid_2d, + q_0, + q_1, + q_2, + q_3, + q_4, + q_5, + q_6, + q_7, + acc_0, + acc_1, + acc_2, + acc_3, + acc_4, + acc_5, + acc_6, + acc_7, + m_i, + l_i, + offs_tile, + sm_scale, + TILE_SIZE, + D_NOPE, + LOG2E, + BLOCK_H, + BLOCK_N, + ) + ) + + # Finalize: compute partial LSE and store partial output + lse = m_i + tl.math.log2(tl.where(l_i == 0.0, 1.0, l_i)) / LOG2E + is_lonely_q = l_i == 0.0 + + output_scale = tl.where(l_i == 0.0, 0.0, 1.0 / l_i) + + acc_0 = tl.where(is_lonely_q[:, None], 0.0, acc_0 * output_scale[:, None]) + acc_1 = tl.where(is_lonely_q[:, None], 0.0, acc_1 * output_scale[:, None]) + acc_2 = tl.where(is_lonely_q[:, None], 0.0, acc_2 * output_scale[:, None]) + acc_3 = tl.where(is_lonely_q[:, None], 0.0, acc_3 * output_scale[:, None]) + acc_4 = tl.where(is_lonely_q[:, None], 0.0, acc_4 * output_scale[:, None]) + acc_5 = tl.where(is_lonely_q[:, None], 0.0, acc_5 * output_scale[:, None]) + acc_6 = tl.where(is_lonely_q[:, None], 0.0, acc_6 * output_scale[:, None]) + acc_7 = tl.where(is_lonely_q[:, None], 0.0, acc_7 * output_scale[:, None]) + lse = tl.where(is_lonely_q, float("+inf"), lse) + + # Store partial output + stride_po_s_64 = tl.cast(stride_po_s, tl.int64) + stride_po_t_64 = tl.cast(stride_po_t, tl.int64) + po_base = PartialOutput + pid_k * stride_po_s_64 + pid_t_64 * stride_po_t_64 + + # Store partial output as float32 for better precision in combine kernel + row_ptrs = po_base + offs_h[:, None] * stride_po_h + + tl.store(row_ptrs + offs_tile[None, :] * stride_po_d, acc_0, mask=mask_h[:, None]) + tl.store( + row_ptrs + (TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_1, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (2 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_2, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (3 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_3, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (4 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_4, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (5 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_5, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (6 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_6, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (7 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_7, + mask=mask_h[:, None], + ) + + # Store partial LSE + stride_plse_s_64 = tl.cast(stride_plse_s, tl.int64) + stride_plse_t_64 = tl.cast(stride_plse_t, tl.int64) + lse_ptrs = ( + PartialLSE + + pid_k * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h + ) + tl.store(lse_ptrs, lse, mask=mask_h) + + +def fused_gather_attn_decode_dsv4_dual_scope( + q: torch.Tensor, + kv_cache_main: torch.Tensor, + indices_main: torch.Tensor, + block_size_main: int, + kv_cache_extra: torch.Tensor, + indices_extra: torch.Tensor, + block_size_extra: int, + sm_scale: float, + topk_length_main: Optional[torch.Tensor] = None, + topk_length_extra: Optional[torch.Tensor] = None, + attn_sink: Optional[torch.Tensor] = None, + s_q: int = 1, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Fused gather+dequant+attention for DSV4 with dual scope (main + extra). + Uses Split-K optimization for large total_topk (>= SPLITK_TOPK_THRESHOLD). + + Args: + q: Query tensor [total_tokens, h_q, d_qk] + kv_cache_main: Quantized main KV cache + indices_main: Main KV indices [total_tokens, topk_main] + block_size_main: Block size for main KV cache + kv_cache_extra: Quantized extra KV cache + indices_extra: Extra KV indices [total_tokens, topk_extra] + block_size_extra: Block size for extra KV cache + sm_scale: Softmax scale + topk_length_main: Optional per-batch topk length for main [b] + topk_length_extra: Optional per-batch topk length for extra [b] + attn_sink: Optional attention sink values [h_q] + s_q: Sequence length per batch + + Returns: + output: Attention output [total_tokens, h_q, d_v] + lse: Log-sum-exp values [total_tokens, h_q] + """ + total_tokens, h_q, d_qk = q.shape + topk_main = indices_main.shape[1] + topk_extra = indices_extra.shape[1] + total_topk = topk_main + topk_extra + d_v = DSV4_D_V + device = q.device + + # Prepare main KV cache + kv_uint8_main = kv_cache_main.view(torch.uint8) + num_blocks_main = kv_cache_main.shape[0] + stride_kv_block_main = kv_uint8_main.stride(0) + kv_flat_main = kv_uint8_main.reshape(num_blocks_main, -1) + + # Prepare extra KV cache + kv_uint8_extra = kv_cache_extra.view(torch.uint8) + num_blocks_extra = kv_cache_extra.shape[0] + stride_kv_block_extra = kv_uint8_extra.stride(0) + kv_flat_extra = kv_uint8_extra.reshape(num_blocks_extra, -1) + + if q.dtype != torch.bfloat16 or not q.is_contiguous(): + q = q.to(torch.bfloat16).contiguous() + + if not indices_main.is_contiguous(): + indices_main = indices_main.contiguous() + if not indices_extra.is_contiguous(): + indices_extra = indices_extra.contiguous() + + kv_cache_size_main = stride_kv_block_main * num_blocks_main + kv_cache_size_extra = stride_kv_block_extra * num_blocks_extra + disable_buffer_ops = ( + kv_cache_size_main > BUFFER_OPS_DISABLE_THRESHOLD + or kv_cache_size_extra > BUFFER_OPS_DISABLE_THRESHOLD + ) + + # Use Split-K for dual scope in these cases: + # 1. Small batch sizes with h_q=128 or large topk to increase GPU parallelism + # 2. Large topk (>= 2048) with medium/large batch sizes + # 3. NEW: h_q=64 + large topk (>=1024) + medium batch sizes (~21% improvement) + SPLITK_DUAL_SCOPE_TOPK_THRESHOLD = 2048 + # For small bs, only use splitk when h_q=128 or total_topk >= 1024 + use_splitk_for_small_bs = total_tokens <= 8 and (h_q >= 128 or total_topk >= 1024) + # NEW: For h_q=64 with large topk, splitk is beneficial for medium batch sizes + # Only for tokens <= 128 based on benchmarking (bs=64 shows 13% improvement) + use_splitk_for_h64_large_topk = ( + h_q <= 64 and total_topk >= 1024 and total_tokens > 8 and total_tokens <= 128 + ) + use_splitk_for_large_topk = ( + total_tokens > 64 and total_topk >= SPLITK_DUAL_SCOPE_TOPK_THRESHOLD + ) + # For h_q > 64 (e.g. h_q=128), the non-splitk grid has very few blocks + # in the H dimension, leading to low GPU utilization at medium batch sizes. + use_splitk_for_large_hq = h_q > 64 and total_tokens > 8 and total_topk >= 256 + if ( + use_splitk_for_small_bs + or use_splitk_for_h64_large_topk + or use_splitk_for_large_topk + or use_splitk_for_large_hq + ): + # Select split_k based on workload and total_topk. + # CUDA graph replay benchmarks show optimal split_k depends on both: + # - High topk (>=512, c4 layers): more splits needed to parallelize + # - Low topk (<512, c128 layers): fewer splits, less combine overhead + if total_tokens <= 8: + if total_topk >= 512 and total_tokens <= 4: + # High topk + very small bs: split_k=8 is 8-33% faster than sk=4 + split_k = 8 + else: + # split_k=4 gives 2x more blocks than split_k=2 + split_k = 4 + elif use_splitk_for_large_hq: + # For h_q > 64 with bs > 8: + if total_topk >= 512: + # High topk: split_k=4 for all medium/large bs + split_k = 4 + else: + # Low topk: split_k=2 is sufficient + split_k = 2 + elif use_splitk_for_h64_large_topk: + # For h_q=64 + large topk + medium bs, split_k=2 is optimal + split_k = 2 + else: + split_k = _select_split_k(total_topk, h_q, total_tokens) + topk_per_split = (total_topk + split_k - 1) // split_k + + partial_output = torch.empty( + split_k, total_tokens, h_q, d_v, dtype=torch.float32, device=device + ) + partial_lse = torch.empty( + split_k, total_tokens, h_q, dtype=torch.float32, device=device + ) + output = torch.empty( + total_tokens, h_q, d_v, dtype=torch.bfloat16, device=device + ) + lse = torch.empty(total_tokens, h_q, dtype=torch.float32, device=device) + + topk_length_main_tensor = ( + topk_length_main if topk_length_main is not None else lse[:1, 0] + ) + topk_length_extra_tensor = ( + topk_length_extra if topk_length_extra is not None else lse[:1, 0] + ) + attn_sink_tensor = attn_sink if attn_sink is not None else lse[0, :] + + grid_splitk = lambda meta: ( + triton.cdiv(h_q, meta["BLOCK_H"]), + total_tokens, + split_k, + ) + + def run_splitk_kernel(): + _fused_gather_attn_dsv4_dual_scope_splitk_kernel[grid_splitk]( + q, + kv_flat_main, + indices_main, + topk_length_main_tensor, + kv_flat_extra, + indices_extra, + topk_length_extra_tensor, + partial_output, + partial_lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + topk_main, + num_blocks_main, + block_size_main, + topk_extra, + num_blocks_extra, + block_size_extra, + s_q, + topk_per_split, + q.stride(0), + q.stride(1), + q.stride(2), + stride_kv_block_main, + stride_kv_block_extra, + indices_main.stride(0), + indices_main.stride(1), + indices_extra.stride(0), + indices_extra.stride(1), + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + HAS_TOPK_LENGTH_MAIN=topk_length_main is not None, + HAS_TOPK_LENGTH_EXTRA=topk_length_extra is not None, + ) + + if disable_buffer_ops: + with triton.knobs.amd.scope(): + triton.knobs.amd.use_buffer_ops = False + run_splitk_kernel() + else: + run_splitk_kernel() + + # Use appropriate combine kernel based on split_k + if split_k == 8: + grid_combine = lambda meta: ( + total_tokens, + triton.cdiv(h_q, meta["BLOCK_H"]), + ) + _combine_splitk_kernel_8_optimized[grid_combine]( + partial_output, + partial_lse, + attn_sink_tensor, + output, + lse, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + d_v, + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_ATTN_SINK=attn_sink is not None, + ) + else: + BLOCK_H_COMBINE = 16 + BLOCK_D_COMBINE = 128 + grid_combine = (total_tokens, triton.cdiv(h_q, BLOCK_H_COMBINE)) + + if split_k == 2: + combine_kernel = _combine_splitk_kernel_2 + elif split_k == 4: + combine_kernel = _combine_splitk_kernel + else: + raise ValueError(f"Unsupported split_k: {split_k}") + + combine_kernel[grid_combine]( + partial_output, + partial_lse, + attn_sink_tensor, + output, + lse, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + d_v, + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_ATTN_SINK=attn_sink is not None, + BLOCK_H=BLOCK_H_COMBINE, + BLOCK_D=BLOCK_D_COMBINE, + num_warps=4, + num_stages=1, + ) + + return output, lse + + # Use original kernel for smaller total_topk + output = torch.empty(total_tokens, h_q, d_v, dtype=torch.bfloat16, device=device) + lse = torch.empty(total_tokens, h_q, dtype=torch.float32, device=device) + + topk_length_main_tensor = ( + topk_length_main if topk_length_main is not None else lse[:1, 0] + ) + topk_length_extra_tensor = ( + topk_length_extra if topk_length_extra is not None else lse[:1, 0] + ) + attn_sink_tensor = attn_sink if attn_sink is not None else lse[0, :] + + grid = lambda meta: (triton.cdiv(h_q, meta["BLOCK_H"]), total_tokens) + + def run_kernel(): + _fused_gather_attn_dsv4_dual_scope_kernel[grid]( + q, + kv_flat_main, + indices_main, + topk_length_main_tensor, + kv_flat_extra, + indices_extra, + topk_length_extra_tensor, + attn_sink_tensor, + output, + lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + topk_main, + num_blocks_main, + block_size_main, + topk_extra, + num_blocks_extra, + block_size_extra, + s_q, + q.stride(0), + q.stride(1), + q.stride(2), + stride_kv_block_main, + stride_kv_block_extra, + indices_main.stride(0), + indices_main.stride(1), + indices_extra.stride(0), + indices_extra.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_TOPK_LENGTH_MAIN=topk_length_main is not None, + HAS_TOPK_LENGTH_EXTRA=topk_length_extra is not None, + HAS_ATTN_SINK=attn_sink is not None, + ) + + if disable_buffer_ops: + with triton.knobs.amd.scope(): + triton.knobs.amd.use_buffer_ops = False + run_kernel() + else: + run_kernel() + + return output, lse + + +# ============================================================================ +# Split-K Optimization for Large TopK (>= 8192) +# ============================================================================ +SPLITK_TOPK_THRESHOLD = 8192 +SPLITK_DEFAULT = 4 + + +@triton.autotune( + configs=[ + # Split-K fused kernel for large topk (≥8192). + # - BLOCK_N={16,32}: small blocks for scattered FP8 KV access pattern. + # - num_warps=4: balanced for fused dequant+attention compute. + # - BLOCK_H={16,64}: covers h_q=64 and h_q=128. + triton.Config({"BLOCK_H": 16, "BLOCK_N": 16}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 16, "BLOCK_N": 32}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 16}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_N": 32}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 16}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 128, "BLOCK_N": 32}, num_warps=4, num_stages=1), + ], + key=["total_tokens_bucket", "h_q", "topk_per_split"], +) +@triton.jit +def _fused_gather_attn_dsv4_splitk_kernel( + Q, + KV_Cache, + Indices, + TopkLength, + PartialOutput, + PartialLSE, + sm_scale, + total_tokens, + total_tokens_bucket, + h_q, + topk, + num_blocks, + block_size, + s_q, + topk_per_split, + stride_q_t, + stride_q_h, + stride_q_d, + stride_kv_block, + stride_idx_t, + stride_idx_k, + stride_po_s, + stride_po_t, + stride_po_h, + stride_po_d, + stride_plse_s, + stride_plse_t, + stride_plse_h, + HAS_TOPK_LENGTH: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """Split-K fused gather+dequant+attention kernel for DSV4.""" + LOG2E: tl.constexpr = 1.4426950408889634 + D_NOPE: tl.constexpr = 448 + TILE_SIZE: tl.constexpr = 64 + BYTES_PER_TOKEN_DATA: tl.constexpr = 576 + BYTES_PER_TOKEN_SCALE: tl.constexpr = 8 + + pid_h = tl.program_id(0) + pid_t = tl.program_id(1) + pid_k = tl.program_id(2) + pid_t_64 = pid_t.to(tl.int64) + + NEG_INF = float("-inf") + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + + k_start = pid_k * topk_per_split + k_end = tl.minimum(k_start + topk_per_split, topk) + + m_i = tl.full([BLOCK_H], NEG_INF, dtype=tl.float32) + l_i = tl.zeros([BLOCK_H], dtype=tl.float32) + + acc_0 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_1 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_2 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_3 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_4 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_5 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_6 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + acc_7 = tl.zeros([BLOCK_H, TILE_SIZE], dtype=tl.float32) + + stride_q_t_64 = tl.cast(stride_q_t, tl.int64) + q_base = Q + pid_t_64 * stride_q_t_64 + + batch_idx = pid_t // s_q + offs_tile = tl.arange(0, TILE_SIZE) + + q_row_base = q_base + offs_h[:, None] * stride_q_h + q_0 = tl.load( + q_row_base + offs_tile[None, :] * stride_q_d, mask=mask_h[:, None], other=0.0 + ).to(tl.bfloat16) + q_1 = tl.load( + q_row_base + (TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_2 = tl.load( + q_row_base + (2 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_3 = tl.load( + q_row_base + (3 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_4 = tl.load( + q_row_base + (4 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_5 = tl.load( + q_row_base + (5 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_6 = tl.load( + q_row_base + (6 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + q_7 = tl.load( + q_row_base + (7 * TILE_SIZE + offs_tile[None, :]) * stride_q_d, + mask=mask_h[:, None], + other=0.0, + ).to(tl.bfloat16) + + stride_kv_block_64 = tl.cast(stride_kv_block, tl.int64) + + # Early-exit: pre-load topk_len and skip invalid blocks + if HAS_TOPK_LENGTH: + topk_len = tl.load(TopkLength + batch_idx) + + for n_start in range(k_start, k_end, BLOCK_N): + # Skip entire block if beyond valid topk range + should_compute = not HAS_TOPK_LENGTH or n_start < topk_len + if should_compute: + offs_n = n_start + tl.arange(0, BLOCK_N) + mask_n = offs_n < k_end + + idx_ptrs = Indices + pid_t * stride_idx_t + offs_n * stride_idx_k + indices = tl.load(idx_ptrs, mask=mask_n, other=-1) + + is_invalid = indices == -1 + if HAS_TOPK_LENGTH: + is_invalid = is_invalid | (offs_n >= topk_len) + + valid = mask_n & ~is_invalid + indices_clamped = tl.maximum(indices, 0) + + block_idx = indices_clamped // block_size + offset_in_block = indices_clamped % block_size + + block_idx_64 = block_idx.to(tl.int64) + offset_in_block_64 = offset_in_block.to(tl.int64) + + kv_block_base = KV_Cache + block_idx_64 * stride_kv_block_64 + nope_rope_offset = offset_in_block_64 * BYTES_PER_TOKEN_DATA + scale_base_offset = ( + block_size * BYTES_PER_TOKEN_DATA + + offset_in_block_64 * BYTES_PER_TOKEN_SCALE + ) + + valid_2d = valid[:, None] + + # Use helper function for KV processing + acc_0, acc_1, acc_2, acc_3, acc_4, acc_5, acc_6, acc_7, m_i, l_i = ( + _process_kv_block_aggressive( + kv_block_base, + nope_rope_offset, + scale_base_offset, + valid, + valid_2d, + q_0, + q_1, + q_2, + q_3, + q_4, + q_5, + q_6, + q_7, + acc_0, + acc_1, + acc_2, + acc_3, + acc_4, + acc_5, + acc_6, + acc_7, + m_i, + l_i, + offs_tile, + sm_scale, + TILE_SIZE, + D_NOPE, + LOG2E, + BLOCK_H, + BLOCK_N, + ) + ) + + lse = m_i + tl.math.log2(tl.where(l_i == 0.0, 1.0, l_i)) / LOG2E + is_lonely_q = l_i == 0.0 + + output_scale = tl.where(l_i == 0.0, 0.0, 1.0 / l_i) + acc_0 = tl.where(is_lonely_q[:, None], 0.0, acc_0 * output_scale[:, None]) + acc_1 = tl.where(is_lonely_q[:, None], 0.0, acc_1 * output_scale[:, None]) + acc_2 = tl.where(is_lonely_q[:, None], 0.0, acc_2 * output_scale[:, None]) + acc_3 = tl.where(is_lonely_q[:, None], 0.0, acc_3 * output_scale[:, None]) + acc_4 = tl.where(is_lonely_q[:, None], 0.0, acc_4 * output_scale[:, None]) + acc_5 = tl.where(is_lonely_q[:, None], 0.0, acc_5 * output_scale[:, None]) + acc_6 = tl.where(is_lonely_q[:, None], 0.0, acc_6 * output_scale[:, None]) + acc_7 = tl.where(is_lonely_q[:, None], 0.0, acc_7 * output_scale[:, None]) + lse = tl.where(is_lonely_q, float("+inf"), lse) + + stride_po_s_64 = tl.cast(stride_po_s, tl.int64) + stride_po_t_64 = tl.cast(stride_po_t, tl.int64) + po_base = PartialOutput + pid_k * stride_po_s_64 + pid_t_64 * stride_po_t_64 + row_ptrs = po_base + offs_h[:, None] * stride_po_h + + # Store partial output as float32 for better precision in combine kernel + tl.store(row_ptrs + offs_tile[None, :] * stride_po_d, acc_0, mask=mask_h[:, None]) + tl.store( + row_ptrs + (TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_1, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (2 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_2, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (3 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_3, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (4 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_4, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (5 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_5, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (6 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_6, + mask=mask_h[:, None], + ) + tl.store( + row_ptrs + (7 * TILE_SIZE + offs_tile[None, :]) * stride_po_d, + acc_7, + mask=mask_h[:, None], + ) + + stride_plse_s_64 = tl.cast(stride_plse_s, tl.int64) + stride_plse_t_64 = tl.cast(stride_plse_t, tl.int64) + plse_ptrs = ( + PartialLSE + + pid_k * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h + ) + tl.store(plse_ptrs, lse, mask=mask_h) + + +@triton.jit +def _combine_splitk_kernel( + PartialOutput, + PartialLSE, + AttnSink, + Output, + LSE, + total_tokens, + total_tokens_bucket, + h_q, + d_v, + stride_po_s, + stride_po_t, + stride_po_h, + stride_po_d, + stride_plse_s, + stride_plse_t, + stride_plse_h, + stride_o_t, + stride_o_h, + stride_o_d, + stride_lse_t, + stride_lse_h, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Combine partial results from split-K kernel (SPLIT_K=4).""" + LOG2E: tl.constexpr = 1.4426950408889634 + NEG_INF = float("-inf") + POS_INF = float("+inf") + INF_THRESHOLD = 1e30 + + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + pid_t_64 = pid_t.to(tl.int64) + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + offs_d = tl.arange(0, BLOCK_D) + + stride_plse_s_64 = tl.cast(stride_plse_s, tl.int64) + stride_plse_t_64 = tl.cast(stride_plse_t, tl.int64) + + lse_0 = tl.load( + PartialLSE + + 0 * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h, + mask=mask_h, + other=POS_INF, + ) + lse_1 = tl.load( + PartialLSE + + 1 * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h, + mask=mask_h, + other=POS_INF, + ) + lse_2 = tl.load( + PartialLSE + + 2 * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h, + mask=mask_h, + other=POS_INF, + ) + lse_3 = tl.load( + PartialLSE + + 3 * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h, + mask=mask_h, + other=POS_INF, + ) + + lse_0_valid = tl.abs(lse_0) < INF_THRESHOLD + lse_1_valid = tl.abs(lse_1) < INF_THRESHOLD + lse_2_valid = tl.abs(lse_2) < INF_THRESHOLD + lse_3_valid = tl.abs(lse_3) < INF_THRESHOLD + + lse_0_safe = tl.where(lse_0_valid, lse_0, NEG_INF) + lse_1_safe = tl.where(lse_1_valid, lse_1, NEG_INF) + lse_2_safe = tl.where(lse_2_valid, lse_2, NEG_INF) + lse_3_safe = tl.where(lse_3_valid, lse_3, NEG_INF) + + max_lse = tl.maximum( + tl.maximum(lse_0_safe, lse_1_safe), tl.maximum(lse_2_safe, lse_3_safe) + ) + + exp_0 = tl.where(lse_0_valid, tl.math.exp2((lse_0_safe - max_lse) * LOG2E), 0.0) + exp_1 = tl.where(lse_1_valid, tl.math.exp2((lse_1_safe - max_lse) * LOG2E), 0.0) + exp_2 = tl.where(lse_2_valid, tl.math.exp2((lse_2_safe - max_lse) * LOG2E), 0.0) + exp_3 = tl.where(lse_3_valid, tl.math.exp2((lse_3_safe - max_lse) * LOG2E), 0.0) + + sum_exp = exp_0 + exp_1 + exp_2 + exp_3 + all_invalid = sum_exp == 0.0 + sum_exp_safe = tl.where(all_invalid, 1.0, sum_exp) + + combined_lse = max_lse + tl.math.log2(sum_exp_safe) / LOG2E + combined_lse = tl.where(all_invalid, POS_INF, combined_lse) + + if HAS_ATTN_SINK: + attn_sink_vals = tl.load(AttnSink + offs_h, mask=mask_h, other=0.0) + is_lonely = combined_lse > INF_THRESHOLD + lse_safe_for_sink = tl.where(is_lonely, 0.0, combined_lse) + diff = attn_sink_vals - lse_safe_for_sink + diff_clamped = tl.minimum(tl.maximum(diff, -100.0), 100.0) + exp_diff = tl.math.exp2(diff_clamped * LOG2E) + exp_diff = tl.where(is_lonely, 0.0, exp_diff) + denominator = 1.0 + exp_diff + sink_scale = 1.0 / denominator + sink_scale = tl.where(is_lonely, 1.0, sink_scale) + + scale_0 = (exp_0 / sum_exp_safe) * sink_scale + scale_1 = (exp_1 / sum_exp_safe) * sink_scale + scale_2 = (exp_2 / sum_exp_safe) * sink_scale + scale_3 = (exp_3 / sum_exp_safe) * sink_scale + else: + scale_0 = exp_0 / sum_exp_safe + scale_1 = exp_1 / sum_exp_safe + scale_2 = exp_2 / sum_exp_safe + scale_3 = exp_3 / sum_exp_safe + + scale_0 = tl.where(all_invalid, 0.0, scale_0) + scale_1 = tl.where(all_invalid, 0.0, scale_1) + scale_2 = tl.where(all_invalid, 0.0, scale_2) + scale_3 = tl.where(all_invalid, 0.0, scale_3) + + stride_po_s_64 = tl.cast(stride_po_s, tl.int64) + stride_po_t_64 = tl.cast(stride_po_t, tl.int64) + + po_base_0 = ( + PartialOutput + + 0 * stride_po_s_64 + + pid_t_64 * stride_po_t_64 + + offs_h[:, None] * stride_po_h + ) + po_base_1 = ( + PartialOutput + + 1 * stride_po_s_64 + + pid_t_64 * stride_po_t_64 + + offs_h[:, None] * stride_po_h + ) + po_base_2 = ( + PartialOutput + + 2 * stride_po_s_64 + + pid_t_64 * stride_po_t_64 + + offs_h[:, None] * stride_po_h + ) + po_base_3 = ( + PartialOutput + + 3 * stride_po_s_64 + + pid_t_64 * stride_po_t_64 + + offs_h[:, None] * stride_po_h + ) + + stride_o_t_64 = tl.cast(stride_o_t, tl.int64) + o_base = Output + pid_t_64 * stride_o_t_64 + offs_h[:, None] * stride_o_h + + for d_idx in range(4): + d_offs = d_idx * BLOCK_D + offs_d[None, :] + po_0 = tl.load( + po_base_0 + d_offs * stride_po_d, mask=mask_h[:, None], other=0.0 + ) + po_1 = tl.load( + po_base_1 + d_offs * stride_po_d, mask=mask_h[:, None], other=0.0 + ) + po_2 = tl.load( + po_base_2 + d_offs * stride_po_d, mask=mask_h[:, None], other=0.0 + ) + po_3 = tl.load( + po_base_3 + d_offs * stride_po_d, mask=mask_h[:, None], other=0.0 + ) + combined = ( + scale_0[:, None] * po_0 + + scale_1[:, None] * po_1 + + scale_2[:, None] * po_2 + + scale_3[:, None] * po_3 + ) + tl.store( + o_base + d_offs * stride_o_d, combined.to(tl.bfloat16), mask=mask_h[:, None] + ) + + stride_lse_t_64 = tl.cast(stride_lse_t, tl.int64) + lse_ptrs = LSE + pid_t_64 * stride_lse_t_64 + offs_h * stride_lse_h + tl.store(lse_ptrs, combined_lse, mask=mask_h) + + +@triton.autotune( + configs=[ + # Simple reduce kernel (weighted sum of 8 splits). + # - BLOCK_D=512: covers d_v=512 in one pass (no D-dimension loop). + # - num_warps=8: memory-bound reduce benefits from more warps. + # - split_k=8 is only used at very small batch sizes (≤4 tokens), + # so BLOCK_H=16/32/64 covers the relevant parallelism range. + triton.Config({"BLOCK_H": 16, "BLOCK_D": 512}, num_warps=8, num_stages=1), + triton.Config({"BLOCK_H": 32, "BLOCK_D": 512}, num_warps=8, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_D": 512}, num_warps=8, num_stages=1), + ], + key=["total_tokens_bucket", "h_q", "d_v"], +) +@triton.jit +def _combine_splitk_kernel_8_optimized( + PartialOutput, + PartialLSE, + AttnSink, + Output, + LSE, + total_tokens, + total_tokens_bucket, + h_q, + d_v, + stride_po_s, + stride_po_t, + stride_po_h, + stride_po_d, + stride_plse_s, + stride_plse_t, + stride_plse_h, + stride_o_t, + stride_o_h, + stride_o_d, + stride_lse_t, + stride_lse_h, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Optimized combine kernel for split-K=8 with autotuning for BLOCK_H.""" + LOG2E: tl.constexpr = 1.4426950408889634 + NEG_INF = float("-inf") + POS_INF = float("+inf") + INF_THRESHOLD = 1e30 + + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + pid_t_64 = pid_t.to(tl.int64) + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + offs_d = tl.arange(0, BLOCK_D) + + stride_plse_s_64 = tl.cast(stride_plse_s, tl.int64) + stride_plse_t_64 = tl.cast(stride_plse_t, tl.int64) + + # Load all 8 LSE values + lse_base = PartialLSE + pid_t_64 * stride_plse_t_64 + offs_h * stride_plse_h + lse_0 = tl.load(lse_base + 0 * stride_plse_s_64, mask=mask_h, other=POS_INF) + lse_1 = tl.load(lse_base + 1 * stride_plse_s_64, mask=mask_h, other=POS_INF) + lse_2 = tl.load(lse_base + 2 * stride_plse_s_64, mask=mask_h, other=POS_INF) + lse_3 = tl.load(lse_base + 3 * stride_plse_s_64, mask=mask_h, other=POS_INF) + lse_4 = tl.load(lse_base + 4 * stride_plse_s_64, mask=mask_h, other=POS_INF) + lse_5 = tl.load(lse_base + 5 * stride_plse_s_64, mask=mask_h, other=POS_INF) + lse_6 = tl.load(lse_base + 6 * stride_plse_s_64, mask=mask_h, other=POS_INF) + lse_7 = tl.load(lse_base + 7 * stride_plse_s_64, mask=mask_h, other=POS_INF) + + lse_0_valid = tl.abs(lse_0) < INF_THRESHOLD + lse_1_valid = tl.abs(lse_1) < INF_THRESHOLD + lse_2_valid = tl.abs(lse_2) < INF_THRESHOLD + lse_3_valid = tl.abs(lse_3) < INF_THRESHOLD + lse_4_valid = tl.abs(lse_4) < INF_THRESHOLD + lse_5_valid = tl.abs(lse_5) < INF_THRESHOLD + lse_6_valid = tl.abs(lse_6) < INF_THRESHOLD + lse_7_valid = tl.abs(lse_7) < INF_THRESHOLD + + lse_0_safe = tl.where(lse_0_valid, lse_0, NEG_INF) + lse_1_safe = tl.where(lse_1_valid, lse_1, NEG_INF) + lse_2_safe = tl.where(lse_2_valid, lse_2, NEG_INF) + lse_3_safe = tl.where(lse_3_valid, lse_3, NEG_INF) + lse_4_safe = tl.where(lse_4_valid, lse_4, NEG_INF) + lse_5_safe = tl.where(lse_5_valid, lse_5, NEG_INF) + lse_6_safe = tl.where(lse_6_valid, lse_6, NEG_INF) + lse_7_safe = tl.where(lse_7_valid, lse_7, NEG_INF) + + max_lse = tl.maximum( + tl.maximum( + tl.maximum(lse_0_safe, lse_1_safe), tl.maximum(lse_2_safe, lse_3_safe) + ), + tl.maximum( + tl.maximum(lse_4_safe, lse_5_safe), tl.maximum(lse_6_safe, lse_7_safe) + ), + ) + + exp_0 = tl.where(lse_0_valid, tl.math.exp2((lse_0_safe - max_lse) * LOG2E), 0.0) + exp_1 = tl.where(lse_1_valid, tl.math.exp2((lse_1_safe - max_lse) * LOG2E), 0.0) + exp_2 = tl.where(lse_2_valid, tl.math.exp2((lse_2_safe - max_lse) * LOG2E), 0.0) + exp_3 = tl.where(lse_3_valid, tl.math.exp2((lse_3_safe - max_lse) * LOG2E), 0.0) + exp_4 = tl.where(lse_4_valid, tl.math.exp2((lse_4_safe - max_lse) * LOG2E), 0.0) + exp_5 = tl.where(lse_5_valid, tl.math.exp2((lse_5_safe - max_lse) * LOG2E), 0.0) + exp_6 = tl.where(lse_6_valid, tl.math.exp2((lse_6_safe - max_lse) * LOG2E), 0.0) + exp_7 = tl.where(lse_7_valid, tl.math.exp2((lse_7_safe - max_lse) * LOG2E), 0.0) + + sum_exp = exp_0 + exp_1 + exp_2 + exp_3 + exp_4 + exp_5 + exp_6 + exp_7 + all_invalid = sum_exp == 0.0 + sum_exp_safe = tl.where(all_invalid, 1.0, sum_exp) + + combined_lse = max_lse + tl.math.log2(sum_exp_safe) / LOG2E + combined_lse = tl.where(all_invalid, POS_INF, combined_lse) + + if HAS_ATTN_SINK: + attn_sink_vals = tl.load(AttnSink + offs_h, mask=mask_h, other=0.0) + is_lonely = combined_lse > INF_THRESHOLD + lse_safe_for_sink = tl.where(is_lonely, 0.0, combined_lse) + diff = attn_sink_vals - lse_safe_for_sink + diff_clamped = tl.minimum(tl.maximum(diff, -100.0), 100.0) + exp_diff = tl.math.exp2(diff_clamped * LOG2E) + exp_diff = tl.where(is_lonely, 0.0, exp_diff) + denominator = 1.0 + exp_diff + sink_scale = 1.0 / denominator + sink_scale = tl.where(is_lonely, 1.0, sink_scale) + + scale_0 = (exp_0 / sum_exp_safe) * sink_scale + scale_1 = (exp_1 / sum_exp_safe) * sink_scale + scale_2 = (exp_2 / sum_exp_safe) * sink_scale + scale_3 = (exp_3 / sum_exp_safe) * sink_scale + scale_4 = (exp_4 / sum_exp_safe) * sink_scale + scale_5 = (exp_5 / sum_exp_safe) * sink_scale + scale_6 = (exp_6 / sum_exp_safe) * sink_scale + scale_7 = (exp_7 / sum_exp_safe) * sink_scale + else: + scale_0 = exp_0 / sum_exp_safe + scale_1 = exp_1 / sum_exp_safe + scale_2 = exp_2 / sum_exp_safe + scale_3 = exp_3 / sum_exp_safe + scale_4 = exp_4 / sum_exp_safe + scale_5 = exp_5 / sum_exp_safe + scale_6 = exp_6 / sum_exp_safe + scale_7 = exp_7 / sum_exp_safe + + scale_0 = tl.where(all_invalid, 0.0, scale_0) + scale_1 = tl.where(all_invalid, 0.0, scale_1) + scale_2 = tl.where(all_invalid, 0.0, scale_2) + scale_3 = tl.where(all_invalid, 0.0, scale_3) + scale_4 = tl.where(all_invalid, 0.0, scale_4) + scale_5 = tl.where(all_invalid, 0.0, scale_5) + scale_6 = tl.where(all_invalid, 0.0, scale_6) + scale_7 = tl.where(all_invalid, 0.0, scale_7) + + stride_po_s_64 = tl.cast(stride_po_s, tl.int64) + stride_po_t_64 = tl.cast(stride_po_t, tl.int64) + + po_base = PartialOutput + pid_t_64 * stride_po_t_64 + offs_h[:, None] * stride_po_h + po_base_0 = po_base + 0 * stride_po_s_64 + po_base_1 = po_base + 1 * stride_po_s_64 + po_base_2 = po_base + 2 * stride_po_s_64 + po_base_3 = po_base + 3 * stride_po_s_64 + po_base_4 = po_base + 4 * stride_po_s_64 + po_base_5 = po_base + 5 * stride_po_s_64 + po_base_6 = po_base + 6 * stride_po_s_64 + po_base_7 = po_base + 7 * stride_po_s_64 + + stride_o_t_64 = tl.cast(stride_o_t, tl.int64) + o_base = Output + pid_t_64 * stride_o_t_64 + offs_h[:, None] * stride_o_h + + # Loop over D dimension with BLOCK_D chunks + num_d_iters: tl.constexpr = (512 + BLOCK_D - 1) // BLOCK_D + for d_idx in tl.static_range(num_d_iters): + d_offs = d_idx * BLOCK_D + offs_d[None, :] + mask_d = d_offs < d_v + mask_hd = mask_h[:, None] & mask_d + + po_0 = tl.load(po_base_0 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + po_1 = tl.load(po_base_1 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + po_2 = tl.load(po_base_2 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + po_3 = tl.load(po_base_3 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + po_4 = tl.load(po_base_4 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + po_5 = tl.load(po_base_5 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + po_6 = tl.load(po_base_6 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + po_7 = tl.load(po_base_7 + d_offs * stride_po_d, mask=mask_hd, other=0.0) + + combined = ( + scale_0[:, None] * po_0 + + scale_1[:, None] * po_1 + + scale_2[:, None] * po_2 + + scale_3[:, None] * po_3 + + scale_4[:, None] * po_4 + + scale_5[:, None] * po_5 + + scale_6[:, None] * po_6 + + scale_7[:, None] * po_7 + ) + tl.store(o_base + d_offs * stride_o_d, combined.to(tl.bfloat16), mask=mask_hd) + + stride_lse_t_64 = tl.cast(stride_lse_t, tl.int64) + lse_ptrs = LSE + pid_t_64 * stride_lse_t_64 + offs_h * stride_lse_h + tl.store(lse_ptrs, combined_lse, mask=mask_h) + + +@triton.jit +def _combine_splitk_kernel_2( + PartialOutput, + PartialLSE, + AttnSink, + Output, + LSE, + total_tokens, + total_tokens_bucket, + h_q, + d_v, + stride_po_s, + stride_po_t, + stride_po_h, + stride_po_d, + stride_plse_s, + stride_plse_t, + stride_plse_h, + stride_o_t, + stride_o_h, + stride_o_d, + stride_lse_t, + stride_lse_h, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Combine partial results from split-K kernel (SPLIT_K=2).""" + LOG2E: tl.constexpr = 1.4426950408889634 + NEG_INF = float("-inf") + POS_INF = float("+inf") + INF_THRESHOLD = 1e30 + + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + pid_t_64 = pid_t.to(tl.int64) + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + offs_d = tl.arange(0, BLOCK_D) + + stride_plse_s_64 = tl.cast(stride_plse_s, tl.int64) + stride_plse_t_64 = tl.cast(stride_plse_t, tl.int64) + + lse_0 = tl.load( + PartialLSE + + 0 * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h, + mask=mask_h, + other=POS_INF, + ) + lse_1 = tl.load( + PartialLSE + + 1 * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h, + mask=mask_h, + other=POS_INF, + ) + + lse_0_valid = tl.abs(lse_0) < INF_THRESHOLD + lse_1_valid = tl.abs(lse_1) < INF_THRESHOLD + + lse_0_safe = tl.where(lse_0_valid, lse_0, NEG_INF) + lse_1_safe = tl.where(lse_1_valid, lse_1, NEG_INF) + + max_lse = tl.maximum(lse_0_safe, lse_1_safe) + + exp_0 = tl.where(lse_0_valid, tl.math.exp2((lse_0_safe - max_lse) * LOG2E), 0.0) + exp_1 = tl.where(lse_1_valid, tl.math.exp2((lse_1_safe - max_lse) * LOG2E), 0.0) + + sum_exp = exp_0 + exp_1 + all_invalid = sum_exp == 0.0 + sum_exp_safe = tl.where(all_invalid, 1.0, sum_exp) + + combined_lse = max_lse + tl.math.log2(sum_exp_safe) / LOG2E + combined_lse = tl.where(all_invalid, POS_INF, combined_lse) + + if HAS_ATTN_SINK: + attn_sink_vals = tl.load(AttnSink + offs_h, mask=mask_h, other=0.0) + is_lonely = combined_lse > INF_THRESHOLD + lse_safe_for_sink = tl.where(is_lonely, 0.0, combined_lse) + diff = attn_sink_vals - lse_safe_for_sink + diff_clamped = tl.minimum(tl.maximum(diff, -100.0), 100.0) + exp_diff = tl.math.exp2(diff_clamped * LOG2E) + exp_diff = tl.where(is_lonely, 0.0, exp_diff) + denominator = 1.0 + exp_diff + sink_scale = 1.0 / denominator + sink_scale = tl.where(is_lonely, 1.0, sink_scale) + + scale_0 = (exp_0 / sum_exp_safe) * sink_scale + scale_1 = (exp_1 / sum_exp_safe) * sink_scale + else: + scale_0 = exp_0 / sum_exp_safe + scale_1 = exp_1 / sum_exp_safe + + scale_0 = tl.where(all_invalid, 0.0, scale_0) + scale_1 = tl.where(all_invalid, 0.0, scale_1) + + stride_po_s_64 = tl.cast(stride_po_s, tl.int64) + stride_po_t_64 = tl.cast(stride_po_t, tl.int64) + + po_base_0 = ( + PartialOutput + + 0 * stride_po_s_64 + + pid_t_64 * stride_po_t_64 + + offs_h[:, None] * stride_po_h + ) + po_base_1 = ( + PartialOutput + + 1 * stride_po_s_64 + + pid_t_64 * stride_po_t_64 + + offs_h[:, None] * stride_po_h + ) + + stride_o_t_64 = tl.cast(stride_o_t, tl.int64) + o_base = Output + pid_t_64 * stride_o_t_64 + offs_h[:, None] * stride_o_h + + for d_idx in range(4): + d_offs = d_idx * BLOCK_D + offs_d[None, :] + po_0 = tl.load( + po_base_0 + d_offs * stride_po_d, mask=mask_h[:, None], other=0.0 + ) + po_1 = tl.load( + po_base_1 + d_offs * stride_po_d, mask=mask_h[:, None], other=0.0 + ) + combined = scale_0[:, None] * po_0 + scale_1[:, None] * po_1 + tl.store( + o_base + d_offs * stride_o_d, combined.to(tl.bfloat16), mask=mask_h[:, None] + ) + + stride_lse_t_64 = tl.cast(stride_lse_t, tl.int64) + lse_ptrs = LSE + pid_t_64 * stride_lse_t_64 + offs_h * stride_lse_h + tl.store(lse_ptrs, combined_lse, mask=mask_h) + + +def _select_split_k(topk: int, h_q: int, total_tokens: int = 64) -> int: + """Select optimal split_k based on topk, h_q, and total_tokens. + + The split_k parameter controls how many parallel splits are used to process + the topk dimension. Larger split_k increases parallelism but also increases + the overhead of the combine kernel. + + Updated heuristics based on benchmarking with optimized BLOCK_N configs: + - For large topk (>= 16384): split_k=4 provides good balance with existing combine kernel + - For medium topk (8192-16383): split_k=4 + - For small topk (< 8192): split_k=2 + """ + if topk >= 8192: + return 4 + else: + return 2 + + +# ============================================================================ +# Low-overhead buffer pool for splitk operations +# ============================================================================ +class SplitKBufferPool: + """ + Pre-allocated buffer pool for split-K intermediate tensors. + + Caches partial_output and partial_lse buffers to avoid repeated allocations. + Output buffers are always freshly allocated to ensure correctness. + """ + + _buffers = {} + _device = None + + @classmethod + def get_buffers( + cls, split_k: int, total_tokens: int, h_q: int, d_v: int, device: torch.device + ): + """Get or create intermediate buffers for the given configuration.""" + key = (split_k, total_tokens, h_q, d_v, device) + + if key not in cls._buffers or cls._device != device: + cls._device = device + partial_output = torch.empty( + split_k, total_tokens, h_q, d_v, dtype=torch.float32, device=device + ) + partial_lse = torch.empty( + split_k, total_tokens, h_q, dtype=torch.float32, device=device + ) + + cls._buffers[key] = { + "partial_output": partial_output, + "partial_lse": partial_lse, + "stride_po": partial_output.stride(), + "stride_plse": partial_lse.stride(), + } + + return cls._buffers[key] + + @classmethod + def clear(cls): + """Clear all cached buffers.""" + cls._buffers.clear() + cls._device = None + + +def fused_gather_attn_decode_dsv4_dual_scope_low_overhead( + q: torch.Tensor, + kv_cache_main: torch.Tensor, + indices_main: torch.Tensor, + block_size_main: int, + kv_cache_extra: torch.Tensor, + indices_extra: torch.Tensor, + block_size_extra: int, + sm_scale: float, + topk_length_main: Optional[torch.Tensor] = None, + topk_length_extra: Optional[torch.Tensor] = None, + attn_sink: Optional[torch.Tensor] = None, + s_q: int = 1, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Low-overhead version of fused_gather_attn_decode_dsv4_dual_scope. + + This version uses pre-allocated intermediate buffers and cached strides + to minimize Python overhead, which is significant for small batch sizes. + + The kernel computation is identical to the original version. + Output buffers are always freshly allocated to ensure correctness. + """ + total_tokens, h_q, d_qk = q.shape + topk_main = indices_main.shape[1] + topk_extra = indices_extra.shape[1] + total_topk = topk_main + topk_extra + d_v = DSV4_D_V + device = q.device + + # Prepare main KV cache + kv_uint8_main = kv_cache_main.view(torch.uint8) + num_blocks_main = kv_cache_main.shape[0] + stride_kv_block_main = kv_uint8_main.stride(0) + kv_flat_main = kv_uint8_main.reshape(num_blocks_main, -1) + + # Prepare extra KV cache + kv_uint8_extra = kv_cache_extra.view(torch.uint8) + num_blocks_extra = kv_cache_extra.shape[0] + stride_kv_block_extra = kv_uint8_extra.stride(0) + kv_flat_extra = kv_uint8_extra.reshape(num_blocks_extra, -1) + + if q.dtype != torch.bfloat16 or not q.is_contiguous(): + q = q.to(torch.bfloat16).contiguous() + + if not indices_main.is_contiguous(): + indices_main = indices_main.contiguous() + if not indices_extra.is_contiguous(): + indices_extra = indices_extra.contiguous() + + # Determine split_k + SPLITK_DUAL_SCOPE_TOPK_THRESHOLD = 2048 + use_splitk_for_small_bs = total_tokens <= 8 and (h_q >= 128 or total_topk >= 1024) + use_splitk_for_h64_large_topk = ( + h_q <= 64 and total_topk >= 1024 and total_tokens > 8 and total_tokens <= 128 + ) + use_splitk_for_large_topk = ( + total_tokens > 64 and total_topk >= SPLITK_DUAL_SCOPE_TOPK_THRESHOLD + ) + # For h_q > 64 (e.g. h_q=128), the non-splitk grid has very few blocks + # in the H dimension (cdiv(128,64)=2), leading to low GPU utilization + # at medium batch sizes. Split-K doubles the parallelism. + use_splitk_for_large_hq = h_q > 64 and total_tokens > 8 and total_topk >= 256 + + if not ( + use_splitk_for_small_bs + or use_splitk_for_h64_large_topk + or use_splitk_for_large_topk + or use_splitk_for_large_hq + ): + # Fall back to non-splitk version + return fused_gather_attn_decode_dsv4_dual_scope( + q, + kv_cache_main, + indices_main, + block_size_main, + kv_cache_extra, + indices_extra, + block_size_extra, + sm_scale, + topk_length_main, + topk_length_extra, + attn_sink, + s_q, + ) + + # Select split_k based on workload and total_topk. + # CUDA graph replay benchmarks show optimal split_k depends on both: + # - High topk (>=512, c4 layers): more splits needed to parallelize + # - Low topk (<512, c128 layers): fewer splits, less combine overhead + if total_tokens <= 8: + if total_topk >= 512 and total_tokens <= 4: + # High topk + very small bs: split_k=8 is 8-33% faster than sk=4 + split_k = 8 + else: + # split_k=4 gives 2x more blocks than split_k=2 + split_k = 4 + elif use_splitk_for_large_hq: + # For h_q > 64 with bs > 8: + if total_topk >= 512: + # High topk: split_k=4 for all medium/large bs + split_k = 4 + else: + # Low topk: split_k=2 is sufficient + split_k = 2 + elif use_splitk_for_h64_large_topk: + split_k = 2 + else: + split_k = _select_split_k(total_topk, h_q, total_tokens) + + topk_per_split = (total_topk + split_k - 1) // split_k + + # Get pre-allocated intermediate buffers + buffers = SplitKBufferPool.get_buffers(split_k, total_tokens, h_q, d_v, device) + partial_output = buffers["partial_output"] + partial_lse = buffers["partial_lse"] + stride_po = buffers["stride_po"] + stride_plse = buffers["stride_plse"] + + # Reuse pre-allocated output buffers to avoid torch.empty() calls + # that would be captured in CUDA graphs (each adds ~7-8us replay overhead). + output = torch.empty(total_tokens, h_q, d_v, dtype=torch.bfloat16, device=device) + lse = torch.empty(total_tokens, h_q, dtype=torch.float32, device=device) + + # Prepare dummy tensors for optional parameters + topk_length_main_tensor = ( + topk_length_main if topk_length_main is not None else lse[:1, 0] + ) + topk_length_extra_tensor = ( + topk_length_extra if topk_length_extra is not None else lse[:1, 0] + ) + attn_sink_tensor = attn_sink if attn_sink is not None else lse[0, :] + + # Pre-compute strides + stride_q = q.stride() + stride_o = output.stride() + stride_lse = lse.stride() + + # Check if buffer ops should be disabled + kv_cache_size_main = stride_kv_block_main * num_blocks_main + kv_cache_size_extra = stride_kv_block_extra * num_blocks_extra + disable_buffer_ops = ( + kv_cache_size_main > BUFFER_OPS_DISABLE_THRESHOLD + or kv_cache_size_extra > BUFFER_OPS_DISABLE_THRESHOLD + ) + + # Grid for splitk kernel + grid_splitk = lambda meta: ( + triton.cdiv(h_q, meta["BLOCK_H"]), + total_tokens, + split_k, + ) + + # Run splitk kernel + if disable_buffer_ops: + with triton.knobs.amd.scope(): + triton.knobs.amd.use_buffer_ops = False + _fused_gather_attn_dsv4_dual_scope_splitk_kernel[grid_splitk]( + q, + kv_flat_main, + indices_main, + topk_length_main_tensor, + kv_flat_extra, + indices_extra, + topk_length_extra_tensor, + partial_output, + partial_lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + topk_main, + num_blocks_main, + block_size_main, + topk_extra, + num_blocks_extra, + block_size_extra, + s_q, + topk_per_split, + stride_q[0], + stride_q[1], + stride_q[2], + stride_kv_block_main, + stride_kv_block_extra, + indices_main.stride(0), + indices_main.stride(1), + indices_extra.stride(0), + indices_extra.stride(1), + stride_po[0], + stride_po[1], + stride_po[2], + stride_po[3], + stride_plse[0], + stride_plse[1], + stride_plse[2], + HAS_TOPK_LENGTH_MAIN=topk_length_main is not None, + HAS_TOPK_LENGTH_EXTRA=topk_length_extra is not None, + ) + else: + _fused_gather_attn_dsv4_dual_scope_splitk_kernel[grid_splitk]( + q, + kv_flat_main, + indices_main, + topk_length_main_tensor, + kv_flat_extra, + indices_extra, + topk_length_extra_tensor, + partial_output, + partial_lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + topk_main, + num_blocks_main, + block_size_main, + topk_extra, + num_blocks_extra, + block_size_extra, + s_q, + topk_per_split, + stride_q[0], + stride_q[1], + stride_q[2], + stride_kv_block_main, + stride_kv_block_extra, + indices_main.stride(0), + indices_main.stride(1), + indices_extra.stride(0), + indices_extra.stride(1), + stride_po[0], + stride_po[1], + stride_po[2], + stride_po[3], + stride_plse[0], + stride_plse[1], + stride_plse[2], + HAS_TOPK_LENGTH_MAIN=topk_length_main is not None, + HAS_TOPK_LENGTH_EXTRA=topk_length_extra is not None, + ) + + # Run combine kernel + if split_k == 8: + grid_combine = lambda meta: (total_tokens, triton.cdiv(h_q, meta["BLOCK_H"])) + _combine_splitk_kernel_8_optimized[grid_combine]( + partial_output, + partial_lse, + attn_sink_tensor, + output, + lse, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + d_v, + stride_po[0], + stride_po[1], + stride_po[2], + stride_po[3], + stride_plse[0], + stride_plse[1], + stride_plse[2], + stride_o[0], + stride_o[1], + stride_o[2], + stride_lse[0], + stride_lse[1], + HAS_ATTN_SINK=attn_sink is not None, + ) + else: + BLOCK_H_COMBINE = 16 + BLOCK_D_COMBINE = 128 + grid_combine = (total_tokens, triton.cdiv(h_q, BLOCK_H_COMBINE)) + + if split_k == 2: + combine_kernel = _combine_splitk_kernel_2 + elif split_k == 4: + combine_kernel = _combine_splitk_kernel + else: + raise ValueError(f"Unsupported split_k: {split_k}") + + combine_kernel[grid_combine]( + partial_output, + partial_lse, + attn_sink_tensor, + output, + lse, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + d_v, + stride_po[0], + stride_po[1], + stride_po[2], + stride_po[3], + stride_plse[0], + stride_plse[1], + stride_plse[2], + stride_o[0], + stride_o[1], + stride_o[2], + stride_lse[0], + stride_lse[1], + HAS_ATTN_SINK=attn_sink is not None, + BLOCK_H=BLOCK_H_COMBINE, + BLOCK_D=BLOCK_D_COMBINE, + num_warps=4, + num_stages=1, + ) + + return output, lse diff --git a/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_optimized.py b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_optimized.py new file mode 100644 index 000000000000..02891b91b21c --- /dev/null +++ b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_optimized.py @@ -0,0 +1,289 @@ +""" +Optimized Triton MLA Decode Kernels for DeepSeek V4. + +This module provides optimized sparse attention decode with reduced Python overhead. + +Key optimizations: +1. Fused gather+dequant+attention kernels (eliminates intermediate buffers) +2. Split-K for better GPU parallelism on small batches +3. Pre-allocated buffer pool for splitk intermediate results +4. Pre-computed strides to reduce tensor metadata operations + +Note: This implementation assumes KV cache is always FP8 quantized. +""" + +from typing import Optional, Tuple + +import torch +import triton + +from .triton_mla_kernels_decode_common import ( + _bucket_total_tokens, + _unified_sparse_decode_kernel, + compute_token_ranges, +) +from .triton_mla_kernels_decode_dsv4 import ( + DSV4_D_QK, + fused_gather_dequant_fp8_dsv4, +) +from .triton_mla_kernels_decode_fused import ( + fused_gather_attn_decode_dsv4, + fused_gather_attn_decode_dsv4_dual_scope_low_overhead, +) + + +def triton_sparse_attn_decode( + q: torch.Tensor, + kv_scope, + extra_kv_scope, + sm_scale: float, + d_v: int = 512, + attn_sink: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Optimized sparse attention decode for DeepSeek V4 (d_qk=512).""" + d_qk = q.shape[-1] + + if d_qk != DSV4_D_QK: + raise ValueError( + f"Unsupported d_qk: {d_qk}. Expected {DSV4_D_QK} (DeepSeek V4)" + ) + + return _triton_sparse_attn_decode_dsv4( + q, kv_scope, extra_kv_scope, sm_scale, d_v, attn_sink + ) + + +def _should_use_fused_dual_scope(total_tokens: int, h_q: int, total_topk: int) -> bool: + """Determine whether to use fused kernel for dual-scope cases. + + The fused kernel avoids allocating a large intermediate gathered_kv + buffer and eliminates a separate gather kernel launch. However, for + h_q > 64 with medium-to-large batch sizes and larger topk, the + non-splitk fused kernel suffers from low GPU utilization (the grid + has only cdiv(h_q, BLOCK_H) blocks in the H dimension). In those + cases the fallback (separate gather + attention) can be faster on + the GPU, though it incurs extra torch.empty() overhead in CUDA + graphs. + + The thresholds below were determined empirically on MI355X (256 CUs). + """ + if total_tokens <= 4: + return True + if h_q <= 64 and total_topk <= 800: + return total_tokens <= 256 + if h_q <= 64 and total_topk >= 1024: + return total_tokens <= 128 + # h_q > 64 (e.g. h_q=128 when q is padded to full n_heads). + # For small topk (c128 layers, topk~192), fused always wins. + # For larger topk (c4 layers, topk~640), fused wins at small bs + # but the fallback catches up at bs>=16 due to better GPU utilization. + # However, the fallback has 4 extra torch.empty() calls that add + # ~30us CUDA-graph replay overhead, roughly cancelling the GPU gain. + # So we route to fused for all practical batch sizes. + if h_q > 64: + return total_tokens <= 256 + return True + + +def _triton_sparse_attn_decode_dsv4( + q: torch.Tensor, + kv_scope, + extra_kv_scope, + sm_scale: float, + d_v: int, + attn_sink: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + """Optimized sparse attention decode for DeepSeek V4 (d_qk=512).""" + b, s_q, h_q, d_qk = q.shape + total_tokens = b * s_q + device = q.device + + topk_main = kv_scope.indices_in_kvcache.shape[-1] + kv_quantized_main = kv_scope.blocked_k_quantized + block_size_main = kv_scope.blocked_k.shape[1] + + # Single scope case + if extra_kv_scope is None: + if topk_main < 8192: + q_reshaped = q.reshape(total_tokens, h_q, d_qk) + if not q_reshaped.is_contiguous(): + q_reshaped = q_reshaped.contiguous() + + indices_main = kv_scope.indices_in_kvcache.reshape(total_tokens, topk_main) + if not indices_main.is_contiguous(): + indices_main = indices_main.contiguous() + + output, lse = fused_gather_attn_decode_dsv4( + q_reshaped, + kv_quantized_main, + indices_main, + block_size_main, + sm_scale, + topk_length=kv_scope.topk_length, + attn_sink=attn_sink, + s_q=s_q, + ) + return output.view(b, s_q, h_q, d_v), lse.view(b, s_q, h_q).transpose(1, 2) + else: + from .triton_mla_kernels_decode_dsv4 import triton_sparse_attn_decode_dsv4 + + return triton_sparse_attn_decode_dsv4( + q, kv_scope, extra_kv_scope, sm_scale, d_v, attn_sink + ) + + # Dual scope case + topk_extra = extra_kv_scope.indices_in_kvcache.shape[-1] + total_topk = topk_main + topk_extra + + # Check if chunking needed (fall back to original implementation) + token_ranges = compute_token_ranges(total_tokens, total_topk, d_qk) + if len(token_ranges) > 1: + from .triton_mla_kernels_decode_dsv4 import triton_sparse_attn_decode_dsv4 + + return triton_sparse_attn_decode_dsv4( + q, kv_scope, extra_kv_scope, sm_scale, d_v, attn_sink + ) + + # Use fused dual-scope kernel with low-overhead buffer pool + if _should_use_fused_dual_scope(total_tokens, h_q, total_topk): + q_reshaped = q.reshape(total_tokens, h_q, d_qk) + if not q_reshaped.is_contiguous(): + q_reshaped = q_reshaped.contiguous() + + indices_main = kv_scope.indices_in_kvcache.reshape(total_tokens, topk_main) + if not indices_main.is_contiguous(): + indices_main = indices_main.contiguous() + + block_size_extra = extra_kv_scope.blocked_k.shape[1] + indices_extra = extra_kv_scope.indices_in_kvcache.reshape( + total_tokens, topk_extra + ) + if not indices_extra.is_contiguous(): + indices_extra = indices_extra.contiguous() + + output, lse = fused_gather_attn_decode_dsv4_dual_scope_low_overhead( + q_reshaped, + kv_quantized_main, + indices_main, + block_size_main, + extra_kv_scope.blocked_k_quantized, + indices_extra, + block_size_extra, + sm_scale, + topk_length_main=kv_scope.topk_length, + topk_length_extra=extra_kv_scope.topk_length, + attn_sink=attn_sink, + s_q=s_q, + ) + return output.view(b, s_q, h_q, d_v), lse.view(b, s_q, h_q).transpose(1, 2) + + # Fallback: Separate gather + attention path + return _fallback_gather_attention( + q, + kv_scope, + extra_kv_scope, + sm_scale, + d_v, + attn_sink, + total_tokens, + h_q, + d_qk, + topk_main, + topk_extra, + block_size_main, + kv_quantized_main, + fused_gather_dequant_fp8_dsv4, + ) + + +def _fallback_gather_attention( + q: torch.Tensor, + kv_scope, + extra_kv_scope, + sm_scale: float, + d_v: int, + attn_sink: Optional[torch.Tensor], + total_tokens: int, + h_q: int, + d_qk: int, + topk_main: int, + topk_extra: int, + block_size_main: int, + kv_quantized_main, + fused_gather_fn, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Fallback path: separate gather + attention kernels.""" + b = q.shape[0] + s_q = q.shape[1] + device = q.device + total_topk = topk_main + topk_extra + + gathered_kv = torch.empty( + total_tokens, total_topk, d_qk, dtype=torch.bfloat16, device=device + ) + invalid_mask = torch.empty( + total_tokens, total_topk, dtype=torch.bool, device=device + ) + output = torch.empty(total_tokens, h_q, d_v, dtype=torch.bfloat16, device=device) + lse = torch.empty(total_tokens, h_q, dtype=torch.float32, device=device) + + indices_main = kv_scope.indices_in_kvcache.reshape(total_tokens, topk_main) + block_size_extra = extra_kv_scope.blocked_k.shape[1] + indices_extra = extra_kv_scope.indices_in_kvcache.reshape(total_tokens, topk_extra) + + fused_gather_fn( + kv_quantized_main, + indices_main, + block_size_main, + kv_scope.topk_length, + extra_kv_scope.blocked_k_quantized, + indices_extra, + block_size_extra, + extra_kv_scope.topk_length, + gathered_kv, + invalid_mask, + s_q, + ) + + if q.dtype == torch.bfloat16 and q.is_contiguous(): + q_reshaped = q.view(total_tokens, h_q, d_qk) + else: + q_reshaped = q.to(torch.bfloat16).reshape(total_tokens, h_q, d_qk) + if not q_reshaped.is_contiguous(): + q_reshaped = q_reshaped.contiguous() + + HAS_ATTN_SINK = attn_sink is not None + attn_sink_tensor = attn_sink if HAS_ATTN_SINK else lse[:1] + + grid = lambda meta: (total_tokens, triton.cdiv(h_q, meta["BLOCK_H"])) + _unified_sparse_decode_kernel[grid]( + q_reshaped, + gathered_kv, + invalid_mask, + attn_sink_tensor, + output, + lse, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + total_topk, + d_qk, + d_v, + q_reshaped.stride(0), + q_reshaped.stride(1), + q_reshaped.stride(2), + gathered_kv.stride(0), + gathered_kv.stride(1), + gathered_kv.stride(2), + invalid_mask.stride(0), + invalid_mask.stride(1), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_ATTN_SINK=HAS_ATTN_SINK, + ) + + return output.view(b, s_q, h_q, d_v), lse.view(b, s_q, h_q).transpose(1, 2) diff --git a/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_splitk.py b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_splitk.py new file mode 100644 index 000000000000..2f6c6e7898a6 --- /dev/null +++ b/python/sglang/srt/layers/attention/nsa/triton_decode/triton_mla_kernels_decode_splitk.py @@ -0,0 +1,534 @@ +""" +Split-K Attention Kernel for Large TopK Cases + +This module implements a split-K version of the attention kernel that: +1. Splits the K (topk) dimension across multiple kernel instances +2. Each instance computes partial results with its own m_i, l_i, and accumulators +3. A combine kernel merges the partial results using online softmax + +This reduces register pressure by processing fewer K tokens per kernel instance, +improving occupancy and overall performance for large topk cases. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from .triton_mla_kernels_decode_common import _bucket_total_tokens + + +# ============================================================================ +# Split-K Attention Kernel +# ============================================================================ +@triton.autotune( + configs=[ + # Split-K attention on already-gathered BF16 KV. + # - BLOCK_N=256: amortizes memory access over KV tokens (memory-bound kernel). + # - BLOCK_D=128: matches KV tile structure. + # - num_warps=8, num_stages=2: memory-bound kernel benefits from more warps + # and software pipelining (overlaps memory loads with compute). + # - BLOCK_H varies for different batch sizes: + triton.Config( + {"BLOCK_H": 16, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=2 + ), + triton.Config( + {"BLOCK_H": 32, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=2 + ), + triton.Config( + {"BLOCK_H": 64, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=2 + ), + triton.Config( + {"BLOCK_H": 128, "BLOCK_N": 256, "BLOCK_D": 128}, num_warps=8, num_stages=2 + ), + ], + key=["total_tokens_bucket", "h_q", "topk_per_split", "d_qk"], +) +@triton.jit +def _splitk_attention_kernel( + Q, + KV, + Mask, + PartialOutput, + PartialLSE, + PartialM, + sm_scale, + total_tokens, + total_tokens_bucket, + h_q, + total_topk, + d_qk, + d_v, + topk_per_split, + stride_q_t, + stride_q_h, + stride_q_d, + stride_kv_t, + stride_kv_k, + stride_kv_d, + stride_mask_t, + stride_mask_k, + stride_po_s, + stride_po_t, + stride_po_h, + stride_po_d, + stride_plse_s, + stride_plse_t, + stride_plse_h, + stride_pm_s, + stride_pm_t, + stride_pm_h, + BLOCK_H: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Split-K attention kernel that processes a subset of K tokens.""" + LOG2E: tl.constexpr = 1.4426950408889634 + + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + pid_k = tl.program_id(2) + pid_t_64 = pid_t.to(tl.int64) + + NEG_INF = float("-inf") + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + + # Compute K range for this split + k_start = pid_k * topk_per_split + k_end = tl.minimum(k_start + topk_per_split, total_topk) + + m_i = tl.full([BLOCK_H], NEG_INF, dtype=tl.float32) + l_i = tl.zeros([BLOCK_H], dtype=tl.float32) + + acc_0 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_1 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_2 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_3 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + + stride_q_t_64 = tl.cast(stride_q_t, tl.int64) + stride_kv_t_64 = tl.cast(stride_kv_t, tl.int64) + stride_mask_t_64 = tl.cast(stride_mask_t, tl.int64) + q_base = Q + pid_t_64 * stride_q_t_64 + kv_base = KV + pid_t_64 * stride_kv_t_64 + mask_base = Mask + pid_t_64 * stride_mask_t_64 + + for n_start in range(k_start, k_end, BLOCK_N): + offs_n = n_start + tl.arange(0, BLOCK_N) + mask_n = offs_n < k_end + + mask_ptrs = mask_base + offs_n * stride_mask_k + invalid = tl.load(mask_ptrs, mask=mask_n, other=True) + valid = mask_n & ~invalid + + qk = tl.zeros([BLOCK_H, BLOCK_N], dtype=tl.float32) + + for d_start in range(0, d_qk, BLOCK_D): + offs_d = d_start + tl.arange(0, BLOCK_D) + mask_d = offs_d < d_qk + + q_ptrs = ( + q_base + offs_h[:, None] * stride_q_h + offs_d[None, :] * stride_q_d + ) + q_chunk = tl.load( + q_ptrs, mask=mask_h[:, None] & mask_d[None, :], other=0.0 + ).to(tl.bfloat16) + + k_ptrs = ( + kv_base + offs_n[:, None] * stride_kv_k + offs_d[None, :] * stride_kv_d + ) + k_chunk = tl.load( + k_ptrs, mask=valid[:, None] & mask_d[None, :], other=0.0 + ).to(tl.bfloat16) + + qk += tl.dot(q_chunk, tl.trans(k_chunk)) + + qk = qk * sm_scale + qk = tl.where(valid[None, :], qk, NEG_INF) + + m_ij = tl.max(qk, axis=1) + m_new = tl.maximum(m_i, m_ij) + alpha = tl.where(m_i == NEG_INF, 0.0, tl.math.exp2((m_i - m_new) * LOG2E)) + p = tl.where(qk == NEG_INF, 0.0, tl.math.exp2((qk - m_new[:, None]) * LOG2E)) + l_new = alpha * l_i + tl.sum(p, axis=1) + p_bf16 = p.to(tl.bfloat16) + + offs_v = tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load(v_ptrs, mask=valid[:, None], other=0.0).to(tl.bfloat16) + acc_0 = acc_0 * alpha[:, None] + tl.dot(p_bf16, v) + + offs_v = BLOCK_D + tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load( + v_ptrs, mask=valid[:, None] & (offs_v[None, :] < d_v), other=0.0 + ).to(tl.bfloat16) + acc_1 = acc_1 * alpha[:, None] + tl.dot(p_bf16, v) + + offs_v = 2 * BLOCK_D + tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load( + v_ptrs, mask=valid[:, None] & (offs_v[None, :] < d_v), other=0.0 + ).to(tl.bfloat16) + acc_2 = acc_2 * alpha[:, None] + tl.dot(p_bf16, v) + + offs_v = 3 * BLOCK_D + tl.arange(0, BLOCK_D) + v_ptrs = kv_base + offs_n[:, None] * stride_kv_k + offs_v[None, :] * stride_kv_d + v = tl.load( + v_ptrs, mask=valid[:, None] & (offs_v[None, :] < d_v), other=0.0 + ).to(tl.bfloat16) + acc_3 = acc_3 * alpha[:, None] + tl.dot(p_bf16, v) + + m_i = m_new + l_i = l_new + + # Store partial results + stride_po_s_64 = tl.cast(stride_po_s, tl.int64) + stride_po_t_64 = tl.cast(stride_po_t, tl.int64) + po_base = PartialOutput + pid_k * stride_po_s_64 + pid_t_64 * stride_po_t_64 + + offs_h_2d = offs_h[:, None] + mask_h_2d = mask_h[:, None] + offs_v_0 = tl.arange(0, BLOCK_D) + offs_v_1 = BLOCK_D + tl.arange(0, BLOCK_D) + offs_v_2 = 2 * BLOCK_D + tl.arange(0, BLOCK_D) + offs_v_3 = 3 * BLOCK_D + tl.arange(0, BLOCK_D) + + tl.store( + po_base + offs_h_2d * stride_po_h + offs_v_0[None, :] * stride_po_d, + acc_0, + mask=mask_h_2d, + ) + tl.store( + po_base + offs_h_2d * stride_po_h + offs_v_1[None, :] * stride_po_d, + acc_1, + mask=mask_h_2d & (offs_v_1[None, :] < d_v), + ) + tl.store( + po_base + offs_h_2d * stride_po_h + offs_v_2[None, :] * stride_po_d, + acc_2, + mask=mask_h_2d & (offs_v_2[None, :] < d_v), + ) + tl.store( + po_base + offs_h_2d * stride_po_h + offs_v_3[None, :] * stride_po_d, + acc_3, + mask=mask_h_2d & (offs_v_3[None, :] < d_v), + ) + + stride_plse_s_64 = tl.cast(stride_plse_s, tl.int64) + stride_plse_t_64 = tl.cast(stride_plse_t, tl.int64) + plse_ptrs = ( + PartialLSE + + pid_k * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h + ) + tl.store(plse_ptrs, l_i, mask=mask_h) + + stride_pm_s_64 = tl.cast(stride_pm_s, tl.int64) + stride_pm_t_64 = tl.cast(stride_pm_t, tl.int64) + pm_ptrs = ( + PartialM + + pid_k * stride_pm_s_64 + + pid_t_64 * stride_pm_t_64 + + offs_h * stride_pm_h + ) + tl.store(pm_ptrs, m_i, mask=mask_h) + + +# ============================================================================ +# Combine Kernel for Split-K +# ============================================================================ +@triton.autotune( + configs=[ + # Simple reduce kernel merging split-K results. + # - BLOCK_D=128: 4 iterations to cover d_v=512. + # - num_warps=4: sufficient for this simple reduce operation. + # - BLOCK_H varies for different batch sizes: + triton.Config({"BLOCK_H": 16, "BLOCK_D": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 32, "BLOCK_D": 128}, num_warps=4, num_stages=1), + triton.Config({"BLOCK_H": 64, "BLOCK_D": 128}, num_warps=4, num_stages=1), + ], + key=["total_tokens_bucket", "h_q", "split_k"], +) +@triton.jit +def _combine_splitk_attention_kernel( + PartialOutput, + PartialLSE, + PartialM, + AttnSink, + Output, + LSE, + total_tokens, + total_tokens_bucket, + h_q, + d_v, + split_k, + stride_po_s, + stride_po_t, + stride_po_h, + stride_po_d, + stride_plse_s, + stride_plse_t, + stride_plse_h, + stride_pm_s, + stride_pm_t, + stride_pm_h, + stride_o_t, + stride_o_h, + stride_o_d, + stride_lse_t, + stride_lse_h, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Combine partial results from split-K attention kernel.""" + LOG2E: tl.constexpr = 1.4426950408889634 + NEG_INF = float("-inf") + POS_INF = float("+inf") + + pid_t = tl.program_id(0) + pid_h = tl.program_id(1) + pid_t_64 = pid_t.to(tl.int64) + + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = offs_h < h_q + + m_acc = tl.full([BLOCK_H], NEG_INF, dtype=tl.float32) + l_acc = tl.zeros([BLOCK_H], dtype=tl.float32) + + acc_0 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_1 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_2 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + acc_3 = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + + stride_po_s_64 = tl.cast(stride_po_s, tl.int64) + stride_po_t_64 = tl.cast(stride_po_t, tl.int64) + stride_plse_s_64 = tl.cast(stride_plse_s, tl.int64) + stride_plse_t_64 = tl.cast(stride_plse_t, tl.int64) + stride_pm_s_64 = tl.cast(stride_pm_s, tl.int64) + stride_pm_t_64 = tl.cast(stride_pm_t, tl.int64) + + offs_h_2d = offs_h[:, None] + mask_h_2d = mask_h[:, None] + offs_v_0 = tl.arange(0, BLOCK_D) + offs_v_1 = BLOCK_D + tl.arange(0, BLOCK_D) + offs_v_2 = 2 * BLOCK_D + tl.arange(0, BLOCK_D) + offs_v_3 = 3 * BLOCK_D + tl.arange(0, BLOCK_D) + + for k in range(split_k): + k_64 = tl.cast(k, tl.int64) + po_base = PartialOutput + k_64 * stride_po_s_64 + pid_t_64 * stride_po_t_64 + + p_acc_0 = tl.load( + po_base + offs_h_2d * stride_po_h + offs_v_0[None, :] * stride_po_d, + mask=mask_h_2d, + other=0.0, + ) + p_acc_1 = tl.load( + po_base + offs_h_2d * stride_po_h + offs_v_1[None, :] * stride_po_d, + mask=mask_h_2d & (offs_v_1[None, :] < d_v), + other=0.0, + ) + p_acc_2 = tl.load( + po_base + offs_h_2d * stride_po_h + offs_v_2[None, :] * stride_po_d, + mask=mask_h_2d & (offs_v_2[None, :] < d_v), + other=0.0, + ) + p_acc_3 = tl.load( + po_base + offs_h_2d * stride_po_h + offs_v_3[None, :] * stride_po_d, + mask=mask_h_2d & (offs_v_3[None, :] < d_v), + other=0.0, + ) + + plse_ptrs = ( + PartialLSE + + k_64 * stride_plse_s_64 + + pid_t_64 * stride_plse_t_64 + + offs_h * stride_plse_h + ) + p_l = tl.load(plse_ptrs, mask=mask_h, other=0.0) + + pm_ptrs = ( + PartialM + + k_64 * stride_pm_s_64 + + pid_t_64 * stride_pm_t_64 + + offs_h * stride_pm_h + ) + p_m = tl.load(pm_ptrs, mask=mask_h, other=NEG_INF) + + m_new = tl.maximum(m_acc, p_m) + alpha_acc = tl.where( + m_acc == NEG_INF, 0.0, tl.math.exp2((m_acc - m_new) * LOG2E) + ) + alpha_p = tl.where(p_m == NEG_INF, 0.0, tl.math.exp2((p_m - m_new) * LOG2E)) + l_new = alpha_acc * l_acc + alpha_p * p_l + + acc_0 = acc_0 * alpha_acc[:, None] + p_acc_0 * alpha_p[:, None] + acc_1 = acc_1 * alpha_acc[:, None] + p_acc_1 * alpha_p[:, None] + acc_2 = acc_2 * alpha_acc[:, None] + p_acc_2 * alpha_p[:, None] + acc_3 = acc_3 * alpha_acc[:, None] + p_acc_3 * alpha_p[:, None] + + m_acc = m_new + l_acc = l_new + + lse = m_acc + tl.math.log2(tl.where(l_acc == 0.0, 1.0, l_acc)) / LOG2E + is_lonely_q = l_acc == 0.0 + + if HAS_ATTN_SINK: + attn_sink_vals = tl.load(AttnSink + offs_h, mask=mask_h, other=0.0) + exp_attn_sink_minus_m = tl.math.exp2((attn_sink_vals - m_acc) * LOG2E) + denominator = l_acc + exp_attn_sink_minus_m + denominator = tl.where(denominator == 0.0, 1.0, denominator) + output_scale = 1.0 / denominator + else: + output_scale = tl.where(l_acc == 0.0, 0.0, 1.0 / l_acc) + + is_lonely_q_2d = is_lonely_q[:, None] + output_scale_2d = output_scale[:, None] + acc_0 = tl.where(is_lonely_q_2d, 0.0, acc_0 * output_scale_2d) + acc_1 = tl.where(is_lonely_q_2d, 0.0, acc_1 * output_scale_2d) + acc_2 = tl.where(is_lonely_q_2d, 0.0, acc_2 * output_scale_2d) + acc_3 = tl.where(is_lonely_q_2d, 0.0, acc_3 * output_scale_2d) + lse = tl.where(is_lonely_q, POS_INF, lse) + + stride_o_t_64 = tl.cast(stride_o_t, tl.int64) + o_base = Output + pid_t_64 * stride_o_t_64 + + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_0[None, :] * stride_o_d, + acc_0.to(tl.bfloat16), + mask=mask_h_2d, + ) + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_1[None, :] * stride_o_d, + acc_1.to(tl.bfloat16), + mask=mask_h_2d & (offs_v_1[None, :] < d_v), + ) + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_2[None, :] * stride_o_d, + acc_2.to(tl.bfloat16), + mask=mask_h_2d & (offs_v_2[None, :] < d_v), + ) + tl.store( + o_base + offs_h_2d * stride_o_h + offs_v_3[None, :] * stride_o_d, + acc_3.to(tl.bfloat16), + mask=mask_h_2d & (offs_v_3[None, :] < d_v), + ) + + stride_lse_t_64 = tl.cast(stride_lse_t, tl.int64) + tl.store(LSE + pid_t_64 * stride_lse_t_64 + offs_h * stride_lse_h, lse, mask=mask_h) + + +# ============================================================================ +# Runner Function +# ============================================================================ +def run_splitk_attention( + q_reshaped: torch.Tensor, + gathered_kv: torch.Tensor, + invalid_mask: torch.Tensor, + d_v: int, + sm_scale: float, + total_tokens: int, + h_q: int, + total_topk: int, + d_qk: int, + attn_sink: Optional[torch.Tensor] = None, + split_k: int = 4, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Run split-K attention kernel.""" + device = q_reshaped.device + + topk_per_split = (total_topk + split_k - 1) // split_k + + partial_output = torch.empty( + split_k, total_tokens, h_q, d_v, dtype=torch.float32, device=device + ) + partial_lse = torch.empty( + split_k, total_tokens, h_q, dtype=torch.float32, device=device + ) + partial_m = torch.empty( + split_k, total_tokens, h_q, dtype=torch.float32, device=device + ) + + output = torch.empty(total_tokens, h_q, d_v, dtype=torch.bfloat16, device=device) + lse = torch.empty(total_tokens, h_q, dtype=torch.float32, device=device) + + grid_splitk = lambda meta: ( + total_tokens, + triton.cdiv(h_q, meta["BLOCK_H"]), + split_k, + ) + _splitk_attention_kernel[grid_splitk]( + q_reshaped, + gathered_kv, + invalid_mask, + partial_output, + partial_lse, + partial_m, + sm_scale, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + total_topk, + d_qk, + d_v, + topk_per_split, + q_reshaped.stride(0), + q_reshaped.stride(1), + q_reshaped.stride(2), + gathered_kv.stride(0), + gathered_kv.stride(1), + gathered_kv.stride(2), + invalid_mask.stride(0), + invalid_mask.stride(1), + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + partial_m.stride(0), + partial_m.stride(1), + partial_m.stride(2), + ) + + HAS_ATTN_SINK = attn_sink is not None + attn_sink_tensor = attn_sink if HAS_ATTN_SINK else lse[:1] + + grid_combine = lambda meta: (total_tokens, triton.cdiv(h_q, meta["BLOCK_H"])) + _combine_splitk_attention_kernel[grid_combine]( + partial_output, + partial_lse, + partial_m, + attn_sink_tensor, + output, + lse, + total_tokens, + _bucket_total_tokens(total_tokens), + h_q, + d_v, + split_k, + partial_output.stride(0), + partial_output.stride(1), + partial_output.stride(2), + partial_output.stride(3), + partial_lse.stride(0), + partial_lse.stride(1), + partial_lse.stride(2), + partial_m.stride(0), + partial_m.stride(1), + partial_m.stride(2), + output.stride(0), + output.stride(1), + output.stride(2), + lse.stride(0), + lse.stride(1), + HAS_ATTN_SINK=HAS_ATTN_SINK, + ) + + return output, lse diff --git a/python/sglang/srt/layers/attention/tbo_backend.py b/python/sglang/srt/layers/attention/tbo_backend.py index 2ae120686d64..76d83b7b73ca 100644 --- a/python/sglang/srt/layers/attention/tbo_backend.py +++ b/python/sglang/srt/layers/attention/tbo_backend.py @@ -15,6 +15,10 @@ def __init__(self, primary: AttentionBackend, children: List[AttentionBackend]): super().__init__() self.primary = primary self.children = children + # Dispatcher aliases the primary's pool refs so get_attn_backend() + # reads through TboAttnBackend resolve to the underlying pool. + self.token_to_kv_pool = primary.token_to_kv_pool + self.req_to_token_pool = primary.req_to_token_pool @classmethod def init_new(cls, creator: Callable[[], AttentionBackend]): diff --git a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py index 9ab17576ae52..af3da62c3b10 100644 --- a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py +++ b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py @@ -134,8 +134,9 @@ def __init__( # branch, which always asks for the LSE. if is_causal is False and return_lse is False: continue + # Runtime feeds fp8_e4m3fn q/k/v config = ( - torch.bfloat16, + torch.float8_e4m3fn, head_dim_qk, self.v_head_dim, is_causal, @@ -146,7 +147,7 @@ def __init__( if config in _compiled_kernels: continue _compiled_kernels[config] = _compile_prefill_kernel( - torch.bfloat16, + torch.float8_e4m3fn, head_dim_qk, self.v_head_dim, is_causal, @@ -249,7 +250,7 @@ def prepare_prefill_qkv( # reproduces the original [tokens, 1, qk_rope] latent layout. kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl()) k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :] - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer.attn_mha, forward_batch.out_cache_loc, kv_a_fp8.unsqueeze(1), diff --git a/python/sglang/srt/layers/attention/torch_flex_backend.py b/python/sglang/srt/layers/attention/torch_flex_backend.py index 69f097efd006..1af8508cb934 100644 --- a/python/sglang/srt/layers/attention/torch_flex_backend.py +++ b/python/sglang/srt/layers/attention/torch_flex_backend.py @@ -19,6 +19,10 @@ def __init__(self, model_runner: ModelRunner): super().__init__() self.forward_metadata = None self.device = model_runner.device + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.flex_attention = torch.compile(flex_attention, dynamic=True) torch._dynamo.config.cache_size_limit = 1024 torch._dynamo.config.accumulated_cache_size_limit = 1024 @@ -248,7 +252,7 @@ def forward_extend( o = torch.empty_like(q) if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) @@ -266,9 +270,9 @@ def forward_extend( self._run_flex_forward_extend( q_, o_, - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), - forward_batch.req_to_token_pool.req_to_token, + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, forward_batch.extend_prefix_lens, @@ -298,7 +302,7 @@ def forward_decode( o = torch.empty_like(q) if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) @@ -309,9 +313,9 @@ def forward_decode( self._run_flex_forward_decode( q_, o_, - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), - forward_batch.req_to_token_pool.req_to_token, + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, scaling=layer.scaling, diff --git a/python/sglang/srt/layers/attention/torch_native_backend.py b/python/sglang/srt/layers/attention/torch_native_backend.py index 00d424c44f09..8894f92ca729 100644 --- a/python/sglang/srt/layers/attention/torch_native_backend.py +++ b/python/sglang/srt/layers/attention/torch_native_backend.py @@ -19,6 +19,10 @@ def __init__(self, model_runner: ModelRunner): super().__init__() self.forward_metadata = None self.device = model_runner.device + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool def init_forward_metadata(self, forward_batch: ForwardBatch): """Init the metadata for a forward pass.""" @@ -235,7 +239,7 @@ def forward_extend( cache_loc = forward_batch.out_cache_loc if save_kv_cache and k is not None and v is not None: - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) use_gqa = layer.tp_q_head_num != layer.tp_k_head_num @@ -249,9 +253,9 @@ def forward_extend( self._run_sdpa_forward_extend( q_, o_, - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), - forward_batch.req_to_token_pool.req_to_token, + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, forward_batch.extend_prefix_lens, @@ -292,9 +296,8 @@ def forward_decode( else: cache_loc = forward_batch.out_cache_loc - if save_kv_cache: - if k is not None and v is not None: - forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + if save_kv_cache and k is not None and v is not None: + self.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) use_gqa = layer.tp_q_head_num != layer.tp_k_head_num @@ -304,9 +307,9 @@ def forward_decode( self._run_sdpa_forward_decode( q_, o_, - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), - forward_batch.req_to_token_pool.req_to_token, + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.req_to_token_pool.req_to_token, forward_batch.req_pool_indices, forward_batch.seq_lens, forward_batch.encoder_lens, diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py index 206037f4957f..1ec56763478e 100644 --- a/python/sglang/srt/layers/attention/triton_backend.py +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -105,6 +105,10 @@ def __init__( self.skip_prefill = skip_prefill max_bs = model_runner.req_to_token_pool.size self.sliding_window_size = model_runner.sliding_window_size + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.token_to_kv_pool_allocator = model_runner.token_to_kv_pool_allocator self.num_draft_tokens = model_runner.server_args.speculative_num_draft_tokens @@ -321,7 +325,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): forward_batch.req_pool_indices, bs, self.device, - self.token_to_kv_pool_allocator, + self.token_to_kv_pool, ) ) window_num_kv_splits = torch.empty( @@ -397,7 +401,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): forward_batch.req_pool_indices, bs, self.device, - self.token_to_kv_pool_allocator, + self.token_to_kv_pool, ) custom_mask = spec_info.custom_mask @@ -464,7 +468,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): forward_batch.req_pool_indices, bs, self.device, - self.token_to_kv_pool_allocator, + self.token_to_kv_pool, ) qo_indptr = self.qo_indptr @@ -621,7 +625,7 @@ def init_forward_metadata_capture_cuda_graph( seq_lens[:bs], req_pool_indices, bs, - self.token_to_kv_pool_allocator, + self.token_to_kv_pool, ) ) else: @@ -670,7 +674,7 @@ def init_forward_metadata_capture_cuda_graph( seq_lens[:bs], req_pool_indices, bs, - self.token_to_kv_pool_allocator, + self.token_to_kv_pool, ) ) @@ -783,7 +787,7 @@ def init_forward_metadata_replay_cuda_graph( seq_lens[:bs], req_pool_indices[:bs], bs, - self.token_to_kv_pool_allocator, + self.token_to_kv_pool, ) self.get_num_kv_splits( window_num_kv_splits[:num_token], window_kv_lens[:bs] @@ -829,7 +833,7 @@ def init_forward_metadata_replay_cuda_graph( seq_lens[:bs], req_pool_indices, bs, - self.token_to_kv_pool_allocator, + self.token_to_kv_pool, ) ) custom_mask = self.cuda_graph_custom_mask @@ -904,7 +908,7 @@ def forward_extend( o = torch.empty_like(q) if k is None and v is None: - pool = forward_batch.token_to_kv_pool + pool = self.token_to_kv_pool cache_loc = forward_batch.out_cache_loc if isinstance(pool, SWAKVPool) and pool.layers_mapping[layer.layer_id][1]: cache_loc = pool.translate_loc_from_full_to_swa(cache_loc) @@ -917,7 +921,7 @@ def forward_extend( # Save KV cache first (must do this before unified kernel) if save_kv_cache: if layer.k_scale is None: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, @@ -928,14 +932,14 @@ def forward_extend( # doesn't accept scale parameters. Clone to protect k from mutation # since it's used later in the attention kernel. k_scaled = k.clone().div_(layer.k_scale) - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k_scaled, v, ) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k.clone(), # cloned to protect k,v from in-place mutation in set_kv_buffer @@ -989,8 +993,8 @@ def forward_extend( k.contiguous(), v.contiguous(), o.view(-1, layer.tp_q_head_num, layer.v_head_dim), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), self.forward_metadata.qo_indptr, kv_indptr, kv_indices, @@ -1058,7 +1062,7 @@ def _forward_extend_unified( window_start_pos = None extend_kv_indices = forward_batch.out_cache_loc - pool = forward_batch.token_to_kv_pool + pool = self.token_to_kv_pool if ( layer.sliding_window_size is not None and layer.sliding_window_size > -1 @@ -1124,8 +1128,8 @@ def _forward_extend_unified( self.extend_attention_fwd_unified( q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), o.view(-1, layer.tp_q_head_num, layer.v_head_dim), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), k_descale, v_descale, self.forward_metadata.qo_indptr, @@ -1174,14 +1178,14 @@ def forward_decode( # MLATokenToKVPool doesn't accept scale parameters; k is unused # after this point in decode, so scale in place. k.div_(layer.k_scale) - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v, ) else: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, @@ -1216,8 +1220,8 @@ def forward_decode( self.decode_attention_fwd( q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), o.view(-1, layer.tp_q_head_num, layer.v_head_dim), kv_indptr, kv_indices, @@ -1275,6 +1279,7 @@ def __init__( ) self.device = model_runner.device # Cached variables for generate_draft_decode_kv_indices + self.req_to_token_pool = model_runner.req_to_token_pool self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] self.page_size = model_runner.server_args.page_size @@ -1295,7 +1300,7 @@ def common_template( (self.speculative_num_steps, num_seqs, self.topk) ]( forward_batch.req_pool_indices, - forward_batch.req_to_token_pool.req_to_token, + self.req_to_token_pool.req_to_token, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, @@ -1453,7 +1458,7 @@ def update_sliding_window_buffer( req_pool_indices, bs, device, - token_to_kv_pool_allocator=None, + token_to_kv_pool=None, ): window_kv_lens = torch.minimum( seq_lens, @@ -1475,13 +1480,16 @@ def update_sliding_window_buffer( req_to_token.stride(0), ) # full to swa index mapping - if hasattr(token_to_kv_pool_allocator, "translate_loc_from_full_to_swa"): + if hasattr(token_to_kv_pool, "translate_loc_from_full_to_swa"): kv_last_index = window_kv_indptr[-1] + # Flush before+after: window_kv_indices is a different tensor than out_cache_loc. + token_to_kv_pool.invalidate_loc_cache() window_kv_indices[:kv_last_index] = ( - token_to_kv_pool_allocator.translate_loc_from_full_to_swa( + token_to_kv_pool.translate_loc_from_full_to_swa( window_kv_indices[:kv_last_index] ) ) + token_to_kv_pool.invalidate_loc_cache() return window_kv_indptr, window_kv_indices, window_kv_lens, window_kv_start_idx @@ -1493,7 +1501,7 @@ def update_sliding_window_buffer_cuda_graph( seq_lens, req_pool_indices, bs, - token_to_kv_pool_allocator=None, + token_to_kv_pool=None, ): window_kv_lens = torch.minimum( seq_lens, @@ -1512,11 +1520,14 @@ def update_sliding_window_buffer_cuda_graph( req_to_token.stride(0), ) # full to swa index mapping - if hasattr(token_to_kv_pool_allocator, "translate_loc_from_full_to_swa"): + if hasattr(token_to_kv_pool, "translate_loc_from_full_to_swa"): kv_last_index = window_kv_indptr[-1] + # Flush before+after: window_kv_indices is a different tensor than out_cache_loc. + token_to_kv_pool.invalidate_loc_cache() window_kv_indices[:kv_last_index] = ( - token_to_kv_pool_allocator.translate_loc_from_full_to_swa( + token_to_kv_pool.translate_loc_from_full_to_swa( window_kv_indices[:kv_last_index] ) ) + token_to_kv_pool.invalidate_loc_cache() return window_kv_indptr, window_kv_indices, window_kv_lens, window_kv_start_idx diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index 722c71022bd6..b74a11846721 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -537,6 +537,11 @@ def init_forward_metadata_replay_cuda_graph( self._copy_swa_page_table(metadata, page_indices, max_seq_pages) self.forward_metadata = metadata + def update_verify_buffers_to_fill_after_draft( + self, spec_info: SpecInput, cuda_graph_bs: Optional[int] + ): + pass + def get_cuda_graph_seq_len_fill_value(self) -> int: """Get the fill value for sequence lengths in CUDA graph.""" return 1 @@ -558,7 +563,7 @@ def _fused_fp8_set_kv_buffer( cache_loc = self._get_layer_cache_loc(layer, forward_batch) # Get K/V cache buffers from token_to_kv_pool - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) fused_fp8_set_kv_buffer( k=k, @@ -598,7 +603,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] else: @@ -611,7 +616,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): metadata.cu_seqlens_k = torch.nn.functional.pad( torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] elif forward_batch.forward_mode.is_target_verify(): @@ -635,7 +640,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -645,7 +650,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): metadata.cu_seqlens_k = torch.nn.functional.pad( torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -713,7 +718,7 @@ def forward_decode( else: # Use original set_kv_buffer path if save_kv_cache and k is not None: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) @@ -721,7 +726,7 @@ def forward_decode( if self.data_type == torch.float8_e4m3fn and (not self.is_xqa_impl): q = q.to(torch.float8_e4m3fn) q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim) - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) # shape conversion: # [num_pages, page_size, num_kv_heads, head_dim] -> [num_pages, num_kv_heads, page_size, head_dim] k_cache = k_cache.view( @@ -799,7 +804,7 @@ def forward_extend( else: # Use original set_kv_buffer path if save_kv_cache and k is not None: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) @@ -807,7 +812,7 @@ def forward_extend( q = q.to(torch.float8_e4m3fn) q = q.reshape(-1, layer.tp_q_head_num, layer.head_dim) # [num_pages, page_size, num_kv_heads, head_dim] -> [num_pages, num_kv_heads, page_size, head_dim] - k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + k_cache, v_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) k_cache = k_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim ).permute(0, 2, 1, 3) diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index 87f9c281d510..4a5fed11cf2e 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -577,6 +577,14 @@ def get_cuda_graph_seq_len_fill_value(self) -> int: """Get the fill value for sequence lengths in CUDA graph.""" return 1 + def init_mha_chunk_metadata(self, forward_batch: "ForwardBatch") -> None: + has_prefix = any(forward_batch.extend_prefix_lens_cpu) + fallback_to_flashinfer_impl = ( + self.disable_chunked_prefix_cache and has_prefix + ) or is_in_piecewise_cuda_graph() + if fallback_to_flashinfer_impl: + super().init_mha_chunk_metadata(forward_batch) + def init_forward_metadata(self, forward_batch: ForwardBatch): """Initialize the metadata for a forward pass.""" # Delegate to parent for non-decode modes. @@ -898,7 +906,7 @@ def forward_decode( assert ( k is not None and k_rope is not None ), "For populating trtllm_mla kv cache, both k_nope and k_rope should be not None." - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, forward_batch.out_cache_loc, k, k_rope ) @@ -924,7 +932,7 @@ def forward_decode( query = query.unsqueeze(1) # Prepare KV cache inline - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1) # Get metadata @@ -1005,7 +1013,7 @@ def forward_extend( assert ( k is not None and k_rope is not None ), "For populating trtllm_mla kv cache, both k_nope and k_rope should be not None." - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, forward_batch.out_cache_loc, k, k_rope ) @@ -1046,7 +1054,7 @@ def forward_extend( # Ensure query has shape [bs, num_draft_tokens, num_q_heads, head_dim] bs = forward_batch.batch_size - k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1) q = q.to(self.data_type) @@ -1227,3 +1235,22 @@ def __init__( kv_indptr_buf=self.kv_indptr[i], q_indptr_decode_buf=self.q_indptr_decode, ) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + for i in range(self.speculative_num_steps - 1): + self.attn_backends[i].init_forward_metadata(forward_batch) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + for i in range(self.speculative_num_steps - 1): + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + seq_lens_sum=None, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=forward_batch.seq_lens_cpu, + ) diff --git a/python/sglang/srt/layers/attention/utils.py b/python/sglang/srt/layers/attention/utils.py index 277d46054bcd..65328b16b836 100644 --- a/python/sglang/srt/layers/attention/utils.py +++ b/python/sglang/srt/layers/attention/utils.py @@ -10,7 +10,7 @@ _is_cuda = is_cuda() if _is_cuda: - from sgl_kernel import concat_mla_absorb_q + from sglang.jit_kernel.concat_mla import concat_mla_absorb_q from sglang.jit_kernel.utils import is_arch_support_pdl diff --git a/python/sglang/srt/layers/attention/wave_backend.py b/python/sglang/srt/layers/attention/wave_backend.py index 2a759c22207d..ebdadc9bf010 100644 --- a/python/sglang/srt/layers/attention/wave_backend.py +++ b/python/sglang/srt/layers/attention/wave_backend.py @@ -117,6 +117,11 @@ def __init__( self.skip_prefill = skip_prefill + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool + max_bs = model_runner.req_to_token_pool.size if kv_indptr_buf is None: @@ -556,7 +561,7 @@ def forward_extend( o = torch.empty_like(q) if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) @@ -571,8 +576,8 @@ def forward_extend( q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), k.contiguous(), v.contiguous(), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), self.forward_metadata.qo_indptr, self.forward_metadata.kv_indptr, self.forward_metadata.kv_indices, @@ -606,14 +611,14 @@ def forward_decode( o = torch.empty_like(q) if save_kv_cache: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, forward_batch.out_cache_loc, k, v ) self.decode_attention_fwd( q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), - forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), - forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.token_to_kv_pool.get_key_buffer(layer.layer_id), + self.token_to_kv_pool.get_value_buffer(layer.layer_id), o.view(-1, layer.tp_q_head_num, layer.v_head_dim), self.forward_metadata.kv_indptr, self.forward_metadata.kv_indices, diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index e918af462dfc..3b5743799dfb 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -61,6 +61,10 @@ def __init__( self.device = model_runner.device self.decode_cuda_graph_metadata = {} self.target_verify_metadata = {} + # Pool refs — captured at construction so they survive deletion of the + # corresponding ForwardBatch fields. + self.req_to_token_pool = model_runner.req_to_token_pool + self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype @@ -122,7 +126,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] else: @@ -142,7 +146,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -186,7 +190,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): metadata.cu_seqlens_k = torch.nn.functional.pad( torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] # TODO: we need to test this part for llama 4 eagle case @@ -214,7 +218,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -236,7 +240,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ), (1, 0), ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -297,7 +301,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): ) _, sort_order = torch.sort(keys, dim=1) non_masked_page_table = ( - forward_batch.req_to_token_pool.req_to_token[ + self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : ] .gather(1, cols) @@ -324,7 +328,7 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): metadata.cu_seqlens_k = torch.nn.functional.pad( torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) ) - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.max_seq_len_k ] @@ -357,12 +361,12 @@ def init_forward_metadata(self, forward_batch: ForwardBatch): (1, 0), ) metadata.encoder_max_seq_len_k = metadata.encoder_lens_int32.max().item() - metadata.encoder_page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.encoder_page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, : metadata.encoder_max_seq_len_k ] # Currently only support forward_batch.encoder_lens.numel() == 1 - metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + metadata.page_table = self.req_to_token_pool.req_to_token[ forward_batch.req_pool_indices, metadata.encoder_max_seq_len_k : ( metadata.encoder_max_seq_len_k + metadata.max_seq_len_k @@ -418,11 +422,11 @@ def forward_extend( else forward_batch.encoder_out_cache_loc ) if not self.use_mla: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) else: - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, @@ -501,9 +505,7 @@ def forward_extend( # Use Flash Attention for prefill if not self.use_mla: # Do multi-head attention - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim ) @@ -614,9 +616,9 @@ def forward_extend( return output else: # Do absorbed multi-latent attention - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer( - layer.layer_id - ).to(q.dtype) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to( + q.dtype + ) k_rope = kv_cache[:, :, layer.v_head_dim :] c_kv = kv_cache[:, :, : layer.v_head_dim] k_rope_cache = k_rope.view( @@ -710,14 +712,14 @@ def forward_decode( else forward_batch.encoder_out_cache_loc ) if not self.use_mla: - forward_batch.token_to_kv_pool.set_kv_buffer( + self.token_to_kv_pool.set_kv_buffer( layer, cache_loc, k, v, layer.k_scale, layer.v_scale ) else: k_rope_val = ( k_rope if k_rope is not None else k[:, :, layer.v_head_dim :] ) - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.token_to_kv_pool.set_mla_kv_buffer( layer, cache_loc, k, @@ -768,9 +770,7 @@ def forward_decode( if not self.use_mla: # Do multi-head attention - key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( - layer.layer_id - ) + key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) key_cache = key_cache.view( -1, self.page_size, layer.tp_k_head_num, layer.head_dim ) @@ -876,9 +876,7 @@ def forward_decode( o = result else: # Do absorbed multi-latent attention - kv_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( - q.dtype - ) + kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype) assert not use_cascade_attn, "Cascade attention is not supported with MLA" if q_rope is not None: @@ -927,6 +925,17 @@ def _init_local_attn_metadata( metadata.local_attn_metadata = None return + # make_local_attention_virtual_batches expects a page-granularity block table: + # column p is the logical page number, and the value stored at that column is the + # physical page index. The raw req_to_token table is token-granularity (column i = + # the KV slot for token i), so when page_size > 1 we must stride and divide first + # so that block_starts = k_seqstarts_absolute // page_size correctly indexes the table. + if self.page_size > 1: + strided_indices = torch.arange( + 0, page_table.shape[1], self.page_size, device=page_table.device + ) + page_table = page_table[:, strided_indices] // self.page_size + cu_seqlens_q_np = cu_seqlens_q.cpu().numpy() seq_lens_np = cache_seqlens_int32.cpu().numpy() ( diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 7c40b0c103eb..2efc1e775981 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -65,6 +65,10 @@ should_use_dp_reduce_scatterv, should_use_flashinfer_cutlass_moe_fp4_allgather, ) +from sglang.srt.layers.utils.cp_utils import ( + is_mla_prefill_cp_enabled, + mla_use_prefill_cp, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.server_args import get_global_server_args from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -202,7 +206,7 @@ class ScatterMode(Enum): @staticmethod def model_input_output(): """The scatter mode for model forward pass input and output data""" - if is_dsa_enable_prefill_cp(): + if is_dsa_enable_prefill_cp() or is_mla_prefill_cp_enabled(): return ScatterMode.SCATTERED return ScatterMode.TP_ATTN_FULL @@ -379,8 +383,10 @@ def _compute_mlp_mode(cls, context: _LayerModeComputationContext): or should_use_flashinfer_cutlass_moe_fp4_allgather() ): return ScatterMode.SCATTERED - # DSA CP doesn't support MOE_FULL yet; fall back to FULL - if is_enable_moe_cp_allgather() and not is_dsa_enable_prefill_cp(): + # DSA CP and MLA CP both don't support MOE_FULL yet; fall back to FULL. + if is_enable_moe_cp_allgather() and not ( + is_dsa_enable_prefill_cp() or is_mla_prefill_cp_enabled() + ): return ScatterMode.MOE_FULL return ScatterMode.FULL else: @@ -709,7 +715,7 @@ def should_use_reduce_scatter(self, forward_batch: ForwardBatch): return True if forward_batch.dp_padding_mode.is_max_len(): return True - if dsa_use_prefill_cp(forward_batch): + if dsa_use_prefill_cp(forward_batch) or mla_use_prefill_cp(forward_batch): return True if get_attn_tp_context().input_scattered and not self.is_last_layer: return True @@ -742,6 +748,11 @@ def should_fuse_mlp_allreduce_with_next_layer( else 0 ) + # When mlp_mode is SCATTERED, the MLP runs on scattered data with no TP + # all-reduce, so there is nothing to fuse with the next layer. + if self.layer_scatter_modes.mlp_mode == ScatterMode.SCATTERED: + return False + return ( ( apply_flashinfer_allreduce_fusion(batch_size) diff --git a/python/sglang/srt/layers/communicator_dsa_cp.py b/python/sglang/srt/layers/communicator_dsa_cp.py index 20b220278f35..46346ca3be0b 100644 --- a/python/sglang/srt/layers/communicator_dsa_cp.py +++ b/python/sglang/srt/layers/communicator_dsa_cp.py @@ -37,6 +37,7 @@ get_attention_cp_group, get_local_dp_buffer, ) +from sglang.srt.layers.utils.cp_utils import mla_use_prefill_cp from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -152,7 +153,7 @@ def _gather_hidden_states_and_residual( hidden_states, residual = layernorm(hidden_states, residual) # for prefill: attn tp scattered -> full # for decode: attn tp full -> full - if dsa_use_prefill_cp(forward_batch): + if dsa_use_prefill_cp(forward_batch) or mla_use_prefill_cp(forward_batch): assert context.attn_dp_size == 1 hidden_states, local_hidden_states = ( get_local_dp_buffer(get_attention_cp_group()), @@ -205,7 +206,7 @@ def _scatter_hidden_states( ): # for prefill: full -> attn tp scattered # for decode: full -> attn tp full - if dsa_use_prefill_cp(forward_batch): + if dsa_use_prefill_cp(forward_batch) or mla_use_prefill_cp(forward_batch): assert context.attn_dp_size == 1 input_hidden_states = hidden_states hidden_states = hidden_states.tensor_split(context.attn_cp_size)[ diff --git a/python/sglang/srt/layers/deepseek_v4_rope.py b/python/sglang/srt/layers/deepseek_v4_rope.py index c8d391426d26..69a8da7bfe06 100644 --- a/python/sglang/srt/layers/deepseek_v4_rope.py +++ b/python/sglang/srt/layers/deepseek_v4_rope.py @@ -2,17 +2,20 @@ from functools import lru_cache from typing import Optional -import tilelang import torch import triton import triton.language as tl -tilelang.set_log_level("WARNING") +try: + import tilelang -pass_configs = { - tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, - tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, -} + tilelang.set_log_level("WARNING") + pass_configs = { + tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, + tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, + } +except ImportError: + pass FP8 = "float8_e4m3" BF16 = "bfloat16" @@ -285,6 +288,92 @@ def _fused_norm_rope_kernel( ) +@triton.jit +def _fused_softmax_pool_kernel( + kv_score_ptr, + out_ptr, + stride_bs: tl.constexpr, + stride_k: tl.constexpr, + K: tl.constexpr, + HEAD_DIM: tl.constexpr, + HEAD_BLOCK: tl.constexpr, +): + pid = tl.program_id(0) + base = pid * stride_bs + + offs = tl.arange(0, HEAD_BLOCK) + mask = offs < HEAD_DIM + + max_val = tl.full([HEAD_BLOCK], float("-inf"), dtype=tl.float32) + for k in range(K): + s = tl.load( + kv_score_ptr + base + k * stride_k + HEAD_DIM + offs, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + max_val = tl.maximum(max_val, s) + + sum_exp = tl.zeros([HEAD_BLOCK], dtype=tl.float32) + weighted = tl.zeros([HEAD_BLOCK], dtype=tl.float32) + for k in range(K): + s = tl.load( + kv_score_ptr + base + k * stride_k + HEAD_DIM + offs, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + v = tl.load( + kv_score_ptr + base + k * stride_k + offs, + mask=mask, + other=0.0, + ).to(tl.float32) + w = tl.exp(s - max_val) + sum_exp += w + weighted += v * w + + result = weighted / sum_exp + tl.store( + out_ptr + pid * HEAD_DIM + offs, result.to(out_ptr.dtype.element_ty), mask=mask + ) + + +def fused_softmax_pool_triton( + kv_score: torch.Tensor, + head_dim: int, +) -> torch.Tensor: + """Fused softmax-weighted-sum: out = (kv * softmax(score, dim=1)).sum(dim=1). + + Replaces the generic cunn_SpatialSoftMaxForward + elementwise multiply + sum + with a single Triton kernel. + + Args: + kv_score: [bs, K, 2 * head_dim] where first head_dim is kv, second is score. + head_dim: dimension of each of kv and score. + Returns: + output: [bs, head_dim] + """ + assert kv_score.dim() == 3 + bs, K, last = kv_score.shape + assert last == 2 * head_dim + assert kv_score.is_contiguous() + + out = torch.empty(bs, head_dim, dtype=kv_score.dtype, device=kv_score.device) + if bs == 0: + return out + + HEAD_BLOCK = triton.next_power_of_2(head_dim) + grid = (bs,) + _fused_softmax_pool_kernel[grid]( + kv_score, + out, + stride_bs=kv_score.stride(0), + stride_k=kv_score.stride(1), + K=K, + HEAD_DIM=head_dim, + HEAD_BLOCK=HEAD_BLOCK, + ) + return out + + def fused_norm_rope_inplace_triton( kv: torch.Tensor, weight: Optional[torch.Tensor], diff --git a/python/sglang/srt/layers/flashinfer_comm_fusion.py b/python/sglang/srt/layers/flashinfer_comm_fusion.py index c66c2cda4d8a..b041be9e6bef 100644 --- a/python/sglang/srt/layers/flashinfer_comm_fusion.py +++ b/python/sglang/srt/layers/flashinfer_comm_fusion.py @@ -1,4 +1,5 @@ import contextlib +import inspect import logging import platform from typing import Optional, Tuple @@ -30,6 +31,9 @@ _flashinfer_comm = None _TorchDistBackend = None _flashinfer_allreduce_unavailable = False +_flashinfer_create_workspace_supports_group = False +_flashinfer_create_workspace_supports_comm_backend = False +_flashinfer_allreduce_supports_trigger_completion = False _posix_transport_override_logged = False @@ -106,6 +110,17 @@ def _always_disable_fabric(_device_idx: int) -> bool: comm, "create_allreduce_fusion_workspace" ): _flashinfer_comm = comm + workspace_params = inspect.signature( + comm.create_allreduce_fusion_workspace + ).parameters + allreduce_params = inspect.signature(comm.allreduce_fusion).parameters + _flashinfer_create_workspace_supports_group = "group" in workspace_params + _flashinfer_create_workspace_supports_comm_backend = ( + "comm_backend" in workspace_params + ) + _flashinfer_allreduce_supports_trigger_completion = ( + "trigger_completion_at_end" in allreduce_params + ) else: _flashinfer_allreduce_unavailable = True logger.warning( @@ -383,14 +398,15 @@ def initialize( hidden_dim=hidden_dim, dtype=dtype, force_oneshot_support=bool(use_oneshot), - # Pin the symmetric-memory rendezvous to the actual - # subgroup. Without this, flashinfer >=0.6.10 falls back - # to WORLD and TP/EP/CP subgroup peers get addressed - # incorrectly (kernel hangs in cuda-graph warmup). - group=device_group, ) + create_workspace = _flashinfer_comm.create_allreduce_fusion_workspace + if _flashinfer_create_workspace_supports_group: + # Pin the symmetric-memory rendezvous to the actual subgroup. + # Older FlashInfer releases only support comm_backend. + kwargs["group"] = device_group if ( _TorchDistBackend is not None + and _flashinfer_create_workspace_supports_comm_backend and device_group is not None and cpu_group is not None ): @@ -398,9 +414,7 @@ def initialize( device_group=device_group, cpu_group=cpu_group ) with _flashinfer_posix_fd_transport_override_if_needed(): - self.workspace = _flashinfer_comm.create_allreduce_fusion_workspace( - **kwargs - ) + self.workspace = create_workspace(**kwargs) except Exception as e: _flashinfer_allreduce_unavailable = True logger.warning( @@ -669,12 +683,11 @@ def flashinfer_allreduce_residual_rmsnorm( norm_out = torch.empty_like(input_tensor) workspace_manager = _get_workspace_manager(use_attn_tp_group) - _flashinfer_comm.allreduce_fusion( + kwargs = dict( input=input_tensor, workspace=workspace_manager.workspace, pattern=_flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm, launch_with_pdl=True, - trigger_completion_at_end=trigger_completion_at_end, residual_out=residual_out, norm_out=norm_out, residual_in=residual, @@ -683,6 +696,9 @@ def flashinfer_allreduce_residual_rmsnorm( use_oneshot=use_oneshot, fp32_acc=fp32_acc, ) + if _flashinfer_allreduce_supports_trigger_completion: + kwargs["trigger_completion_at_end"] = trigger_completion_at_end + _flashinfer_comm.allreduce_fusion(**kwargs) return norm_out, residual_out diff --git a/python/sglang/srt/layers/fused_qk_norm.py b/python/sglang/srt/layers/fused_qk_norm.py new file mode 100644 index 000000000000..ce4bc0e42e41 --- /dev/null +++ b/python/sglang/srt/layers/fused_qk_norm.py @@ -0,0 +1,157 @@ +"""Fused Q/K RMSNorm in a single Triton kernel launch. + +Ported from ATOM (atom/model_ops/layernorm.py). Fuses per-head Q RMSNorm +(optionally weightless) and KV RMSNorm into one kernel, halving the number +of norm kernel launches per attention layer. +""" + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _fused_qk_norm_kernel( + q_ptr, + k_ptr, + q_out_ptr, + k_out_ptr, + q_weight_ptr, + k_weight_ptr, + eps, + num_tokens, + head_dim, + q_in_stride0, + k_in_stride0, + q_out_stride0, + k_out_stride0, + num_q_heads, + num_k_heads, + Q_HAS_WEIGHT: tl.constexpr, + RBLOCK: tl.constexpr, + XBLOCK: tl.constexpr, +): + num_q_rows = num_tokens * num_q_heads + total_rows = num_tokens * (num_q_heads + num_k_heads) + + xoffset = tl.program_id(0) * XBLOCK + xindex = xoffset + tl.arange(0, XBLOCK)[:, None] + xmask = xindex < total_rows + cols = tl.arange(0, RBLOCK)[None, :] + col_mask = cols < head_dim + + is_q = xindex < num_q_rows + row_in_section = tl.where(is_q, xindex, xindex - num_q_rows) + cur_num_heads = tl.where(is_q, num_q_heads, num_k_heads) + + tokens = row_in_section // cur_num_heads + heads = row_in_section % cur_num_heads + + in_stride = tl.where(is_q, q_in_stride0, k_in_stride0) + in_bases = tokens * in_stride + heads * head_dim + + out_stride0 = tl.where(is_q, q_out_stride0, k_out_stride0) + out_bases = tokens * out_stride0 + heads * head_dim + + mask = xmask & col_mask + + if Q_HAS_WEIGHT: + qw = tl.load( + q_weight_ptr + cols, mask=col_mask, other=0.0, eviction_policy="evict_last" + ).to(tl.float32) + else: + qw = tl.full((RBLOCK,), 1.0, tl.float32) + kw = tl.load( + k_weight_ptr + cols, mask=col_mask, other=0.0, eviction_policy="evict_last" + ).to(tl.float32) + w = tl.where(is_q, qw, kw) + + x = tl.load( + q_ptr + in_bases + cols, + mask=mask & is_q, + other=0.0, + eviction_policy="evict_first", + ).to(tl.float32) + x = x + tl.load( + k_ptr + in_bases + cols, + mask=mask & ~is_q, + other=0.0, + eviction_policy="evict_first", + ).to(tl.float32) + + var = tl.sum(x * x, 1)[:, None] + rstd = tl.rsqrt(var / head_dim + eps) + + out = (x * rstd * w).to(q_out_ptr.dtype.element_ty) + tl.store( + q_out_ptr + out_bases + cols, + out, + mask=mask & is_q, + eviction_policy="evict_first", + ) + tl.store( + k_out_ptr + out_bases + cols, + out, + mask=mask & ~is_q, + eviction_policy="evict_first", + ) + + +def fused_qk_norm( + q: torch.Tensor, + k: torch.Tensor, + q_weight: Optional[torch.Tensor], + k_weight: torch.Tensor, + eps: float, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Fused Q/K RMSNorm in a single Triton kernel launch. + + Args: + q: [num_tokens, num_heads, head_dim] + k: [num_tokens, num_kv_heads, head_dim] + q_weight: [head_dim] norm weight, or None for weightless Q norm + k_weight: [head_dim] norm weight (always required) + eps: epsilon for numerical stability + + Returns: + (q_normed, k_normed) same shapes as inputs + """ + head_dim = k_weight.shape[0] + if q_weight is not None: + assert q_weight.shape[0] == head_dim + num_tokens = q.shape[0] + num_q_heads = q.shape[1] + num_k_heads = k.shape[1] + total_rows = num_tokens * (num_q_heads + num_k_heads) + RBLOCK = triton.next_power_of_2(head_dim) + + q_out = torch.empty_like(q) + k_out = torch.empty_like(k) + + XBLOCK = 2 if total_rows > 8192 else 1 + NUM_WARPS = 1 + q_weight_arg = q_weight if q_weight is not None else k_weight + _fused_qk_norm_kernel[((total_rows + XBLOCK - 1) // XBLOCK,)]( + q, + k, + q_out, + k_out, + q_weight_arg, + k_weight, + eps, + num_tokens, + head_dim, + q.stride(0), + k.stride(0), + q_out.stride(0), + k_out.stride(0), + num_q_heads, + num_k_heads, + Q_HAS_WEIGHT=q_weight is not None, + RBLOCK=RBLOCK, + XBLOCK=XBLOCK, + num_warps=NUM_WARPS, + ) + return q_out, k_out diff --git a/python/sglang/srt/layers/fused_qk_norm_rope_store.py b/python/sglang/srt/layers/fused_qk_norm_rope_store.py new file mode 100644 index 000000000000..c6499902a8b9 --- /dev/null +++ b/python/sglang/srt/layers/fused_qk_norm_rope_store.py @@ -0,0 +1,380 @@ +"""Fused Q per-head RMSNorm + KV RMSNorm + RoPE + FP8 nope quant + paged SWA store. + +Single Triton kernel replacing the 2-kernel path: + 1. fused_reduce_qk_norm_rope_swa_write (norm + RoPE) + 2. store_cache -> fused_store_cache (FP8 quant + paged scatter) + +Grid: (cdiv(M, BLOCK_SIZE_M), num_local_heads + 1). + pid_h < num_local_heads: Q head programs (split-K reduce + norm + RoPE) + pid_h == num_local_heads: KV program (norm + RoPE + FP8 quant nope + paged scatter) +""" + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz + +_fp8_fnuz = is_fp8_fnuz() + + +# --------------------------------------------------------------------------- +# Triton JIT helpers +# --------------------------------------------------------------------------- + + +@triton.jit +def _batched_rmsnorm(row, weight, n_cols, epsilon): + row_norm = tl.sum(row * row, axis=-1) + norm_factor = tl.math.rsqrt((row_norm / n_cols) + epsilon) + if weight is not None: + return row * norm_factor[:, None] * weight[None, :] + return row * norm_factor[:, None] + + +@triton.jit +def _gptj_rotate(x, mask, BM: tl.constexpr, BD: tl.constexpr, BDH: tl.constexpr): + x_rot = tl.where(mask, x, -x) + x_rot = tl.reshape(x_rot, (BM, BDH, 2)) + x_rot = tl.flip(x_rot, 2) + return tl.reshape(x_rot, (BM, BD)) + + +@triton.jit +def _batched_rope( + x_pe, cos, sin, d_pe_offs, BM: tl.constexpr, BD: tl.constexpr, BDH: tl.constexpr +): + mask = (d_pe_offs % 2 == 0)[None, :] + x_rot = _gptj_rotate(x_pe, mask, BM, BD, BDH) + return x_pe * cos + x_rot * sin + + +# --------------------------------------------------------------------------- +# Main kernel +# --------------------------------------------------------------------------- + + +@triton.jit +def _fused_qk_norm_rope_store_kernel( + q_in_ptr, + q_out_ptr, + kv_ptr, + q_norm_weight_ptr, + kv_norm_weight_ptr, + positions_ptr, + cos_ptr, + sin_ptr, + swa_cache_ptr, + swa_loc_ptr, + M, + q_in_splitk_stride, + q_in_m_stride, + q_in_d_stride, + stride_qm, + stride_qh, + stride_qd, + stride_kv_m, + stride_kv_d, + cos_stride_t, + cos_stride_d, + swa_cache_stride_page, + q_eps, + kv_eps, + BLOCK_SIZE_M: tl.constexpr, + HEAD_DIM: tl.constexpr, + ROPE_DIM: tl.constexpr, + NUM_LOCAL_HEADS: tl.constexpr, + NUM_SPLITK: tl.constexpr, + HAS_SWA_STORE: tl.constexpr, + DIM_NOPE: tl.constexpr, + TILE_SIZE: tl.constexpr, + NUM_NOPE_TILES: tl.constexpr, + FP8_MIN: tl.constexpr, + FP8_MAX: tl.constexpr, + BYTES_PER_TOKEN: tl.constexpr, + SWA_PAGE_SIZE: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + pid_h = tl.program_id(1).to(tl.int64) + NOPE_DIM: tl.constexpr = HEAD_DIM - ROPE_DIM + NUM_PE_CHUNKS: tl.constexpr = HEAD_DIM // ROPE_DIM + + m_offs = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + m_mask = m_offs < M + + offs_d_full = tl.arange(0, HEAD_DIM) + nope_d_mask = offs_d_full < NOPE_DIM + + d_pe_offs = tl.arange(0, ROPE_DIM).to(tl.int64) + d_cos_offs = d_pe_offs // 2 + + # ===== Q path ===== + if pid_h < NUM_LOCAL_HEADS: + head_id = pid_h.to(tl.int32) + offs_n = head_id * HEAD_DIM + offs_d_full + + splitk_offs = tl.arange(0, NUM_SPLITK).to(tl.int64) + q_ptrs = ( + q_in_ptr + + splitk_offs[:, None, None] * q_in_splitk_stride + + m_offs[None, :, None] * q_in_m_stride + + offs_n[None, None, :] * q_in_d_stride + ) + q_tile = tl.load(q_ptrs, mask=m_mask[None, :, None], other=0.0).to(tl.float32) + q_acc = tl.sum(q_tile, axis=0) + + if q_norm_weight_ptr is not None: + w_q = tl.load(q_norm_weight_ptr + offs_d_full).to(tl.float32) + else: + w_q = None + q_normed = _batched_rmsnorm(q_acc, w_q, HEAD_DIM, q_eps) + + q_base = q_out_ptr + m_offs[:, None] * stride_qm + pid_h * stride_qh + tl.store( + q_base + offs_d_full[None, :] * stride_qd, + q_normed.to(q_out_ptr.dtype.element_ty), + mask=m_mask[:, None] & nope_d_mask[None, :], + ) + + q_pe = tl.where((offs_d_full >= NOPE_DIM)[None, :], q_normed, 0.0) + q_pe = tl.reshape(q_pe, (BLOCK_SIZE_M, NUM_PE_CHUNKS, ROPE_DIM)) + q_pe = tl.sum(q_pe, axis=1) + + pos = tl.load(positions_ptr + m_offs, mask=m_mask, other=0) + cos_o = pos[:, None] * cos_stride_t + d_cos_offs[None, :] * cos_stride_d + cos = tl.load(cos_ptr + cos_o, mask=m_mask[:, None], other=0) + sin = tl.load(sin_ptr + cos_o, mask=m_mask[:, None], other=0) + + q_pe = _batched_rope( + q_pe, cos, sin, d_pe_offs, BLOCK_SIZE_M, ROPE_DIM, ROPE_DIM // 2 + ) + tl.store( + q_base + (NOPE_DIM + d_pe_offs[None, :]) * stride_qd, + q_pe.to(q_out_ptr.dtype.element_ty), + mask=m_mask[:, None], + ) + return + + # ===== KV path ===== + src_id = m_offs.to(tl.int32) + src_mask = m_mask + + pos = tl.load(positions_ptr + src_id, mask=src_mask, other=0) + cos_o = pos[:, None] * cos_stride_t + d_cos_offs[None, :] * cos_stride_d + cos = tl.load(cos_ptr + cos_o, mask=src_mask[:, None], other=0) + sin = tl.load(sin_ptr + cos_o, mask=src_mask[:, None], other=0) + + kv_base = kv_ptr + src_id[:, None].to(tl.int64) * stride_kv_m + kv_full_ptrs = kv_base + offs_d_full[None, :] * stride_kv_d + + kv_full = tl.load(kv_full_ptrs, mask=src_mask[:, None], other=0.0).to(tl.float32) + + if kv_norm_weight_ptr is not None: + w_kv = tl.load(kv_norm_weight_ptr + offs_d_full).to(tl.float32) + else: + w_kv = None + kv_normed = _batched_rmsnorm(kv_full, w_kv, HEAD_DIM, kv_eps) + + tl.store( + kv_full_ptrs, + kv_normed.to(kv_ptr.dtype.element_ty), + mask=src_mask[:, None] & nope_d_mask[None, :], + ) + + kv_pe = tl.where((offs_d_full >= NOPE_DIM)[None, :], kv_normed, 0.0) + kv_pe = tl.reshape(kv_pe, (BLOCK_SIZE_M, NUM_PE_CHUNKS, ROPE_DIM)) + kv_pe = tl.sum(kv_pe, axis=1) + + kv_pe = _batched_rope( + kv_pe, cos, sin, d_pe_offs, BLOCK_SIZE_M, ROPE_DIM, ROPE_DIM // 2 + ) + tl.store( + kv_base + (NOPE_DIM + d_pe_offs[None, :]) * stride_kv_d, + kv_pe.to(kv_ptr.dtype.element_ty), + mask=src_mask[:, None], + ) + + # ===== Paged SWA store: FP8 quant nope + BF16 rope + scales ===== + # Layout within a page (matches fused_store_flashmla_cache CUDA kernel): + # Values region: [page_size tokens * 576 bytes/token] + # Per token: 448 bytes FP8 nope + 128 bytes BF16 rope + # Scales region: [page_size tokens * 8 bytes/token] + # Per token: 7 scale bytes + 1 pad byte + # Total per page before padding: page_size * 584 + VALUE_STRIDE: tl.constexpr = DIM_NOPE + ROPE_DIM * 2 + SCALE_BYTES: tl.constexpr = NUM_NOPE_TILES + 1 + + if HAS_SWA_STORE: + loc = tl.load(swa_loc_ptr + src_id, mask=src_mask, other=0) + page_id = loc // SWA_PAGE_SIZE + page_off = loc % SWA_PAGE_SIZE + page_base = page_id.to(tl.int64) * swa_cache_stride_page + value_base = page_base + page_off.to(tl.int64) * VALUE_STRIDE + scale_base = ( + page_base + + SWA_PAGE_SIZE * VALUE_STRIDE + + page_off.to(tl.int64) * SCALE_BYTES + ) + + EPS: tl.constexpr = 1e-8 + nope_tile_offs = tl.arange(0, TILE_SIZE) + + for tile_i in tl.static_range(NUM_NOPE_TILES): + tile_start = tile_i * TILE_SIZE + tile_data = tl.load( + kv_ptr + + src_id[:, None].to(tl.int64) * stride_kv_m + + (tile_start + nope_tile_offs[None, :]) * stride_kv_d, + mask=src_mask[:, None], + other=0.0, + ).to(tl.float32) + + abs_max = tl.max(tl.abs(tile_data), axis=-1) + abs_max_c = tl.maximum(abs_max, EPS) + scale_f = abs_max_c / FP8_MAX + log2_s = tl.log2(scale_f) + ceil_log2 = tl.math.ceil(log2_s) + scale_pow2 = tl.exp2(ceil_log2) + inv_scale = 1.0 / scale_pow2 + x_scaled = tile_data * inv_scale[:, None] + x_fp8 = tl.clamp(x_scaled, FP8_MIN, FP8_MAX) + + x_fp8_cast = x_fp8.to(tl.float8e4nv) + x_fp8_bytes = x_fp8_cast.to(tl.uint8, bitcast=True) + fp8_byte_offs = value_base[:, None] + tile_start + nope_tile_offs[None, :] + tl.store( + swa_cache_ptr + fp8_byte_offs, + x_fp8_bytes, + mask=src_mask[:, None], + ) + + scale_uint8 = (ceil_log2.to(tl.int32) + 127).to(tl.uint8) + tl.store( + swa_cache_ptr + scale_base + tile_i, + scale_uint8, + mask=src_mask, + ) + + rope_data = kv_pe.to(tl.bfloat16) + rope_offs = tl.arange(0, ROPE_DIM) + rope_byte_base = value_base[:, None] + DIM_NOPE + rope_offs[None, :] * 2 + rope_data_as_i16 = rope_data.to(tl.int16, bitcast=True) + lo = (rope_data_as_i16 & 0xFF).to(tl.uint8) + hi = ((rope_data_as_i16 >> 8) & 0xFF).to(tl.uint8) + tl.store(swa_cache_ptr + rope_byte_base, lo, mask=src_mask[:, None]) + tl.store(swa_cache_ptr + rope_byte_base + 1, hi, mask=src_mask[:, None]) + + +# --------------------------------------------------------------------------- +# Python wrapper +# --------------------------------------------------------------------------- + + +def fused_qk_norm_rope_swa_store( + q: torch.Tensor, + kv: torch.Tensor, + q_norm_weight: Optional[torch.Tensor], + kv_norm_weight: Optional[torch.Tensor], + q_rms_eps: float, + kv_rms_eps: float, + rope_head_dim: int, + cos_cache: torch.Tensor, + sin_cache: torch.Tensor, + positions: torch.Tensor, + swa_cache: Optional[torch.Tensor] = None, + swa_loc: Optional[torch.Tensor] = None, + swa_page_size: int = 128, + q_out: Optional[torch.Tensor] = None, + dtype: torch.dtype = torch.bfloat16, +) -> torch.Tensor: + """Fused Q norm + KV norm + RoPE + optional FP8 paged SWA store. + + Args: + q: [M, N] or [splitk, M, N] where N = num_local_heads * head_dim + kv: [M, head_dim=512] mutated in-place (norm + RoPE) + swa_cache: paged SWA KV pool buffer [num_pages, bytes_per_page] uint8 + swa_loc: [M] int32 pre-translated paged indices + swa_page_size: tokens per SWA page (default 128) + """ + head_dim = kv.shape[1] + + if q.dim() == 3: + num_splitk, M, N = q.shape + q_in_splitk_stride = q.stride(0) + q_in_m_stride = q.stride(1) + q_in_d_stride = q.stride(2) + else: + M, N = q.shape + num_splitk = 1 + q_in_splitk_stride = 0 + q_in_m_stride = q.stride(0) + q_in_d_stride = q.stride(1) + + num_local_heads = N // head_dim + + if q_out is None: + q_out = torch.empty( + (M, num_local_heads, head_dim), dtype=dtype, device=q.device + ) + + HAS_SWA_STORE = swa_cache is not None and swa_loc is not None + + dim_nope = 448 + dim_rope = 64 + tile_size = 64 + num_nope_tiles = dim_nope // tile_size + scale_pad = 1 + bytes_per_token = dim_nope + dim_rope * 2 + num_nope_tiles + scale_pad + + if _fp8_fnuz: + fp8_info = torch.finfo(torch.float8_e4m3fnuz) + else: + fp8_info = torch.finfo(torch.float8_e4m3fn) + + BLOCK_SIZE_M = min(4, triton.next_power_of_2(M)) if M < 4 else 4 + num_warps = 4 + + grid = (triton.cdiv(M, BLOCK_SIZE_M), num_local_heads + 1) + _fused_qk_norm_rope_store_kernel[grid]( + q, + q_out, + kv, + q_norm_weight, + kv_norm_weight, + positions, + cos_cache, + sin_cache, + swa_cache if HAS_SWA_STORE else None, + swa_loc if HAS_SWA_STORE else None, + M, + q_in_splitk_stride, + q_in_m_stride, + q_in_d_stride, + q_out.stride(0), + q_out.stride(1), + q_out.stride(2), + kv.stride(0), + kv.stride(1), + cos_cache.stride(0), + cos_cache.stride(-1), + swa_cache.stride(0) if HAS_SWA_STORE else 0, + q_rms_eps, + kv_rms_eps, + BLOCK_SIZE_M=BLOCK_SIZE_M, + HEAD_DIM=head_dim, + ROPE_DIM=rope_head_dim, + NUM_LOCAL_HEADS=num_local_heads, + NUM_SPLITK=num_splitk, + HAS_SWA_STORE=HAS_SWA_STORE, + DIM_NOPE=dim_nope, + TILE_SIZE=tile_size, + NUM_NOPE_TILES=num_nope_tiles, + FP8_MIN=fp8_info.min, + FP8_MAX=fp8_info.max, + BYTES_PER_TOKEN=bytes_per_token, + SWA_PAGE_SIZE=swa_page_size, + num_warps=num_warps, + ) + return q_out diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index e9c9e7be8c71..6e4b7eee4882 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -53,7 +53,26 @@ if _is_cuda or _is_xpu or _is_musa: if _is_flashinfer_available: try: - from flashinfer.norm import layernorm + import flashinfer.norm + + from sglang.srt.utils.custom_op import register_custom_op + + def _layernorm_fake_impl( + input: torch.Tensor, + gamma: torch.Tensor, + beta: torch.Tensor, + eps: float = 1e-6, + ) -> torch.Tensor: + return torch.empty_like(input) + + @register_custom_op(fake_impl=_layernorm_fake_impl) + def layernorm( + input: torch.Tensor, + gamma: torch.Tensor, + beta: torch.Tensor, + eps: float = 1e-6, + ) -> torch.Tensor: + return flashinfer.norm.layernorm(input, gamma, beta, eps) _flashinfer_layernorm_available = True except (ImportError, AttributeError): @@ -284,6 +303,12 @@ def forward_aiter( residual: Optional[torch.Tensor] = None, post_residual_addition: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + # Fix dsv4 dp attenton issue + # the symptom is torch.AcceleratorError: HIP error: invalid configuration argument + if x.shape[0] == 0: + if residual is not None: + return x, residual + return x # Aiter's RMSNorm kernels expect 2D contiguous inputs. Keep the # already-safe layout as a zero-copy path, and only normalize strided or # higher-rank views such as Q/K slices from packed QKV projections. diff --git a/python/sglang/srt/layers/moe/cutlass_moe.py b/python/sglang/srt/layers/moe/cutlass_moe.py index fd02d6718f08..05cfe00fc6f9 100755 --- a/python/sglang/srt/layers/moe/cutlass_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_moe.py @@ -491,10 +491,12 @@ def cutlass_moe_fp4( params.to_gemm2_args(), ) del int_fp4, int_blockscale - c2 = shuffle_rows(c2, c_map, (m_a * num_topk, params.hidden_size)) - c2 = c2.view(m_a, num_topk, params.hidden_size) + if no_combine: + c2 = shuffle_rows(c2, c_map, (m_a * num_topk, params.hidden_size)) + c2 = c2.view(m_a, num_topk, params.hidden_size) return c2.to(out_dtype) - if not apply_router_weight_on_input: - c2 = c2 * topk_weights.view(m_a, num_topk, 1).to(out_dtype) - return c2.sum(dim=1).to(out_dtype) + output = torch.empty((m_a, k_a), device=device, dtype=out_dtype) + weights = topk_weights.to(out_dtype) if not apply_router_weight_on_input else None + apply_shuffle_mul_sum(c2, output, c_map, weights) + return output diff --git a/python/sglang/srt/layers/moe/ep_moe/layer.py b/python/sglang/srt/layers/moe/ep_moe/layer.py index 11c6c5e5d11e..691401230e29 100644 --- a/python/sglang/srt/layers/moe/ep_moe/layer.py +++ b/python/sglang/srt/layers/moe/ep_moe/layer.py @@ -1,13 +1,12 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional import torch from sglang.srt.compilation.piecewise_context_manager import is_in_piecewise_cuda_graph from sglang.srt.environ import envs -from sglang.srt.hardware_backend.npu.utils import FusedMoEMode, npu_format_cast from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.moe import ( get_deepep_mode, @@ -85,7 +84,7 @@ def __init__( if _use_aiter: self.deprecate_flag = True elif _is_npu: - self.deprecate_flag = False + self.deprecate_flag = True elif deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and isinstance( quant_config, Fp8Config ): @@ -203,10 +202,7 @@ def run_moe_core( from sglang.srt.layers.moe.token_dispatcher import DispatchOutputChecker - if _is_npu: - assert DispatchOutputChecker.format_is_deepep(dispatch_output) - output = self.forward_npu(dispatch_output) - elif DispatchOutputChecker.format_is_deepep_normal(dispatch_output): + if DispatchOutputChecker.format_is_deepep_normal(dispatch_output): if self.quant_config is None: raise NotImplementedError( "Unquantized DeepEP MoE currently supports low_latency mode only" @@ -269,241 +265,6 @@ def forward_cutlass_w4afp8_masked( dispatch_output=dispatch_output, ) - def forward_npu( - self, - dispatch_output: Union[DeepEPNormalDispatchOutput, DeepEPLLDispatchOutput], - ): - assert self.quant_method is not None - assert self.moe_runner_config.activation == "silu" - - from sglang.srt.hardware_backend.npu.quantization.fused_moe_method_npu import ( - npu_fused_moe_without_routing_weights_bf16, - ) - from sglang.srt.layers.moe.token_dispatcher import DispatchOutputChecker - - # NOTE: Ascend's Dispatch & Combine does not support FP16 - output_dtype = torch.bfloat16 - group_list_type = 1 - - if DispatchOutputChecker.format_is_deepep_normal(dispatch_output): - if TYPE_CHECKING: - assert isinstance(dispatch_output, DeepEPNormalDispatchOutput) - hidden_states, hidden_states_scale, _, _, num_recv_tokens_per_expert = ( - dispatch_output - ) - - group_list = torch.tensor( - num_recv_tokens_per_expert, - dtype=torch.int64, - device=hidden_states.device, - ) - - if self.w13_weight.dtype == torch.bfloat16: - hidden_states = npu_fused_moe_without_routing_weights_bf16( - self, hidden_states, group_list_type, group_list, output_dtype - ) - else: - hidden_states = self.quant_method.apply_without_routing_weights( - self, - hidden_states, - hidden_states_scale, - group_list_type, - group_list, - output_dtype, - ) - elif DispatchOutputChecker.format_is_deepep_ll(dispatch_output): - if TYPE_CHECKING: - assert isinstance(dispatch_output, DeepEPLLDispatchOutput) - ( - hidden_states, - hidden_states_scale, - topk_ids, - topk_weights, - group_list, - _, - ) = dispatch_output - - group_list = group_list.to(torch.int64) - - if self.w13_weight.dtype == torch.bfloat16: - hidden_states = npu_fused_moe_without_routing_weights_bf16( - self, hidden_states, group_list_type, group_list, output_dtype - ) - else: - hidden_states = self.quant_method.apply_without_routing_weights( - self, - hidden_states, - hidden_states_scale, - group_list_type, - group_list, - output_dtype, - ) - else: - raise ValueError(f"Not Supported DeepEP format {dispatch_output.format}") - - return hidden_states - - -class NpuFuseEPMoE(DeepEPMoE): - def __init__( - self, - num_experts: int, - top_k: int, - hidden_size: int, - intermediate_size: int, - layer_id: int, - num_fused_shared_experts: int = 0, - params_dtype: Optional[torch.dtype] = None, - quant_config: Optional[QuantizationConfig] = None, - prefix: str = "", - activation: str = "silu", - routed_scaling_factor: Optional[float] = None, - **kwargs, - ): - super().__init__( - num_experts=num_experts, - top_k=top_k, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - layer_id=layer_id, - num_fused_shared_experts=num_fused_shared_experts, - params_dtype=params_dtype, - quant_config=quant_config, - prefix=prefix, - activation=activation, - routed_scaling_factor=routed_scaling_factor, - **kwargs, - ) - - self.quant_method.process_weights_after_loading = ( - self._process_weights_after_loading - ) - - def forward( - self, - hidden_states: torch.Tensor, - topk_output: TopKOutput, - forward_shared_experts=None, - alt_stream=None, - disable_sbo=False, - ): - return self.dispatcher.dispatch( - hidden_states=hidden_states, - topk_output=topk_output, - gmm1_permuted_weight=self.w13_weight, - gmm1_permuted_weight_scale=self.w13_weight_scale, - gmm2_weight=self.w2_weight, - gmm2_weight_scale=self.w2_weight_scale, - ).hidden_state - - def permute_w13_weight_scale(self, w: torch.Tensor, tile_n: int): - if tile_n % 2 != 0: - raise ValueError(f"tile_n must be even, got {tile_n}") - - *dims, n = w.shape - if n % tile_n != 0: - raise ValueError(f"Last dimension {n} must be divisible by tile_n {tile_n}") - - w_reshaped = w.reshape(*dims, 2, n // tile_n, tile_n // 2) - - # Permute the last two dimensions. - perm_order = list(range(len(dims))) + [-2, -3, -1] - w_permuted = w_reshaped.permute(perm_order) - - return w_permuted.reshape(*dims, n) - - def reshape_w13_weight(self, weight: torch.Tensor, dim: int, chunk_size: int = 64): - # Achieving greater computing power through reshape on Ascend. - original_shape = weight.shape - if dim < 0: - dim += len(original_shape) - - if original_shape[dim] % (2 * chunk_size) != 0: - raise ValueError( - f"Dimension {dim} size {original_shape[dim]} must be divisible by {2 * chunk_size}" - ) - - new_shape = ( - *original_shape[:dim], - 2, - original_shape[dim] // (2 * chunk_size), - chunk_size, - *original_shape[dim + 1 :], - ) - - weight = weight.view(new_shape) - weight = weight.transpose(dim, dim + 1).contiguous() - - return weight.view(*original_shape[:dim], -1, *original_shape[dim + 1 :]) - - def release_weight_cache(self, weight: torch.Tensor): - # .contiguous() introduces additional memory overhead and needs to be released using resize_(0) - origin_weight = weight.data.transpose(1, 2) - new_weight = origin_weight.contiguous() - origin_weight.untyped_storage().resize_(0) - return new_weight - - def scale_from_float_to_int64(self, scale): - import numpy as np - - scale = torch.from_numpy( - np.frombuffer( - scale.cpu().to(torch.float32).numpy().tobytes(), dtype=np.int32 - ).astype(np.int64) - ).to(scale.device) - return torch.nn.Parameter(scale, requires_grad=False) - - def _process_weights_after_loading(self, layer: torch.nn.Module) -> None: - if ( - envs.SGLANG_NPU_FUSED_MOE_MODE.get() - == FusedMoEMode.DISPATCH_FFN_COMBINE.value - ): - w13_weight = self.release_weight_cache(layer.w13_weight) - layer.w13_weight.data = npu_format_cast(w13_weight) - w2_weight = self.release_weight_cache(layer.w2_weight) - layer.w2_weight.data = npu_format_cast(w2_weight) - - layer.w13_weight_scale.data = layer.w13_weight_scale.data.view( - layer.w13_weight_scale.data.shape[0], -1 - ) - w2_scale = layer.w2_weight_scale.data.squeeze(-1).contiguous() - layer.w2_weight_scale = torch.nn.Parameter( - w2_scale.to(torch.float32), requires_grad=False - ) - - layer.w13_weight_scale = self.scale_from_float_to_int64( - layer.w13_weight_scale.data - ) - layer.w2_weight_scale = self.scale_from_float_to_int64( - layer.w2_weight_scale.data - ) - else: - cpu_w13 = layer.w13_weight.data.transpose(1, 2).cpu() - layer.w13_weight.data = self.reshape_w13_weight(cpu_w13, -1).npu() - w13_scale = layer.w13_weight_scale.data.squeeze(-1).contiguous() - w13_scale = self.permute_w13_weight_scale(w13_scale, 128) - layer.w13_weight_scale = torch.nn.Parameter( - w13_scale.to(torch.float32), requires_grad=False - ) - layer.w13_weight.data = npu_format_cast(layer.w13_weight.data) - layer.w2_weight.data = npu_format_cast(layer.w2_weight.data) - - w2_scale = layer.w2_weight_scale.data.squeeze(-1).contiguous() - layer.w2_weight_scale = torch.nn.Parameter( - w2_scale.to(torch.float32), requires_grad=False - ) - - if hasattr(layer, "w13_weight_offset"): - layer.w13_weight_offset = torch.nn.Parameter( - layer.w13_weight_offset.data.squeeze(-1).contiguous(), - requires_grad=False, - ) - if hasattr(layer, "w2_weight_offset"): - layer.w2_weight_offset = torch.nn.Parameter( - layer.w2_weight_offset.data.squeeze(-1).contiguous(), - requires_grad=False, - ) - def get_moe_impl_class(quant_config: Optional[QuantizationConfig]): # [TODO] kk, temporary solution @@ -515,6 +276,8 @@ def get_moe_impl_class(quant_config: Optional[QuantizationConfig]): ): return DeepEPMoE if get_moe_a2a_backend().is_ascend_fuseep(): - return NpuFuseEPMoE + # ascend_fuseep bypasses dispatch/combine inside FusedMoE.forward + # (see forward_fuseep in hardware_backend/npu/moe/fuseep.py). + return FusedMoE return FusedMoE diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 3c7c141b4784..04831e08e3ca 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -83,7 +83,14 @@ def create_moe_dispatcher(moe_runner_config: MoeRunnerConfig) -> BaseDispatcher: a2a_backend = get_moe_a2a_backend() - if a2a_backend.is_none() or a2a_backend.is_megamoe(): + if ( + a2a_backend.is_none() + or a2a_backend.is_megamoe() + or a2a_backend.is_ascend_fuseep() + ): + # ascend_fuseep bypasses the dispatcher abstraction (see + # forward_fuseep in hardware_backend/npu/moe/fuseep.py); a + # StandardDispatcher is created but never invoked. return StandardDispatcher(moe_runner_config) elif ( a2a_backend.is_deepep() @@ -107,19 +114,6 @@ def create_moe_dispatcher(moe_runner_config: MoeRunnerConfig) -> BaseDispatcher: async_finish=True, return_recv_hook=True, ) - elif a2a_backend.is_ascend_fuseep(): - from sglang.srt.layers.moe.token_dispatcher import NpuFuseEPDispatcher - - return NpuFuseEPDispatcher( - group=get_tp_group().device_group, - router_topk=moe_runner_config.top_k, - permute_fusion=True, - num_experts=moe_runner_config.num_experts, - num_local_experts=moe_runner_config.num_local_experts, - hidden_size=moe_runner_config.hidden_size, - params_dtype=moe_runner_config.params_dtype, - ) - elif a2a_backend.is_flashinfer(): return FlashinferDispatcher( group=get_tp_group().device_group, @@ -308,6 +302,7 @@ def __init__( self.quant_method.create_moe_runner(self, self.moe_runner_config) self.dispatcher = create_moe_dispatcher(self.moe_runner_config) + self._use_ascend_fuseep = get_moe_a2a_backend().is_ascend_fuseep() if ( get_moe_runner_backend().is_flashinfer_trtllm_routed() @@ -1058,6 +1053,10 @@ def weight_loader_fused( ) def forward(self, hidden_states: torch.Tensor, topk_output: TopKOutput): + if self._use_ascend_fuseep: + from sglang.srt.hardware_backend.npu.moe.fuseep import forward_fuseep + + return forward_fuseep(self, hidden_states, topk_output) if is_in_piecewise_cuda_graph(): if TopKOutputChecker.format_is_standard(topk_output): return moe_forward_piecewise_cuda_graph_impl( diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index 959880f84f83..1e13881dc2db 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -7,6 +7,9 @@ from torch import nn from sglang.srt.environ import envs +from sglang.srt.eplb.expert_distribution import ( + get_global_expert_distribution_recorder, +) from sglang.srt.eplb.expert_location_dispatch import ( ExpertLocationDispatchInfo, topk_ids_logical_to_physical, @@ -32,6 +35,20 @@ def __init__( apply_routed_scaling_factor_on_output=False, ): super().__init__() + self.layer_id = None + from sglang.srt.server_args import get_global_server_args + + self.enable_deepep_waterfill = ( + num_fused_shared_experts > 0 + and get_global_server_args().enable_deepep_waterfill + ) + self.deepep_waterfill_balancer = None + + if self.enable_deepep_waterfill: + # Waterfill appends the shared expert after EPLB maps routed IDs. + topk -= num_fused_shared_experts + num_fused_shared_experts = 0 + self.num_experts = num_experts self.topk = topk self.routed_scaling_factor = routed_scaling_factor @@ -67,7 +84,21 @@ def empty_topk_output(self, device: torch.device): topk_weights = torch.empty((0, topk), dtype=torch.float32, device=device) topk_ids = torch.full((0, topk), -1, dtype=torch.int32, device=device) router_logits = torch.empty((0, topk), dtype=torch.float32, device=device) - return StandardTopKOutput(topk_weights, topk_ids, router_logits) + return self._apply_deepep_waterfill( + StandardTopKOutput(topk_weights, topk_ids, router_logits), + num_tokens=0, + ) + + def _apply_deepep_waterfill( + self, topk_output: StandardTopKOutput, num_tokens: int + ) -> StandardTopKOutput: + if self.enable_deepep_waterfill and self.deepep_waterfill_balancer is None: + raise RuntimeError( + "DeepEP waterfill HashTopK must be prepared by ModelRunner before forward." + ) + if self.deepep_waterfill_balancer is None: + return topk_output + return self.deepep_waterfill_balancer.expand_topk(topk_output, num_tokens) def _forward_torch( self, router_logits: torch.Tensor, input_ids: torch.Tensor @@ -145,7 +176,8 @@ def forward( topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info) _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids) topk_output = StandardTopKOutput( topk_weights=topk_weights, topk_ids=topk_ids, router_logits=router_logits ) - return topk_output + return self._apply_deepep_waterfill(topk_output, hidden_states.shape[0]) diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py index 0e4ab204c147..ed402a283ed7 100644 --- a/python/sglang/srt/layers/moe/moe_runner/aiter.py +++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py @@ -56,6 +56,7 @@ class AiterMoeQuantInfo(MoeQuantInfo): doweight_stage1: bool = False hidden_pad: int = 0 intermediate_pad: int = 0 + swiglu_limit: float = 0.0 @dataclass @@ -116,6 +117,7 @@ def run( return AiterRunnerOutput(hidden_states=runner_input.hidden_states) from aiter.fused_moe import fused_moe + from aiter.ops.flydsl.moe_common import GateMode a1_scale = ( runner_input.a1_scale @@ -128,6 +130,9 @@ def run( extra["num_local_tokens"] = runner_input.num_local_tokens if runner_input.output_dtype is not None: extra["dtype"] = runner_input.output_dtype + if quant_info.swiglu_limit > 0: + extra["gate_mode"] = GateMode.INTERLEAVE.value + extra["swiglu_limit"] = quant_info.swiglu_limit output = fused_moe( hidden_states=runner_input.hidden_states, diff --git a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py index f313668680af..bad52b9597c3 100644 --- a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py +++ b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py @@ -863,8 +863,11 @@ def _varlen_deep_gemm_silu_mul_quant( dtype=torch.float8_e4m3fn, ) - if envs.SGLANG_OPT_USE_JIT_EP_ACTIVATION.get(): - assert N % 4 == 0 and G % 4 == 0 + use_jit_ep_activation = envs.SGLANG_OPT_USE_JIT_EP_ACTIVATION.get() + if N % 4 != 0 or G % 4 != 0: + use_jit_ep_activation = False + + if use_jit_ep_activation: packed_ue8m0 = deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 down_input_scale = torch.empty( (E, G // 4, N) if packed_ue8m0 else (E, N, G), diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 439840ad14d1..4e65c2a67fed 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -698,11 +698,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( assert TopKOutputChecker.format_is_bypassed(topk_output) output = trtllm_fp8_block_scale_moe_wrapper( - routing_logits=( - router_logits.to(torch.float32) - if routing_method_type == RoutingMethodType.DeepSeekV3 - else router_logits - ), + routing_logits=router_logits, routing_bias=correction_bias, hidden_states=a_q, hidden_states_scale=a_sf_t, @@ -758,11 +754,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( # during torch.compile for piecewise cuda graph. # Use custom op wrapper for torch.compile compatibility. - # The DeepSeekV3 routing method requires float32 router logits. - if routing_method_type == RoutingMethodType.DeepSeekV3: - router_logits = router_logits.to(torch.float32) - else: - router_logits = router_logits.to(torch.bfloat16) + router_logits = router_logits.to(torch.bfloat16) output = trtllm_fp8_per_tensor_scale_moe_wrapper( routing_logits=router_logits, @@ -977,10 +969,6 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( topk_config = topk_output.topk_config routing_method_type = quant_info.routing_method_type - # DeepSeekV3 style routing requires float32 router logits - if routing_method_type == RoutingMethodType.DeepSeekV3: - router_logits = router_logits.to(torch.float32) - correction_bias = ( None if topk_config.correction_bias is None diff --git a/python/sglang/srt/layers/moe/token_dispatcher/__init__.py b/python/sglang/srt/layers/moe/token_dispatcher/__init__.py index cb69096603de..f1ebac970836 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/__init__.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/__init__.py @@ -20,7 +20,6 @@ FlashinferDispatcher, FlashinferDispatchOutput, ) -from sglang.srt.layers.moe.token_dispatcher.fuseep import NpuFuseEPDispatcher from sglang.srt.layers.moe.token_dispatcher.mooncake import ( MooncakeCombineInput, MooncakeDispatchOutput, @@ -75,5 +74,4 @@ "DeepEPLLDispatchOutput", "DeepEPLLCombineInput", "DeepEPNormalCombineInput", - "NpuFuseEPDispatcher", ] diff --git a/python/sglang/srt/layers/moe/token_dispatcher/fuseep.py b/python/sglang/srt/layers/moe/token_dispatcher/fuseep.py deleted file mode 100644 index c33c337e2882..000000000000 --- a/python/sglang/srt/layers/moe/token_dispatcher/fuseep.py +++ /dev/null @@ -1,98 +0,0 @@ -from __future__ import annotations - -import logging -from typing import NamedTuple - -import torch - -from sglang.srt.environ import envs -from sglang.srt.layers.moe.token_dispatcher.base import ( - BaseDispatcher, - CombineInput, - CombineInputFormat, - DispatchOutput, - DispatchOutputFormat, -) -from sglang.srt.layers.moe.token_dispatcher.deepep import DeepEPBuffer -from sglang.srt.layers.moe.topk import TopKOutput -from sglang.srt.layers.moe.utils import DeepEPMode - -logger = logging.getLogger(__name__) - - -class FuseEPDispatchOutput(NamedTuple): - """DeepEP low latency dispatch output.""" - - hidden_state: torch.Tensor - - @property - def format(self) -> DispatchOutputFormat: - return DispatchOutputFormat.DEEPEP_LL - - -class FuseEPCombineInput(NamedTuple): - """DeepEP low latency combine input.""" - - hidden_state: torch.Tensor - - @property - def format(self) -> CombineInputFormat: - return CombineInputFormat.DEEPEP_LL - - -class NpuFuseEPDispatcher(BaseDispatcher): - def __init__( - self, - group: torch.distributed.ProcessGroup, - router_topk: int, - permute_fusion: bool = False, - num_experts: int = None, - num_local_experts: int = None, - hidden_size: int = None, - params_dtype: torch.dtype = None, - deepep_mode: DeepEPMode = DeepEPMode.LOW_LATENCY, - ): - self.group = group - self.router_topk = router_topk - self.permute_fusion = permute_fusion - self.num_experts = num_experts - self.num_local_experts = num_local_experts - self.hidden_size = hidden_size - self.params_dtype = params_dtype - self.deepep_mode = deepep_mode - - self.params_bytes = 2 - self.num_max_dispatch_tokens_per_rank = ( - envs.SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() - ) - - def dispatch( - self, hidden_states: torch.Tensor, topk_output: TopKOutput, **kwargs - ) -> DispatchOutput: - hidden_states, _ = self._get_buffer().fused_deep_moe( - hidden_states, - topk_idx=topk_output.topk_ids, - topk_weights=topk_output.topk_weights, - gmm1_permuted_weight=kwargs["gmm1_permuted_weight"], - gmm1_permuted_weight_scale=kwargs["gmm1_permuted_weight_scale"], - gmm2_weight=kwargs["gmm2_weight"], - gmm2_weight_scale=kwargs["gmm2_weight_scale"], - num_max_dispatch_tokens_per_rank=self.num_max_dispatch_tokens_per_rank, - num_experts=self.num_experts, - fuse_mode=envs.SGLANG_NPU_FUSED_MOE_MODE.get(), - ) - return FuseEPDispatchOutput(hidden_states) - - def combine(self, combine_input: CombineInput, **kwargs) -> torch.Tensor: - pass - - def _get_buffer(self): - DeepEPBuffer.set_dispatch_mode_as_low_latency() - return DeepEPBuffer.get_deepep_buffer( - self.group, - self.hidden_size, - self.params_bytes, - self.deepep_mode, - self.num_max_dispatch_tokens_per_rank, - self.num_experts, - ) diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index ca716dc331c2..d9127143ae93 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -339,17 +339,12 @@ def __init__( assert num_expert_group is not None and topk_group is not None self.layer_id = layer_id - if num_fused_shared_experts > 0: - from sglang.srt.server_args import get_global_server_args + from sglang.srt.server_args import get_global_server_args - try: - self.enable_deepep_waterfill = ( - get_global_server_args().enable_deepep_waterfill - ) - except ValueError: - self.enable_deepep_waterfill = False - else: - self.enable_deepep_waterfill = False + self.enable_deepep_waterfill = ( + num_fused_shared_experts > 0 + and get_global_server_args().enable_deepep_waterfill + ) self.deepep_waterfill_balancer = None if self.enable_deepep_waterfill: @@ -903,20 +898,46 @@ def biased_topk_jit_kernel_impl( ): assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" - from sglang.jit_kernel.moe_fused_gate import moe_fused_gate + if _use_aiter and scoring_func == "sqrtsoftplus" and num_fused_shared_experts == 0: + from aiter import topk_gating - topk_weights, topk_ids = moe_fused_gate( - gating_output, - correction_bias, - topk=topk, - scoring_func=scoring_func, - num_fused_shared_experts=num_fused_shared_experts, - renormalize=renormalize, - routed_scaling_factor=routed_scaling_factor, - apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, - ) - topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32) - return topk_weights, topk_ids + num_tokens = gating_output.shape[0] + topk_weights = torch.empty( + (num_tokens, topk), dtype=torch.float32, device=gating_output.device + ) + topk_ids = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + + topk_gating( + topk_weights, + topk_ids, + gating_output, + correction_bias, + renormalize, + routed_scaling_factor, + score_func="sqrtsoftplus", + ) + + return topk_weights, topk_ids + + else: + from sglang.jit_kernel.moe_fused_gate import moe_fused_gate + + topk_weights, topk_ids = moe_fused_gate( + gating_output, + correction_bias, + topk=topk, + scoring_func=scoring_func, + num_fused_shared_experts=num_fused_shared_experts, + renormalize=renormalize, + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to( + torch.int32 + ) + return topk_weights, topk_ids @torch.compile(dynamic=True, backend=get_compiler_backend(), disable=_is_npu) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py index 81056a17e03d..e9be6db6e187 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py @@ -1023,7 +1023,6 @@ def apply( layer input. See LinearMethodBase for param details """ - scheme = layer.scheme if scheme is None: raise ValueError("A scheme must be defined for each layer") diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16_moe.py index 645807e84382..60e4ef5d631b 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16_moe.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16_moe.py @@ -17,7 +17,10 @@ CompressedTensorsMoEScheme, ) from sglang.srt.layers.quantization.gptq import gptq_marlin_moe_repack -from sglang.srt.layers.quantization.marlin_utils import marlin_moe_permute_scales +from sglang.srt.layers.quantization.marlin_utils import ( + marlin_make_workspace, + marlin_moe_permute_scales, +) from sglang.srt.layers.quantization.utils import replace_parameter from sglang.srt.utils import get_bool_env_var, is_cuda, is_hip, set_weight_attrs @@ -334,6 +337,7 @@ def replace_tensor(name, new_t): ) replace_tensor("w2_weight_scale", marlin_w2_scales) + layer.workspace = marlin_make_workspace(layer.w13_weight_packed.device, 4) layer.is_marlin_converted = True def restore_weights_before_loading(self, layer: torch.nn.Module): @@ -419,6 +423,7 @@ def apply_weights( num_bits=self.num_bits, is_k_full=self.is_k_full, routed_scaling_factor=self.moe_runner_config.routed_scaling_factor, + workspace=layer.workspace, ) return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 78d6666464d5..3fe0b18848de 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -125,8 +125,11 @@ def _require_fp4_dtype(): if _use_aiter or _use_hip_int4: - from aiter.ops.shuffle import shuffle_weight - from aiter.utility.fp4_utils import e8m0_shuffle + from aiter.ops.shuffle import ( + shuffle_scale_a16w4, + shuffle_weight, + shuffle_weight_a16w4, + ) if _use_aiter: from sglang.srt.layers.quantization.fp8_utils import ( @@ -1217,8 +1220,10 @@ def process_weights_after_loading_block_quant(self, layer: Module) -> None: for scale_name in ("w13_weight_scale_inv", "w2_weight_scale_inv"): scale = getattr(layer, scale_name) num_experts, num_rows, _ = scale.shape - scale.data = e8m0_shuffle(scale.view(num_experts * num_rows, -1)).view( - num_experts, num_rows, -1 + # a8w4: aiter flydsl scale layout + is_w13_scale = scale_name == "w13_weight_scale_inv" + scale.data = shuffle_scale_a16w4( + scale.view(num_experts * num_rows, -1), num_experts, is_w13_scale ) layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype) @@ -1226,11 +1231,12 @@ def process_weights_after_loading_block_quant(self, layer: Module) -> None: is_shuffled = _is_shuffle_moe_mxfp4 if is_shuffled: - layer.w13_weight.data = shuffle_weight( - layer.w13_weight.contiguous(), (16, 16) + # a8w4: aiter flydsl weight layout + layer.w13_weight.data = shuffle_weight_a16w4( + layer.w13_weight.contiguous(), 16, True ) - layer.w2_weight.data = shuffle_weight( - layer.w2_weight.contiguous(), (16, 16) + layer.w2_weight.data = shuffle_weight_a16w4( + layer.w2_weight.contiguous(), 16, False ) layer.w13_weight.is_shuffled = is_shuffled layer.w2_weight.is_shuffled = is_shuffled @@ -2075,6 +2081,7 @@ def maybe_get_hip_aiter_quant_info( w13_scale=w13_scale, w2_scale=w2_scale, expert_mask=layer.dispatcher.expert_mask_gpu if _use_aiter else None, + swiglu_limit=self.moe_runner_config.swiglu_limit or 0.0, ) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index e4573987eb59..806c64f71f0d 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -62,6 +62,7 @@ is_cuda, is_sm120_supported, next_power_of_2, + round_up, ) from sglang.srt.utils.custom_op import register_custom_op from sglang.srt.utils.patch_torch import register_fake_if_exists @@ -970,6 +971,43 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: ) layer.fc1_input_dequant = Parameter(input_scale, requires_grad=False) + # flashinfer_cutlass kernel requires intermediate_size to be a + # multiple of 16. Pad weight tensors with zeros after loading. + # For gated activations (swiglu), w13 is [Up, Gate] concatenated + # along dim 1 — we must split, pad each half separately, and + # re-concat so the kernel's half-split stays aligned. + num_shards = 2 if layer.moe_runner_config.is_gated else 1 + isp = layer.w13_weight.shape[1] // num_shards + if isp % 16 != 0: + pad_amount = round_up(isp, 16) - isp + w13_data = layer.w13_weight.data + if num_shards == 2: + up_weight = w13_data[:, :isp, :] + gate_weight = w13_data[:, isp:, :] + layer.w13_weight = Parameter( + torch.cat( + [ + torch.nn.functional.pad( + up_weight, (0, 0, 0, pad_amount) + ), + torch.nn.functional.pad( + gate_weight, (0, 0, 0, pad_amount) + ), + ], + dim=1, + ), + requires_grad=False, + ) + else: + layer.w13_weight = Parameter( + torch.nn.functional.pad(w13_data, (0, 0, 0, pad_amount)), + requires_grad=False, + ) + layer.w2_weight = Parameter( + torch.nn.functional.pad(layer.w2_weight.data, (0, pad_amount)), + requires_grad=False, + ) + def create_moe_runner( self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig ): diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index 0302ae0646c9..99d0a2468ea1 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -44,6 +44,7 @@ if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher import ( CombineInput, + DispatchOutput, StandardDispatchOutput, ) @@ -637,10 +638,14 @@ def forward_xpu( def forward_npu( self, layer: torch.nn.Module, - dispatch_output: StandardDispatchOutput, + dispatch_output: "DispatchOutput", ) -> CombineInput: from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + from sglang.srt.layers.moe.token_dispatcher.base import DispatchOutputChecker + + if DispatchOutputChecker.format_is_deepep(dispatch_output): + return self._forward_npu_deepep(layer, dispatch_output) # x.shape = [B*S, H] x = dispatch_output.hidden_states @@ -719,6 +724,46 @@ def forward_npu( return StandardCombineInput(hidden_states=final_hidden_states) + def _forward_npu_deepep( + self, + layer: torch.nn.Module, + dispatch_output: "DispatchOutput", + ) -> CombineInput: + from sglang.srt.hardware_backend.npu.quantization.fused_moe_method_npu import ( + npu_fused_moe_without_routing_weights_bf16, + ) + from sglang.srt.layers.moe.token_dispatcher import ( + DeepEPLLCombineInput, + DeepEPNormalCombineInput, + ) + from sglang.srt.layers.moe.token_dispatcher.base import DispatchOutputChecker + + # NOTE: Ascend's Dispatch & Combine does not support FP16 + output_dtype = torch.bfloat16 + group_list_type = 1 + + if DispatchOutputChecker.format_is_deepep_normal(dispatch_output): + hidden_states, _, _, _, num_recv_tokens_per_expert = dispatch_output + group_list = torch.tensor( + num_recv_tokens_per_expert, + dtype=torch.int64, + device=hidden_states.device, + ) + combine_cls = DeepEPNormalCombineInput + else: + hidden_states, _, _, _, group_list, _ = dispatch_output + group_list = group_list.to(torch.int64) + combine_cls = DeepEPLLCombineInput + + hidden_states = npu_fused_moe_without_routing_weights_bf16( + layer, hidden_states, group_list_type, group_list, output_dtype + ) + return combine_cls( + hidden_states=hidden_states, + topk_ids=dispatch_output.topk_ids, + topk_weights=dispatch_output.topk_weights, + ) + def forward_tpu(self, *args, **kwargs) -> CombineInput: raise NotImplementedError("The TPU backend currently does not support MoE.") diff --git a/python/sglang/srt/layers/quantization/utils.py b/python/sglang/srt/layers/quantization/utils.py index 2c54d901ebd3..99e3218cfd3f 100644 --- a/python/sglang/srt/layers/quantization/utils.py +++ b/python/sglang/srt/layers/quantization/utils.py @@ -50,6 +50,8 @@ def _module_path_match(ignored: str, prefix: str) -> bool: # match `mlp.gate_up_proj`. Needed for quant configs (e.g. Qwen3.6-FP8) # whose `modules_to_not_convert` lists MoE-template names like `mlp.gate` # that collide with fused dense MLP names by plain substring. + ignored = ignored.rstrip(".") + prefix = prefix.rstrip(".") if ignored == prefix: return True if prefix.startswith(ignored + "."): diff --git a/python/sglang/srt/layers/radix_attention.py b/python/sglang/srt/layers/radix_attention.py index 468766011b0b..c3409c8c27e9 100644 --- a/python/sglang/srt/layers/radix_attention.py +++ b/python/sglang/srt/layers/radix_attention.py @@ -29,6 +29,7 @@ from sglang.srt.model_executor.breakable_cuda_graph.context import ( is_in_breakable_cuda_graph, ) +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.utils.custom_op import register_custom_op if TYPE_CHECKING: @@ -135,7 +136,7 @@ def forward( ) return output else: - return forward_batch.attn_backend.forward( + return get_attn_backend().forward( q, k, v, @@ -159,6 +160,12 @@ def unified_attention_with_output( q_rope: Optional[torch.Tensor] = None, k_rope: Optional[torch.Tensor] = None, sinks: Optional[torch.Tensor] = None, + # MLA / TRT-LLM / NSA paths pass these through RadixAttention.forward(**kwargs); + # they must appear in the schema when --enforce-piecewise-cuda-graph is on. + cos_sin_cache: Optional[torch.Tensor] = None, + is_neox: Optional[bool] = None, + llama_4_scaling: Optional[torch.Tensor] = None, + topk_indices: Optional[torch.Tensor] = None, ) -> None: context = get_forward_context() forward_batch = context.forward_batch @@ -177,6 +184,14 @@ def unified_attention_with_output( kwargs["k_rope"] = k_rope[:real_num_tokens] if sinks is not None: kwargs["sinks"] = sinks + if cos_sin_cache is not None: + kwargs["cos_sin_cache"] = cos_sin_cache + if is_neox is not None: + kwargs["is_neox"] = is_neox + if llama_4_scaling is not None: + kwargs["llama_4_scaling"] = llama_4_scaling + if topk_indices is not None: + kwargs["topk_indices"] = topk_indices[:real_num_tokens] original_out_cache_loc = forward_batch.out_cache_loc # Keep the original ForwardBatch object and only narrow cache locations for @@ -188,7 +203,7 @@ def unified_attention_with_output( # the FA kernel validates out.size(0) == q.size(0). forward_batch._attn_output = output[:real_num_tokens] - ret = forward_batch.attn_backend.forward( + ret = get_attn_backend().forward( query, key, value, diff --git a/python/sglang/srt/layers/radix_linear_attention.py b/python/sglang/srt/layers/radix_linear_attention.py index edaac21253c3..019b981b1fb3 100644 --- a/python/sglang/srt/layers/radix_linear_attention.py +++ b/python/sglang/srt/layers/radix_linear_attention.py @@ -22,6 +22,13 @@ from sglang.srt.compilation.compilation_config import register_split_op from sglang.srt.compilation.piecewise_context_manager import get_forward_context +from sglang.srt.model_executor.breakable_cuda_graph.breakable_cuda_graph import ( + eager_on_graph, +) +from sglang.srt.model_executor.breakable_cuda_graph.context import ( + is_in_breakable_cuda_graph, +) +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.utils.custom_op import register_custom_op if TYPE_CHECKING: @@ -83,16 +90,25 @@ def forward( dtype=mixed_qkv.dtype, device=mixed_qkv.device, ) - unified_linear_attention_with_output( - mixed_qkv, - a, - b, - output, - self.layer_id, - ) + if is_in_breakable_cuda_graph(): + bcg_unified_linear_attention_with_output( + mixed_qkv, + a, + b, + output, + self.layer_id, + ) + else: + unified_linear_attention_with_output( + mixed_qkv, + a, + b, + output, + self.layer_id, + ) return output else: - return forward_batch.attn_backend.forward( + return get_attn_backend().forward( layer=self, forward_batch=forward_batch, mixed_qkv=mixed_qkv, @@ -124,7 +140,7 @@ def unified_linear_attention_with_output( # this backend call so model/backend state is still written to the same batch. forward_batch.out_cache_loc = original_out_cache_loc[:real_num_tokens] - ret = forward_batch.attn_backend.forward( + ret = get_attn_backend().forward( layer=attention_layer, forward_batch=forward_batch, mixed_qkv=mixed_qkv[:real_num_tokens], @@ -135,3 +151,8 @@ def unified_linear_attention_with_output( output[:, :real_num_tokens].copy_(ret) return + + +bcg_unified_linear_attention_with_output = eager_on_graph(True)( + unified_linear_attention_with_output +) diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index 9181fbac5e4e..816702f3ca87 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -17,7 +17,6 @@ from sglang.srt.sampling.sampling_params import TOP_K_ALL from sglang.srt.server_args import get_global_server_args from sglang.srt.utils.common import ( - crash_on_warnings, get_bool_env_var, is_cuda, is_musa, @@ -57,7 +56,6 @@ class Sampler(nn.Module): def __init__(self): super().__init__() - self.use_nan_detection = get_global_server_args().enable_nan_detection self.tp_sync_group = get_tp_group().device_group if is_dp_attention_enabled(): self.tp_sync_group = get_attention_tp_group().device_group @@ -74,20 +72,9 @@ def __init__(self): def _preprocess_logits( self, logits: torch.Tensor, sampling_info: SamplingBatchInfo ) -> torch.Tensor: - """Apply custom logit processors and handle NaN detection.""" - # Apply the custom logit processors if registered in the sampling info + """Apply custom logit processors.""" if sampling_info.has_custom_logit_processor: apply_custom_logit_processor(logits, sampling_info) - - # Detect and handle NaN values in logits - if self.use_nan_detection and torch.any(torch.isnan(logits)): - logger.warning("Detected errors during sampling! NaN in the logits.") - logits = torch.where( - torch.isnan(logits), torch.full_like(logits, -1e5), logits - ) - if crash_on_warnings(): - raise ValueError("Detected errors during sampling! NaN in the logits.") - return logits def forward( @@ -238,7 +225,6 @@ def _sample_from_probs( sampling_info.top_ks, sampling_info.top_ps, filter_apply_order="joint", - check_nan=self.use_nan_detection, ) elif backend == "pytorch": # A slower fallback implementation with torch native operations. diff --git a/python/sglang/srt/layers/utils/cp_utils.py b/python/sglang/srt/layers/utils/cp_utils.py index 885dfed3b6f0..b87225dc7882 100644 --- a/python/sglang/srt/layers/utils/cp_utils.py +++ b/python/sglang/srt/layers/utils/cp_utils.py @@ -14,6 +14,7 @@ get_attention_cp_size, is_allocation_symmetric, ) +from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.server_args import get_global_server_args @@ -50,19 +51,41 @@ def is_prefill_cp_in_seq_split(): ) +def is_mla_prefill_cp_enabled() -> bool: + sa = get_global_server_args() + return sa.enable_prefill_context_parallel and sa.use_mla_backend + + +def mla_use_prefill_cp(forward_batch, mla_enable_prefill_cp=None): + if mla_enable_prefill_cp is None: + mla_enable_prefill_cp = is_mla_prefill_cp_enabled() + return ( + forward_batch.attn_cp_metadata is not None + and mla_enable_prefill_cp + and forward_batch.forward_mode.is_context_parallel_extend() + ) + + def can_cp_split(seq_len: int, cp_size: int, forward_batch): - # CP metadata (zigzag split) only supports batch=1 for now. + from sglang.srt.model_executor.forward_batch_info import ForwardMode + + # TODO current just support prefill batch=1 and len(input_ids) > self.cp_size * 2 + # Note: (self.cp_size * 2) To achieve load balancing for seq computation, + # the seq data needs to be divided and recombined at twice the size of cp_size. cur_cp_seq_len = seq_len // (cp_size * 2) - if ( + return ( cur_cp_seq_len != 0 and cp_size > 1 + # prepare_context_parallel_metadata hard-codes bs_per_cp_group = 1; + # guard explicitly to avoid silent mis-partitioning under continuous batching. + # TODO: remove this guard once we support multi-batch-cp-split + and forward_batch.batch_size == 1 and forward_batch.forward_mode.is_context_parallel_extend() + # is_context_parallel_extend() returns True for MIXED (prefill+decode + # in one step), but the zigzag split only makes sense on pure extend. + and forward_batch.forward_mode != ForwardMode.MIXED and is_prefill_context_parallel_enabled() - and forward_batch.seq_lens_cpu.shape[0] == 1 - ): - return True - else: - return False + ) def cp_split_and_rebuild_data(forward_batch, input_: torch.Tensor): @@ -342,7 +365,7 @@ def cp_allgather_and_save_kv_cache(forward_batch, layer, k, v, cp_size): v, cp_size, forward_batch, torch.cuda.current_stream() ) - forward_batch.token_to_kv_pool.set_kv_buffer( + get_token_to_kv_pool().set_kv_buffer( layer, cache_loc, key_cache_full, @@ -394,6 +417,7 @@ def prepare_context_parallel_metadata( cp_rank, cp_size, seqs_len, + extend_lens, ): from sglang.srt.layers.attention.dsa.utils import ( is_dsa_prefill_cp_round_robin_split, @@ -448,18 +472,12 @@ def prepare_context_parallel_metadata( bs_per_cp_group = 1 kv_len_origin = kv_len - # Derive prefix offset from the full sequence length on CPU. - # NOTE: forward_batch.seq_lens_cpu includes cached prefix + extend tokens. - # In CP we only split the extend tokens, but cache_seqlens passed to FA must - # include the cached prefix. - prefix_len = 0 - try: - if seqs_len is not None and len(seqs_len) == 1: - prefix_len = int(seqs_len[0]) - int(kv_len_origin.item()) - if prefix_len < 0: - prefix_len = 0 - except Exception: - prefix_len = 0 + # Derive prefix offset from unpadded CPU tensors. Both `seqs_len` and `extend_lens` are unpadded by the caller + # Using the padded `kv_len` here would undercount `prefix_len` by the padding amount and shift the FA causal horizon. + assert ( + len(seqs_len) == 1 and len(extend_lens) == 1 + ), "Prefill Context Parallel only supports batch_size == 1 for now" + prefix_len = max(0, int(seqs_len[0]) - int(extend_lens[0])) # get zigzag index cp_segment_num = cp_size * 2 seq_per_batch = kv_len // cp_segment_num # seq_len for each batch and segment diff --git a/python/sglang/srt/lora/lora_overlap_loader.py b/python/sglang/srt/lora/lora_overlap_loader.py index bc7b3dd71d2e..6d5845ba084d 100644 --- a/python/sglang/srt/lora/lora_overlap_loader.py +++ b/python/sglang/srt/lora/lora_overlap_loader.py @@ -35,6 +35,10 @@ def try_overlap_load_lora( Check a LoRA adapter's asynchronous load status, and try to load it if there's capacity in the memory pool. Returns whether or not the adapter has been loaded. """ + # Drain completed async loads before status/capacity checks so finished + # adapters no longer count as in-flight. + self._drain_completed_overlap_loads() + lora_pipeline_load_status = self._check_overlap_load_status(lora_id) if lora_pipeline_load_status == LoRAOverlapLoadStatus.LOADING: return False @@ -51,18 +55,25 @@ def try_overlap_load_lora( def _check_overlap_load_status( self, lora_id: Optional[str] ) -> LoRAOverlapLoadStatus: - if lora_id not in self.lora_to_overlap_load_event: - return LoRAOverlapLoadStatus.NOT_LOADED - - event = self.lora_to_overlap_load_event[lora_id] - - if not event.query(): + if lora_id in self.lora_to_overlap_load_event: return LoRAOverlapLoadStatus.LOADING - torch.cuda.current_stream().wait_event(event) - del self.lora_to_overlap_load_event[lora_id] - - return LoRAOverlapLoadStatus.LOADED + # After completed events have been drained, a memory-pool entry with no + # pending event is safe to use on the current stream. + if lora_id in self.lora_manager.memory_pool.uid_to_buffer_id: + return LoRAOverlapLoadStatus.LOADED + + return LoRAOverlapLoadStatus.NOT_LOADED + + def _drain_completed_overlap_loads(self) -> None: + completed_loads = [ + (lora_id, event) + for lora_id, event in self.lora_to_overlap_load_event.items() + if event.query() + ] + for lora_id, event in completed_loads: + torch.cuda.current_stream().wait_event(event) + del self.lora_to_overlap_load_event[lora_id] def _try_start_overlap_load( self, lora_id: Optional[str], running_loras: set[Optional[str]] diff --git a/python/sglang/srt/managers/detokenizer_manager.py b/python/sglang/srt/managers/detokenizer_manager.py index 83d82af0cadc..4f4d8331c65c 100644 --- a/python/sglang/srt/managers/detokenizer_manager.py +++ b/python/sglang/srt/managers/detokenizer_manager.py @@ -238,7 +238,7 @@ def _decode_batch_token_id_output(self, recv_obj: BatchTokenIDOutput): if rid not in self.decode_status: s = DecodeStatus( decoded_text=recv_obj.decoded_texts[i], - decode_ids=recv_obj.decode_ids[i], + decode_ids=list(recv_obj.decode_ids[i]), surr_offset=0, read_offset=recv_obj.read_offsets[i], ) diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index b15c44084ef0..fdda0115114c 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -23,6 +23,7 @@ import copy import uuid from abc import ABC +from array import array from collections import Counter from dataclasses import dataclass, field from enum import Enum @@ -714,7 +715,7 @@ class TokenizedGenerateReqInput(BaseReq): # The input text input_text: str # The input token ids - input_ids: List[int] + input_ids: Optional[array[int]] # The multimodal inputs mm_inputs: object # The sampling parameters @@ -1039,7 +1040,7 @@ class TokenizedEmbeddingReqInput(BaseReq): # The input text input_text: str # The input token ids - input_ids: List[int] + input_ids: array[int] # The image inputs image_inputs: dict # The token type ids @@ -1087,10 +1088,10 @@ class BatchTokenIDOutput(BaseBatchReq, SpeculativeDecodingMetricsMixin): finished_reasons: List[BaseFinishReason] # For incremental decoding decoded_texts: List[str] - decode_ids: List[int] + decode_ids: List[array[int]] read_offsets: List[int] # Only used when `--skip-tokenizer-init` is on - output_ids: Optional[List[int]] + output_ids: Optional[List[array[int]]] # Detokenization configs skip_special_tokens: List[bool] spaces_between_special_tokens: List[bool] diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index 1c7d3afe1348..174eece9304b 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -459,16 +459,40 @@ def _get_precomputed_embedding( ] +def _can_skip_pre_embed_feature_move(data_embedding_func: DataEmbeddingFunc) -> bool: + """qwen-vl visual forward already moves batched features to the target device. + + instead of performing multiple H2D for each mm feature from all mm_items (followed by concatenation on device), + for some models which internally performs H2D on concated mm feature, these small H2D calls could be replaced with a single big H2D + """ + owner = getattr(data_embedding_func, "__self__", None) + if owner is None: + return False + if getattr(data_embedding_func, "__name__", None) not in ( + "get_image_feature", + "get_video_feature", + ): + return False + return owner.__class__.__name__ in { + "Qwen3VLForConditionalGeneration", + "Qwen3VLMoeForConditionalGeneration", + "Qwen3_5ForConditionalGeneration", + "Qwen3_5MoeForConditionalGeneration", + } + + def _move_items_to_device( items: List[MultimodalDataItem], device: torch.device ) -> None: - """Move item features to the target device (in-place, non-blocking).""" + """Move item features to the target device (in-place, non-blocking). + Saves a CPU reference so the offload path can restore without GPU->CPU copy.""" for item in items: if isinstance(item.feature, torch.Tensor) and item.feature.device != device: + item._cpu_feature = item.feature item.feature = item.feature.to(device, non_blocking=True) -def _get_chunked_embedding_full( +def get_chunked_embedding_legacy( data_embedding_func: DataEmbeddingFunc, embedding_items_per_req: List[MultimodalDataItem], items_offset: List[Tuple[int, int]], @@ -486,7 +510,8 @@ def _get_chunked_embedding_full( embedding_per_req = embedding_cache.get(item_hashes) if embedding_per_req is None: - _move_items_to_device(embedding_items_per_req, device) + if not _can_skip_pre_embed_feature_move(data_embedding_func): + _move_items_to_device(embedding_items_per_req, device) embedding = data_embedding_func(embedding_items_per_req) embedding_per_req = ( EmbeddingResult(embedding=embedding) @@ -516,77 +541,85 @@ def _get_chunked_embedding_full( return embedding_per_req_chunk, input_ids -def _get_chunked_embedding_by_item( - data_embedding_func: DataEmbeddingFunc, +def find_chunk_items_and_check_cache( embedding_items_per_req: List[MultimodalDataItem], items_offset: List[Tuple[int, int]], - extend_prefix_len: int, - extend_seq_len: int, - device: torch.device, -) -> Optional[torch.Tensor]: - """ - Per-image chunk-aware encoding: only encode images overlapping with the - current chunk, cache each image individually. - Items must already be split per-image (each item has exactly one offset). - """ - chunk_start = extend_prefix_len - chunk_end = extend_prefix_len + extend_seq_len # exclusive - - if extend_seq_len <= 0: - return None - - # 1. Find items overlapping with current chunk - # offsets are (start, end) inclusive on both ends - overlapping = [] - for idx, (item, offset) in enumerate(zip(embedding_items_per_req, items_offset)): - start, end = offset + chunk_start: int, + chunk_end: int, +) -> List[Tuple[MultimodalDataItem, Optional[torch.Tensor], int, int]]: + """Return (item, cached_embedding_or_None, start, end) for items in [chunk_start, chunk_end).""" + chunk_entries = [] + for item, (start, end) in zip(embedding_items_per_req, items_offset): if end >= chunk_start and start < chunk_end: - overlapping.append((idx, item, start, end)) - - if not overlapping: - return None - - # 2. Check per-image cache for each overlapping item - cached_embeddings = {} # idx -> tensor - miss_items = [] # (idx, item, start, end) - for idx, item, start, end in overlapping: - cached = embedding_cache.get_single(item.hash) - if cached is not None: - cached_embeddings[idx] = cached.embedding - else: - miss_items.append((idx, item, start, end)) + cached = embedding_cache.get_single(item.hash) + emb = cached.embedding if cached is not None else None + chunk_entries.append((item, emb, start, end)) + return chunk_entries - # 3. Batch encode all cache-miss items in one ViT call - if miss_items: - miss_item_list = [item for _, item, _, _ in miss_items] - _move_items_to_device(miss_item_list, device) - all_miss_embedding = data_embedding_func(miss_item_list) - all_miss_embedding = all_miss_embedding.reshape( - -1, all_miss_embedding.shape[-1] - ) - # Split output by per-item token count - token_counts = [end - start + 1 for _, _, start, end in miss_items] - split_embeddings = torch.split(all_miss_embedding, token_counts, dim=0) - - for (idx, item, _, _), emb in zip(miss_items, split_embeddings): - cached_embeddings[idx] = emb - emb_result = EmbeddingResult(embedding=emb) - embedding_cache.set(item.hash, emb_result) - - # 4. Assemble chunk: for each overlapping item, extract the overlap slice +def assemble_chunk_embedding( + chunk_entries: List[Tuple[Any, torch.Tensor, int, int]], + chunk_start: int, + chunk_end: int, +) -> Optional[torch.Tensor]: + """ + Assemble a chunk of embeddings by slicing each item's embedding + to the portion that falls within [chunk_start, chunk_end). + """ chunk_slices = [] - for idx, _, start, end in overlapping: - emb = cached_embeddings[idx] # shape: (end - start + 1, hidden) + for _, emb, start, end in chunk_entries: overlap_start = max(start, chunk_start) overlap_end = min(end, chunk_end - 1) # inclusive local_start = overlap_start - start local_end = overlap_end - start + 1 # exclusive for slicing chunk_slices.append(emb[local_start:local_end]) + if not chunk_slices: + return None return torch.cat(chunk_slices, dim=0) +def get_chunked_prefill_embedding_legacy( + data_embedding_func: DataEmbeddingFunc, + embedding_items: List[MultimodalDataItem], + items_size: List[int], + prefix_length: List[int], + extend_length: List[int], + items_offset_list: List[List[Tuple[int, int]]], + input_ids: torch.Tensor, + max_iterations: int, +) -> tuple[torch.Tensor | None, torch.Tensor]: + """Non-per-image path: encode each request independently.""" + embedding_list = [] + device = input_ids.device + + for i in range(max_iterations): + if items_size[i] == items_size[i + 1]: + continue + embedding_items_per_req = embedding_items[items_size[i] : items_size[i + 1]] + items_offset = items_offset_list[i] + assert items_offset is not None, items_offset + + extend_prefix_len = prefix_length[i] + extend_seq_len = extend_length[i] if i < len(extend_length) else 0 + + chunk_embedding, input_ids = get_chunked_embedding_legacy( + data_embedding_func, + embedding_items_per_req, + items_offset, + extend_prefix_len, + extend_seq_len, + input_ids, + device, + ) + if chunk_embedding is not None: + embedding_list.append(chunk_embedding) + + if len(embedding_list) == 0: + return None, input_ids + return torch.concat(embedding_list, dim=0), input_ids + + def _get_chunked_prefill_embedding( data_embedding_func: DataEmbeddingFunc, embedding_items: List[MultimodalDataItem], @@ -597,56 +630,99 @@ def _get_chunked_prefill_embedding( input_ids: torch.Tensor, ) -> tuple[torch.Tensor | None, torch.Tensor]: """ - Chunked prefill embedding: encode per-request items and extract the chunk. - Items are already split per-image at processor stage. + Chunked prefill embedding: collect cache misses across all per-image + requests, batch them into a single ViT call, then assemble per-request + chunk embeddings from the results. """ embedding_list = [] device = input_ids.device # FIXME(Xinyuan): temporary workaround for eagle3 + # FIXME(yhyang201): check this max_iterations = min(len(items_size) - 1, len(prefix_length)) + per_image_process = ( + len(embedding_items) > 0 and len(embedding_items[0].offsets) == 1 + ) + + if not per_image_process: + return get_chunked_prefill_embedding_legacy( + data_embedding_func, + embedding_items, + items_size, + prefix_length, + extend_length, + items_offset_list, + input_ids, + max_iterations, + ) + + # collect chunk entries per request, accumulate all misses + pending_requests = [] + all_miss_items = [] + all_miss_token_counts = [] + for i in range(max_iterations): if items_size[i] == items_size[i + 1]: continue + extend_seq_len = extend_length[i] if i < len(extend_length) else 0 + if extend_seq_len <= 0: + continue + + extend_prefix_len = prefix_length[i] embedding_items_per_req = embedding_items[items_size[i] : items_size[i + 1]] items_offset = items_offset_list[i] assert items_offset is not None, items_offset - extend_prefix_len = prefix_length[i] - extend_seq_len = extend_length[i] if i < len(extend_length) else 0 - - # Skip if all items already prefilled - if all(offset_end < prefix_length[i] for _, offset_end in items_offset): + chunk_start = extend_prefix_len + chunk_end = extend_prefix_len + extend_seq_len + chunk_entries = find_chunk_items_and_check_cache( + embedding_items_per_req, + items_offset, + chunk_start, + chunk_end, + ) + if not chunk_entries: continue - # Use per-image path when all items have exactly one offset (already - # split per-image) — this avoids encoding images not in this chunk. - # Fall back to combined path for non-split items or EVS. - is_per_image = all(len(item.offsets) == 1 for item in embedding_items_per_req) + for item, emb, start, end in chunk_entries: + if emb is None: + all_miss_items.append(item) + all_miss_token_counts.append(end - start + 1) + + pending_requests.append((chunk_entries, chunk_start, chunk_end)) + + miss_embeddings = [] + if all_miss_items: + if not _can_skip_pre_embed_feature_move(data_embedding_func): + _move_items_to_device(all_miss_items, device) + # vit_input_tokens = sum( + # item.feature.shape[0] for item in all_miss_items + # if isinstance(item.feature, torch.Tensor) + # ) + # logger.info(f"ViT batch: {len(all_miss_items)} items, {vit_input_tokens} input patches, {sum(all_miss_token_counts)} output tokens") + all_miss_embedding = data_embedding_func(all_miss_items) + all_miss_embedding = all_miss_embedding.reshape( + -1, all_miss_embedding.shape[-1] + ) + miss_embeddings = list( + torch.split(all_miss_embedding, all_miss_token_counts, dim=0) + ) + for item, emb in zip(all_miss_items, miss_embeddings): + embedding_cache.set(item.hash, EmbeddingResult(embedding=emb)) + + # fill in miss embeddings and assemble per-request chunks + miss_iter = iter(miss_embeddings) + for chunk_entries, chunk_start, chunk_end in pending_requests: + chunk_entries = [ + (item, next(miss_iter) if emb is None else emb, start, end) + for item, emb, start, end in chunk_entries + ] - if is_per_image: - chunk_embedding = _get_chunked_embedding_by_item( - data_embedding_func, - embedding_items_per_req, - items_offset, - extend_prefix_len, - extend_seq_len, - device, - ) - if chunk_embedding is not None: - embedding_list.append(chunk_embedding) - else: - chunk_embedding, input_ids = _get_chunked_embedding_full( - data_embedding_func, - embedding_items_per_req, - items_offset, - extend_prefix_len, - extend_seq_len, - input_ids, - device, - ) - if chunk_embedding is not None: - embedding_list.append(chunk_embedding) + chunk_embedding = assemble_chunk_embedding( + chunk_entries, chunk_start, chunk_end + ) + if chunk_embedding is not None: + embedding_list.append(chunk_embedding) if len(embedding_list) == 0: return None, input_ids @@ -805,19 +881,18 @@ def embed_mm_inputs( device=input_ids.device, ) # calculate per request items length offset - items_size = torch.zeros(len(mm_inputs_list) + 1, dtype=int) + items_size = [0] items_offsets = [] - for i, mm_inputs in enumerate(mm_inputs_list): + for mm_inputs in mm_inputs_list: mm_items = [ item for item in mm_inputs.mm_items if item.is_modality(modality=modality) ] - items_size[i + 1] = len(mm_items) + items_size.append(items_size[-1] + len(mm_items)) items_offsets.append( flatten_nested_list([item.offsets for item in mm_items]) ) - items_size = torch.cumsum(items_size, dim=0).tolist() embedding, mask, input_ids = get_embedding_and_mask( data_embedding_func=embedder, @@ -987,6 +1062,25 @@ def _embed_mm_inputs_with_split( return input_embeds, other_info +def offload_mm_features_to_cpu(mm_inputs_list: List[MultimodalInputs]): + """Free GPU features after embedding. CPU copies are kept for later use + (e.g. chunked prefill or recovery after retraction).""" + language_only = get_global_server_args().language_only + for mm_input in mm_inputs_list or []: + if not mm_input or not hasattr(mm_input, "mm_items"): + continue + for item in mm_input.mm_items: + if isinstance(item.feature, torch.Tensor) and item.feature.is_cuda: + if item._cpu_feature is not None: + item.feature = item._cpu_feature + else: + item.feature = item.feature.to("cpu", non_blocking=True) + if language_only: + pe = item.precomputed_embeddings + if isinstance(pe, torch.Tensor) and pe.is_cuda: + item.precomputed_embeddings = pe.to("cpu", non_blocking=True) + + def general_mm_embed_routine( input_ids: torch.Tensor, forward_batch: ForwardBatch, @@ -999,18 +1093,6 @@ def general_mm_embed_routine( ) -> torch.Tensor: """ Process multimodal inputs and forward through language model. - - Args: - input_ids: Input token IDs tensor - forward_batch: Batch information for model forward pass - language_model: Base language model to use - data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function. - placeholder_tokens: Token IDs for multimodal placeholders - use_deepstack: Whether to use deepstack embeddings for each modality, default False - **kwargs: Additional arguments passed to language model - - Returns: - Hidden states from language model forward pass """ assert hasattr(language_model, "get_input_embeddings") embed_tokens = language_model.get_input_embeddings() @@ -1064,34 +1146,9 @@ def general_mm_embed_routine( # add for qwen3_vl deepstack if use_deepstack: kwargs["input_deepstack_embeds"] = other_info["input_deepstack_embeds"] - # Offload GPU features to CPU instead of discarding them to balance memory - # efficiency and data persistence. - # In chunked-prefill, a request is processed across multiple batches, and - # the original multimodal data must remain accessible until the entire - # prefill phase is complete. Since the multimodal embedding cache is - # best-effort, offloading to CPU ensures we have a reliable fallback - # if a cache miss occurs in subsequent chunks, while still freeing up - # critical GPU memory. - if mm_inputs_list: - for mm_input_obj in mm_inputs_list: - if mm_input_obj and hasattr(mm_input_obj, "mm_items"): - for mm_item in mm_input_obj.mm_items: - feature = getattr(mm_item, "feature", None) - if isinstance(feature, torch.Tensor) and feature.is_cuda: - mm_item.feature = feature.to("cpu", non_blocking=True) - if get_global_server_args().language_only: - precomputed_embeddings = getattr( - mm_item, "precomputed_embeddings", None - ) - if ( - isinstance(precomputed_embeddings, torch.Tensor) - and precomputed_embeddings.is_cuda - ): - mm_item.precomputed_embeddings = ( - precomputed_embeddings.to( - "cpu", non_blocking=True - ) - ) + # Free GPU features after embedding. CPU copies are kept for + # later use (e.g. chunked prefill or recovery after retraction). + offload_mm_features_to_cpu(mm_inputs_list) forward_batch.mm_inputs = None forward_batch.mm_input_embeds = input_embeds else: @@ -1112,66 +1169,6 @@ def general_mm_embed_routine( return hidden_states -def get_multimodal_data_bounds( - input_ids: torch.Tensor, pad_values: List[int], token_pairs: List[Tuple[int, int]] -) -> torch.Tensor: - """ - Returns a tensor indicating the bounds of multimodal data (images, video, audio, etc.) - - Returns: - [bounds_count, 2] - """ - # All the multimodal data in the batch should share the same special bound token ids. - start_tokens = {s for s, _e in token_pairs} - end_tokens = {e for _s, e in token_pairs} - - assert all(isinstance(t, int) for t in start_tokens) - assert all(isinstance(t, int) for t in end_tokens) - - start_cond = torch.isin( - input_ids, torch.as_tensor(start_tokens, device=input_ids.device) - ) - end_cond = torch.isin( - input_ids, torch.as_tensor(end_tokens, device=input_ids.device) - ) - - (data_start_tokens,) = torch.where(start_cond) - (data_end_tokens,) = torch.where(end_cond) - - data_start_tokens_cpu = data_start_tokens.cpu().tolist() - data_end_tokens_cpu = data_end_tokens.cpu().tolist() - - # the im_start_id sometimes can be cached as prefix, but it is needed for the embedding of the multimodal data - if len(data_start_tokens_cpu) != len(data_end_tokens_cpu): - if ( - len(data_start_tokens_cpu) + 1 == len(data_end_tokens_cpu) - and input_ids[0].item() in pad_values - and data_end_tokens_cpu - and data_start_tokens_cpu - and data_end_tokens_cpu[0] < data_start_tokens_cpu[0] - ): - data_start_tokens_cpu.insert(0, 0) - valid_mm_data_nums = min(len(data_start_tokens_cpu), len(data_end_tokens_cpu)) - - if valid_mm_data_nums == 0: - return torch.zeros((0, 2), device=input_ids.device) - - # Filter out pairs where start_token >= end_token - valid_pairs = [] - for i in range(valid_mm_data_nums): - start_token = data_start_tokens_cpu[i] - end_token = data_end_tokens_cpu[i] - if start_token < end_token: - valid_pairs.append((start_token + 1, end_token - 1)) - - if not valid_pairs: - return torch.zeros((0, 2), device=input_ids.device) - - # Convert valid pairs to tensor - valid_pairs_tensor = torch.as_tensor(valid_pairs, device=input_ids.device) - return valid_pairs_tensor - - def data_hash(data) -> int: hash_bytes = hashlib.sha256(data).digest()[:8] return int.from_bytes(hash_bytes, byteorder="big", signed=False) @@ -1382,10 +1379,12 @@ def get_new_expanded_mm_items(original_mm_items): expanded_mm_items.append(item) continue - patches_per_item = [] - for grid in image_grid_thw: - grid_tensor = torch.as_tensor(grid, dtype=torch.long) - patches_per_item.append(int(torch.prod(grid_tensor).item())) + if isinstance(image_grid_thw, torch.Tensor): + patches_per_item = ( + torch.prod(image_grid_thw, dim=-1).long().tolist() + ) + else: + patches_per_item = [int(np.prod(grid)) for grid in image_grid_thw] cumulative = torch.cumsum( torch.tensor(patches_per_item, dtype=torch.long), dim=0 @@ -1435,17 +1434,11 @@ def get_new_expanded_mm_items(original_mm_items): num_videos = grid_len # Calculate total frames and frames per video - frames_per_video = [] - total_frames = 0 - for i in range(num_videos): - grid = video_grid_thw[i] - if isinstance(grid, torch.Tensor): - T = int(grid[0].item()) # T is the first element [T, H, W] - else: - grid_tensor = torch.as_tensor(grid, dtype=torch.long) - T = int(grid_tensor[0].item()) - frames_per_video.append(T) - total_frames += T + if isinstance(video_grid_thw, torch.Tensor): + frames_per_video = video_grid_thw[:, 0].long().tolist() + else: + frames_per_video = [int(grid[0]) for grid in video_grid_thw] + total_frames = sum(frames_per_video) # num_items should equal total_frames when T > 1 if num_items != total_frames: @@ -1453,14 +1446,12 @@ def get_new_expanded_mm_items(original_mm_items): continue # Calculate patches per video: T * H * W for each video - patches_per_video = [] - for i in range(num_videos): - grid = video_grid_thw[i] - if isinstance(grid, torch.Tensor): - patches_per_video.append(int(torch.prod(grid).item())) - else: - grid_tensor = torch.as_tensor(grid, dtype=torch.long) - patches_per_video.append(int(torch.prod(grid_tensor).item())) + if isinstance(video_grid_thw, torch.Tensor): + patches_per_video = ( + torch.prod(video_grid_thw, dim=-1).long().tolist() + ) + else: + patches_per_video = [int(np.prod(grid)) for grid in video_grid_thw] # Calculate cumulative patches to get slice indices for each video cumulative = torch.cumsum( diff --git a/python/sglang/srt/managers/overlap_utils.py b/python/sglang/srt/managers/overlap_utils.py index 99b951fab629..4856fe7ddbc1 100644 --- a/python/sglang/srt/managers/overlap_utils.py +++ b/python/sglang/srt/managers/overlap_utils.py @@ -1,7 +1,7 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional +import os +from typing import TYPE_CHECKING, Optional, Union import torch @@ -17,6 +17,40 @@ _is_cuda = is_cuda() _is_hip = is_hip() +# Token-buf consume tracking: init to -1, assert non-negative on gather, +# write -1 back. Catches "gather without intermediate stash" bugs. CI enables +# via the existing SGLANG_IS_IN_CI; off in production. +_DEBUG_ASSERT = os.getenv("SGLANG_IS_IN_CI", "").lower() == "true" + + +@torch.compile(dynamic=True) +def _assert_nonneg_and_invalidate( + values: torch.Tensor, buf: torch.Tensor, indices: torch.Tensor +) -> None: + """Fused: assert all `values >= 0` and scatter -1 into `buf[indices]`. + Compiled so the reduction + assert + scatter run as one kernel launch.""" + torch._assert_async((values >= 0).all()) + buf[indices] = -1 + + +@torch.compile(dynamic=True) +def _gather_spec_extras( + indices: torch.Tensor, + topk_p_buf: torch.Tensor, + topk_index_buf: torch.Tensor, + output_tokens_buf: torch.Tensor, + hidden_states_buf: Optional[torch.Tensor], +): + """Compiled gather of spec extras. `hidden_states_buf` is None when the + build does not capture hidden states.""" + topk_p = topk_p_buf[indices] + topk_index = topk_index_buf[indices] + bonus_tokens = output_tokens_buf[indices] + hidden_states = ( + hidden_states_buf[indices] if hidden_states_buf is not None else None + ) + return topk_p, topk_index, bonus_tokens, hidden_states + def _resolve_future_token_ids_native(input_ids, future_token_ids_map): input_ids[:] = torch.where( @@ -36,39 +70,44 @@ def _resolve_future_token_ids_native(input_ids, future_token_ids_map): _resolve_future_token_ids = _resolve_future_token_ids_native -@dataclass -class FutureIndices: - indices: torch.Tensor +class FutureMap: + """Cross-iter relay buffer for values the next iter's schedule cannot + compute locally (e.g. spec_v2 seq_lens after accept_lens, sampled tokens). + Forward stream publishes into a buf; next iter's schedule pulls lazily. + Schedule-deterministic values (e.g. non-spec seq_lens via +1) stay + maintained by SB directly and do not need the relay. + + SB.seq_lens GPU is always a faithful seq_lens_cpu mirror; forward path + treats it as read-only, spec mutations land on forward_batch.seq_lens. + """ -class FutureMap: def __init__( self, device: torch.device, spec_algo: SpeculativeAlgorithm, req_to_token_pool: ReqToTokenPool, ): - # All buffers are indexed by req_pool_idx. Slot 0 mirrors the KV cache - # pool's padding row, so CUDA-graph padded batches (req_pool_idx == 0) - # read/write here harmlessly. + # Bufs indexed by req_pool_idx; slot 0 mirrors KV padding row so + # CUDA-graph padded batches (req_pool_idx == 0) are harmless. self.device = device self.spec_algo = spec_algo self.req_pool_size = req_to_token_pool.req_to_token.shape[0] - if self.spec_algo.is_none(): - self.token_ids_buf = torch.empty( - (self.req_pool_size,), dtype=torch.int64, device=self.device - ) - else: - # Schedule-consumed buf, eager fixed dtype. - self.new_seq_lens_buf = torch.empty( + self.output_tokens_buf = ( + torch.full((self.req_pool_size,), -1, dtype=torch.int64, device=self.device) + if _DEBUG_ASSERT + else torch.empty( (self.req_pool_size,), dtype=torch.int64, device=self.device ) - # Forward-only bufs are lazy (worker-dependent shape). + ) + self.new_seq_lens_buf = torch.empty( + (self.req_pool_size,), dtype=torch.int64, device=self.device + ) + if self.spec_algo.is_some(): self._forward_buf_initialized = False - # Fences the schedule-consumed buf fields. - self.publish_ready: Optional[torch.cuda.Event] = None + self.publish_ready = None # lazy device.Event(); only spec_v2 needs it def _lazy_init_forward_buf(self, draft_input: EagleDraftInput): self._forward_buf_initialized = True @@ -85,9 +124,6 @@ def _lazy_init_forward_buf(self, draft_input: EagleDraftInput): dtype=topk_index0.dtype, device=self.device, ) - self.bonus_tokens_buf = torch.empty( - (self.req_pool_size,), dtype=torch.int64, device=self.device - ) if spec_need_hidden_states(): hidden_states0 = draft_input.hidden_states[0] self.hidden_states_buf = torch.empty( @@ -97,66 +133,100 @@ def _lazy_init_forward_buf(self, draft_input: EagleDraftInput): ) def resolve_future(self, batch: ScheduleBatch): + # seq_lens is already real on entry (SB +1 for non-spec; + # resolve_seq_lens_cpu pulled from buf for spec_v2). Only resolve + # input_ids tokens / spec extras here. if self.spec_algo.is_none(): - _resolve_future_token_ids(batch.input_ids, self.token_ids_buf) + _resolve_future_token_ids(batch.input_ids, self.output_tokens_buf) + if _DEBUG_ASSERT: + _assert_nonneg_and_invalidate( + batch.input_ids, self.output_tokens_buf, batch.req_pool_indices + ) else: - draft_input: EagleDraftInput = batch.spec_info - if draft_input is None: - # FIXME(lsyin): No future exists, only for prefill batch, not compatible with mixed mode - return - indices = draft_input.future_indices.indices - # FIXME: redundant. `indices` = batch.req_pool_indices, pinned via - # record_batch_in_overlap's attr_snapshot for 2 iters; refcount > 0 - # across forward's read, allocator can't reclaim. Safe to remove. - indices.record_stream(torch.get_device_module(self.device).current_stream()) - draft_input.topk_p = self.topk_p_buf[indices] - draft_input.topk_index = self.topk_index_buf[indices] - draft_input.bonus_tokens = self.bonus_tokens_buf[indices] - draft_input.new_seq_lens = self.new_seq_lens_buf[indices] - # Resolve seq_lens placeholder (-indices) to the post-verify view. - batch.seq_lens = draft_input.new_seq_lens - if spec_need_hidden_states(): - draft_input.hidden_states = self.hidden_states_buf[indices] + self._resolve_spec_extras(batch) + + def _resolve_spec_extras(self, batch: ScheduleBatch) -> None: + draft_input: EagleDraftInput = batch.spec_info + if draft_input is None: + # FIXME(lsyin): only prefill; not compatible with mixed mode + return + indices = draft_input.future_indices + # FIXME: indices = batch.req_pool_indices, pinned 2 iters via + # record_batch_in_overlap; record_stream here is redundant. + indices.record_stream(torch.get_device_module(self.device).current_stream()) + hidden_states_buf = ( + self.hidden_states_buf if spec_need_hidden_states() else None + ) + ( + draft_input.topk_p, + draft_input.topk_index, + draft_input.bonus_tokens, + hidden_states, + ) = _gather_spec_extras( + indices, + self.topk_p_buf, + self.topk_index_buf, + self.output_tokens_buf, + hidden_states_buf, + ) + if hidden_states is not None: + draft_input.hidden_states = hidden_states + if _DEBUG_ASSERT: + _assert_nonneg_and_invalidate( + draft_input.bonus_tokens, self.output_tokens_buf, indices + ) + + def set_input_ids_sentinel( + self, batch: ScheduleBatch, future_indices: torch.Tensor + ) -> None: + # Sentinel for the decode portion so mixed batches can cat extend + # (positive real tokens) + decode (negative sentinels) into one + # input_ids; resolve_future translates negatives via output_tokens_buf. + batch.input_ids = -future_indices def resolve_seq_lens_cpu(self, batch: ScheduleBatch) -> None: + # Lazy pull from new_seq_lens_buf for spec_v2 (accept_lens not known to + # schedule). Write into both CPU and GPU so SB.seq_lens stays a faithful + # seq_lens_cpu mirror. fi = batch.spec_info.future_indices if batch.spec_info is not None else None if fi is None: return if self.publish_ready is not None: self.publish_ready.wait() - batch.seq_lens_cpu = self.new_seq_lens_buf[fi.indices].cpu() + new_seq_lens = self.new_seq_lens_buf[fi] + batch.seq_lens = new_seq_lens + batch.seq_lens_cpu = new_seq_lens.cpu() batch.seq_lens_sum = int(batch.seq_lens_cpu.sum()) - def publish( - self, future_indices: FutureIndices, new_seq_lens: torch.Tensor - ) -> None: - """Store schedule-consumed fields and signal publish_ready.""" - if self.spec_algo.is_none(): - return - indices = future_indices.indices + def publish(self, future_indices: torch.Tensor, new_seq_lens: torch.Tensor) -> None: + indices = future_indices if indices.shape[0] == 0: return # DP idle self.new_seq_lens_buf[indices] = new_seq_lens.to(self.new_seq_lens_buf.dtype) - if self.publish_ready is None: - self.publish_ready = torch.get_device_module(self.device).Event() - self.publish_ready.record() + # Fast path: only spec_v2 needs the event (schedule-stream D2H sync). + if self.spec_algo.is_some(): + if self.publish_ready is None: + self.publish_ready = torch.get_device_module(self.device).Event() + self.publish_ready.record() - def stash(self, future_indices: FutureIndices, payload) -> None: - """Store forward-only fields for the next forward batch to pick up.""" - indices = future_indices.indices + def stash( + self, + future_indices: torch.Tensor, + payload: Union[torch.Tensor, EagleDraftInput], + ) -> None: + indices = future_indices if indices.shape[0] == 0: - return # DP idle + # DP idle: payload is empty stub; lazy-init shape peek would IndexError. + return if self.spec_algo.is_none(): - # next_token_ids is int32; buf is int64. Advanced indexing requires - # an explicit cast. - self.token_ids_buf[indices] = payload.to(torch.int64) + self.output_tokens_buf[indices] = payload.to(torch.int64) return draft_input: EagleDraftInput = payload if not self._forward_buf_initialized: self._lazy_init_forward_buf(draft_input) - self.bonus_tokens_buf[indices] = draft_input.bonus_tokens.to( - self.bonus_tokens_buf.dtype + self.output_tokens_buf[indices] = draft_input.bonus_tokens.to( + self.output_tokens_buf.dtype ) self.topk_p_buf[indices] = draft_input.topk_p.to(self.topk_p_buf.dtype) self.topk_index_buf[indices] = draft_input.topk_index.to( diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index b0aa7a2ed61c..9b73613cb0b0 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -2,7 +2,11 @@ from sglang.srt.dllm.config import DllmConfig from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.utils.common import ceil_align, is_pin_memory_available +from sglang.srt.utils.common import ( + ceil_align, + flatten_arrays_to_int64_tensor, + is_pin_memory_available, +) # ENGRAM_MODIFIED — Snapshot batch field @@ -38,11 +42,11 @@ import dataclasses import logging import re +from array import array from concurrent.futures import Future from enum import Enum, auto from functools import lru_cache from http import HTTPStatus -from itertools import chain from typing import ( TYPE_CHECKING, Any, @@ -67,7 +71,6 @@ from sglang.srt.distributed.parallel_state import get_tensor_model_parallel_rank from sglang.srt.dllm.mixin.req import ReqDllmMixin from sglang.srt.environ import envs -from sglang.srt.layers.attention.fla.chunk_delta_h import CHUNK_SIZE as FLA_CHUNK_SIZE from sglang.srt.managers.embed_types import PositionalEmbeds from sglang.srt.managers.scheduler_components.new_token_ratio_tracker import ( NewTokenRatioTracker, @@ -255,6 +258,8 @@ class MultimodalDataItem: # the raw features returned by processor, e.g. pixel_values or audio_features feature: Union[torch.Tensor, np.ndarray] = None + # CPU reference kept during GPU encoding, used to skip GPU->CPU copy on offload + _cpu_feature: Optional[torch.Tensor] = None # the precomputed embeddings, passed as final encoder embeddings # One and only one of the feature and precomputed_embeddings will be empty precomputed_embeddings: Optional[Union[torch.Tensor, np.ndarray]] = None @@ -344,18 +349,23 @@ def from_dict(obj: dict): ret.validate() return ret - def reconstruct(self): - if not isinstance(self.feature, CudaIpcTensorTransportProxy): - return + def has_cuda_ipc_proxy(self): + return ( + isinstance(self.feature, CudaIpcTensorTransportProxy) + or isinstance(self.precomputed_embeddings, CudaIpcTensorTransportProxy) + or any( + isinstance(value, CudaIpcTensorTransportProxy) + for value in self.model_specific_data.values() + ) + ) - reconstruct_device = torch.cuda.current_device() + def reconstruct(self, target_device: int): + """materialize cuda ipc proxy tensors in-place on target_device""" if isinstance(self.feature, CudaIpcTensorTransportProxy): - self.feature = self.feature.reconstruct_on_target_device(reconstruct_device) + self.feature = self.feature.reconstruct_on_target_device(target_device) if isinstance(self.precomputed_embeddings, CudaIpcTensorTransportProxy): self.precomputed_embeddings = ( - self.precomputed_embeddings.reconstruct_on_target_device( - reconstruct_device - ) + self.precomputed_embeddings.reconstruct_on_target_device(target_device) ) for extra_key in self.model_specific_data: if isinstance( @@ -363,21 +373,23 @@ def reconstruct(self): ): extra_data = self.model_specific_data[ extra_key - ].reconstruct_on_target_device(reconstruct_device) + ].reconstruct_on_target_device(target_device) self.model_specific_data[extra_key] = extra_data @dataclasses.dataclass class MultimodalProcessorOutput: - """Raw output from multimodal processors, before pad/hash computation. + """Raw output from multimodal processors before scheduler-side preparation (pad, hash). This is the typed replacement for the dict previously returned by - ``BaseMultimodalProcessor.process_mm_data_async``. Unlike - ``MultimodalInputs``, items here do NOT carry pad_value or hash yet. + ``BaseMultimodalProcessor.process_mm_data_async``. Preprocessed inputs may + already carry ``pad_value`` and ``hash`` to avoid hashing the same tensor once + per scheduler TP rank. """ mm_items: List[MultimodalDataItem] input_ids: Optional[List[int]] = None + padded_input_ids: Optional[List[int]] = None # image im_token_id: Optional[int] = None @@ -411,6 +423,7 @@ def from_dict(d: dict) -> "MultimodalProcessorOutput": return MultimodalProcessorOutput( mm_items=d["mm_items"], input_ids=d.get("input_ids"), + padded_input_ids=d.get("padded_input_ids"), im_token_id=d.get("im_token_id"), im_start_id=d.get("im_start_id"), im_end_id=d.get("im_end_id"), @@ -427,6 +440,26 @@ def from_dict(d: dict) -> "MultimodalProcessorOutput": visible_frame_counts=d.get("visible_frame_counts"), ) + @staticmethod + def build_padded_input_ids(input_ids, mm_items: List[MultimodalDataItem]): + """pad the input_ids with mm_items if it's not already padded""" + if input_ids is None or not mm_items: + return None + + for item in mm_items: + if item.pad_value is None or item.offsets is None: + return None + + if isinstance(input_ids, torch.Tensor): + padded_input_ids = input_ids.flatten().tolist() + else: + padded_input_ids = list(input_ids) + + for item in mm_items: + for start, end in item.offsets: + padded_input_ids[start : end + 1] = [item.pad_value] * (end - start + 1) + return padded_input_ids + @dataclasses.dataclass class MultimodalInputs: @@ -434,6 +467,7 @@ class MultimodalInputs: # items of data mm_items: List[MultimodalDataItem] + padded_input_ids: Optional[List[int]] = None image_pad_len: Optional[list] = None num_image_tokens: Optional[int] = None @@ -470,15 +504,16 @@ def release_features(self): @staticmethod def from_processor_output(obj: "MultimodalProcessorOutput"): mm_items = obj.mm_items - for mm_item in mm_items: - mm_item.reconstruct() - - ret = MultimodalInputs( - mm_items=mm_items, - ) + assert isinstance(mm_items, list) + mm_items = [item for item in mm_items if item.is_valid()] - assert isinstance(ret.mm_items, list) - ret.mm_items = [item for item in ret.mm_items if item.is_valid()] + # try reconstructing from cuda-ipc + reconstruct_device = None + for mm_item in mm_items: + if mm_item.has_cuda_ipc_proxy(): + if reconstruct_device is None: + reconstruct_device = torch.cuda.current_device() + mm_item.reconstruct(reconstruct_device) if envs.SGLANG_MM_BUFFER_SIZE_MB.get() > 0: # Multi-modal feature hashing optimization: @@ -495,19 +530,23 @@ def from_processor_output(obj: "MultimodalProcessorOutput"): if not is_feature_buffer_initialized(): init_feature_buffer(device) reset_buffer_offset() - for item in ret.mm_items: + for item in mm_items: if item.feature is not None: if isinstance(item.feature, torch.Tensor): item.feature = try_add_to_buffer(item.feature) - for item in ret.mm_items: + for item in mm_items: item.set_pad_value() if envs.SGLANG_MM_BUFFER_SIZE_MB.get() > 0: - for item in ret.mm_items: + for item in mm_items: if item.feature is not None: item.feature = item.feature.to("cpu", non_blocking=True) + mm_inputs = MultimodalInputs( + mm_items=mm_items, + padded_input_ids=obj.padded_input_ids, + ) optional_args = [ "mrope_positions", "mrope_position_delta", @@ -527,9 +566,9 @@ def from_processor_output(obj: "MultimodalProcessorOutput"): for arg in optional_args: val = getattr(obj, arg, None) if val is not None: - setattr(ret, arg, val) + setattr(mm_inputs, arg, val) - return ret + return mm_inputs def contains_image_inputs(self) -> bool: return any(item.is_image() for item in self.mm_items) @@ -612,14 +651,14 @@ def __init__( self, rid: str, origin_input_text: str, - origin_input_ids: List[int], + origin_input_ids: array[int], sampling_params: SamplingParams, return_logprob: bool = False, top_logprobs_num: int = 0, dllm_config: Optional[DllmConfig] = None, token_ids_logprob: List[int] = None, stream: bool = False, - origin_input_ids_unpadded: Optional[Tuple[int]] = None, + origin_input_ids_unpadded: Optional[array[int]] = None, lora_id: Optional[str] = None, input_embeds: Optional[List[List[float]]] = None, positional_embed_overrides: Optional[PositionalEmbeds] = None, @@ -659,16 +698,17 @@ def __init__( # --- BEGIN ENGRAM: request conversation ID storage --- self.conversation_id = conversation_id # --- END ENGRAM --- + self.origin_input_ids = array("q", origin_input_ids) self.origin_input_ids_unpadded = ( - origin_input_ids_unpadded + array("q", origin_input_ids_unpadded) if origin_input_ids_unpadded - else origin_input_ids # Before image padding - ) - self.origin_input_ids = origin_input_ids + else self.origin_input_ids + ) # Before image padding # Each decode stage's output ids - self.output_ids = [] + self.output_ids = array("q") # fill_ids = origin_input_ids + output_ids. Updated if chunked. - self.fill_ids = [] + self.fill_ids = array("q") + self.session = session self.input_embeds = input_embeds self.positional_embed_overrides = positional_embed_overrides @@ -955,7 +995,7 @@ def is_prefill_only(self) -> bool: return self.sampling_params.max_new_tokens == 0 and spec_alg is None @property - def output_ids_through_stop(self) -> List[int]: + def output_ids_through_stop(self) -> array[int]: """Get the output ids through the stop condition. Stop position is included.""" if self.finished_len is not None: return self.output_ids[: self.finished_len] @@ -1044,7 +1084,7 @@ def init_next_round_input( # Disable prefix caching when embed overrides are present: same token IDs # with different override vectors must not share cached KV values. if self.positional_embed_overrides is not None: - token_ids_to_match = [] + token_ids_to_match = array("q") if tree_cache is not None: if cow_mamba is None: @@ -1291,7 +1331,7 @@ def reset_for_retract(self): self.extend_logprob_start_len = 0 self.inflight_middle_chunks = 0 self.mamba_pool_idx = None - self.mamba_has_restored_state = False # --- BEGIN ENGRAM KHA-390 --- + self.mamba_has_restored_state = False # ENGRAM_CHANGED: KHA-390 mamba snapshot restore state self.mamba_ping_pong_track_buffer = None self.mamba_next_track_idx = None self.mamba_last_track_seqlen = None @@ -1313,7 +1353,7 @@ def reset_for_retract(self): # Therefore, we discard the generated output_ids and restart prefill and generation # to ensure shape consistency in KV cache. if self.input_embeds is not None: - self.output_ids = [] + self.output_ids = array("q") def offload_kv_cache(self, req_to_token_pool, token_to_kv_pool_allocator): token_indices = req_to_token_pool.req_to_token[ @@ -1378,7 +1418,9 @@ def set_finish_with_abort(self, error_msg: str): logger.error(f"{error_msg}, {self.rid=}") self.multimodal_inputs = None self.grammar = None - self.origin_input_ids = [0] # set it to one token to skip the long prefill + self.origin_input_ids = array( + "q", [0] + ) # set it to one token to skip the long prefill self.return_logprob = False self.logprob_start_len = -1 self.to_finish = FINISH_ABORT( @@ -1456,6 +1498,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # For chunked prefill in PP chunked_req: Optional[Req] = None + contains_last_prefill_chunk: bool = True # Sampling info sampling_info: SamplingBatchInfo = None @@ -1637,7 +1680,9 @@ def is_empty(self): def is_dllm(self): return self.dllm_config is not None - def prepare_encoder_info_extend(self, input_ids: List[int], seq_lens: List[int]): + def prepare_encoder_info_extend( + self, input_ids: List[array[int]], seq_lens: List[int] + ): _pin = is_pin_memory_available(self.device) self.encoder_lens_cpu = [] self.encoder_cached = [] @@ -1686,9 +1731,7 @@ def prepare_encoder_info_extend(self, input_ids: List[int], seq_lens: List[int]) pt += req.extend_input_len # Reassign - self.input_ids = torch.tensor( - sum(input_ids, []), dtype=torch.int64, pin_memory=_pin - ).to(self.device, non_blocking=True) + self.input_ids = flatten_arrays_to_int64_tensor(input_ids, self.device, _pin) self.seq_lens = torch.tensor(seq_lens, dtype=torch.int64, pin_memory=_pin).to( self.device, non_blocking=True ) @@ -1791,9 +1834,7 @@ def prepare_for_extend(self): ] _pin = is_pin_memory_available(self.device) - input_ids_tensor = torch.tensor( - list(chain.from_iterable(input_ids)), dtype=torch.int64, pin_memory=_pin - ).to(self.device, non_blocking=True) + input_ids_tensor = flatten_arrays_to_int64_tensor(input_ids, self.device, _pin) seq_lens_tensor = torch.tensor(seq_lens, dtype=torch.int64, pin_memory=_pin).to( self.device, non_blocking=True ) @@ -2059,18 +2100,19 @@ def _mamba_radix_cache_v2_req_prepare_for_extend( self, req: Req, ) -> "_MambaRadixCacheV2TrackEntry": + mamba_cache_chunk_size = get_global_server_args().mamba_cache_chunk_size + def _force_track_h(i: int) -> int: - assert i % FLA_CHUNK_SIZE == 0 + assert i % mamba_cache_chunk_size == 0 # There are 3 cases for mamba_track_seqlen passed to mamba_track_seqlens_cpu: - # 1) aligned with FLA_CHUNK_SIZE-> retrieve from last_recurrent_state + # 1) aligned with mamba_cache_chunk_size-> retrieve from last_recurrent_state # a) is the last position -> retrieve from last_recurrent_state # b) is NOT the last position -> retrieve from h - # 2) unaligned with FLA_CHUNK_SIZE -> retrieve from h + # 2) unaligned with mamba_cache_chunk_size -> retrieve from h # Currently, the math calculation only supports case 1a and 2. So for 1b, we need to add 1 # to force the math calculation to retrieve the correct mamba state from h. return i + 1 - mamba_cache_chunk_size = get_global_server_args().mamba_cache_chunk_size mask = req.extend_input_len >= mamba_cache_chunk_size track_index = req.mamba_ping_pong_track_buffer[req.mamba_next_track_idx].item() mamba_track_seqlen = -1 @@ -2092,13 +2134,14 @@ def _force_track_h(i: int) -> int: * mamba_cache_chunk_size ) - # mamba_track_fla_chunk_aligned is the aligned seqlen based on FLA_CHUNK_SIZE + # mamba_track_fla_chunk_aligned is the aligned seqlen based on mamba_cache_chunk_size # If mamba_track_fla_chunk_aligned != mamba_track_seqlen_aligned, which can be true when - # page_size > FLA_CHUNK_SIZE, we need to force the math calculation to retrieve the correct mamba state from h + # page_size > mamba_cache_chunk_size, we need to force the math calculation to retrieve the correct mamba state from h # by _force_track_h() mamba_track_fla_chunk_aligned = ( len(req.prefix_indices) - + (req.extend_input_len // FLA_CHUNK_SIZE) * FLA_CHUNK_SIZE + + (req.extend_input_len // mamba_cache_chunk_size) + * mamba_cache_chunk_size ) if mamba_track_fla_chunk_aligned != mamba_track_seqlen_aligned: # We want to track mamba_track_seqlen_aligned, and it's not the last position, @@ -2419,14 +2462,13 @@ def prepare_for_decode(self): req.kv_committed_len += 1 req.kv_allocated_len += 1 - # Update seq_lens after allocation if self.enable_overlap: - # Do not use in-place operations in the overlap mode + # New-tensor avoids racing model_worker_batch refs queued for + # overlap forward. self.seq_lens = self.seq_lens + 1 self.seq_lens_cpu = self.seq_lens_cpu + 1 self.orig_seq_lens = self.orig_seq_lens + 1 else: - # A faster in-place version self.seq_lens.add_(1) self.seq_lens_cpu.add_(1) self.orig_seq_lens.add_(1) diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index b6474603777f..1ed7bd9ff437 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from array import array from sglang.srt.environ import envs from sglang.srt.managers.prefill_delayer import PrefillDelayerSinglePassExecutor @@ -84,7 +85,7 @@ def match_prefix_for_req( tree_cache: BasePrefixCache, req: Req, - token_ids: Optional[List[int]] = None, + token_ids: Optional[array[int]] = None, *, cow_mamba: bool = False, include_req: bool = False, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 5e8091326247..5fa8c0bfab36 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -22,6 +22,7 @@ import signal import sys import time +from array import array from collections import deque from contextlib import contextmanager, nullcontext from functools import partial @@ -158,7 +159,6 @@ UpdateWeightsFromTensorReqInput, ) from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors -from sglang.srt.managers.overlap_utils import FutureIndices from sglang.srt.managers.prefill_delayer import ( PrefillDelayer, PrefillDelayerSinglePassExecutor, @@ -545,11 +545,7 @@ def __init__( self.init_watch_dog_memory_saver_input_blocker() # Init profiler - self.profiler_manager = SchedulerProfilerManager( - ps=self.ps, - dp_tp_cpu_group=self.dp_tp_cpu_group, - get_forward_ct=lambda: self.forward_ct, - ) + self.init_profiler() # Init prefill-decodedisaggregation self.init_disaggregation() @@ -563,182 +559,35 @@ def __init__( # Init prefill kv split size when deterministic inference is enabled with various attention backends self.init_deterministic_inference_config() - self.weight_updater = SchedulerWeightUpdaterManager( - tp_worker=self.tp_worker, - draft_worker=self.draft_worker, - tp_cpu_group=self.tp_cpu_group, - memory_saver_adapter=self.memory_saver_adapter, - flush_cache=self.flush_cache, - is_fully_idle=self.is_fully_idle, - ) + self.init_weight_updater() # Init request dispatcher self.init_request_dispatcher() # Init LoRA drainer for fair scheduling - if self.server_args.lora_drain_wait_threshold > 0.0: - self.lora_drainer = LoRADrainer( - server_args.max_loras_per_batch, - server_args.lora_drain_wait_threshold, - ) - else: - self.lora_drainer = None + self.init_lora_drainer() # Init LoRA overlap loader - if self.enable_lora_overlap_loading: - self.lora_overlap_loader = LoRAOverlapLoader( - self.tp_worker.model_runner.lora_manager - ) + self.init_lora_overlap_loader() # Init the grammar backend for constrained generation - self.grammar_manager = GrammarManager(self) + self.init_grammar_manager() - self.request_receiver = SchedulerRequestReceiver( - recv_from_tokenizer=self.ipc_channels.recv_from_tokenizer, - recv_from_rpc=self.ipc_channels.recv_from_rpc, - recv_skipper=self.recv_skipper, - input_blocker=self.input_blocker, - mm_receiver=self.mm_receiver, - ps=self.ps, - tp_group=self.tp_group, - tp_cpu_group=self.tp_cpu_group, - attn_tp_group=self.attn_tp_group, - attn_tp_cpu_group=self.attn_tp_cpu_group, - attn_cp_group=self.attn_cp_group, - attn_cp_cpu_group=self.attn_cp_cpu_group, - world_group=self.world_group, - server_args=self.server_args, - model_config=self.model_config, - max_recv_per_poll=self.max_recv_per_poll, - stream_output=lambda *a, **kw: self.output_streamer.stream_output(*a, **kw), - get_last_forward_mode=lambda: ( - self.last_batch.forward_mode if self.last_batch is not None else None - ), - ) + self.init_request_receiver() - self.dp_attn_adapter = SchedulerDPAttnAdapter( - tp_group=self.tp_group, - req_to_token_pool=self.req_to_token_pool, - token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - tree_cache=self.tree_cache, - offload_tags=self.weight_updater.offload_tags, - ps=self.ps, - server_args=self.server_args, - model_config=self.model_config, - enable_overlap=self.enable_overlap, - spec_algorithm=self.spec_algorithm, - get_require_mlp_sync=lambda: self.require_mlp_sync, - ) + self.init_dp_attn_adapter() - self.pool_stats_observer = SchedulerPoolStatsObserver( - tree_cache=self.tree_cache, - token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - req_to_token_pool=self.req_to_token_pool, - session_controller=self.session_controller, - hisparse_coordinator=self.hisparse_coordinator, - is_hybrid_swa=self.is_hybrid_swa, - is_hybrid_ssm=self.is_hybrid_ssm, - enable_hisparse=self.enable_hisparse, - full_tokens_per_layer=self.full_tokens_per_layer, - swa_tokens_per_layer=self.swa_tokens_per_layer, - max_total_num_tokens=self.max_total_num_tokens, - get_last_batch=lambda: self.last_batch, - get_running_batch=lambda: self.running_batch, - ) + self.init_pool_stats_observer() - self.invariant_checker = SchedulerInvariantChecker( - is_hybrid_swa=self.is_hybrid_swa, - is_hybrid_ssm=self.is_hybrid_ssm, - disaggregation_mode=self.disaggregation_mode, - page_size=self.page_size, - full_tokens_per_layer=self.full_tokens_per_layer, - swa_tokens_per_layer=self.swa_tokens_per_layer, - max_total_num_tokens=self.max_total_num_tokens, - server_args=self.server_args, - tree_cache=self.tree_cache, - token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - req_to_token_pool=self.req_to_token_pool, - pool_stats_observer=self.pool_stats_observer, - get_last_batch=lambda: self.last_batch, - get_running_batch=lambda: self.running_batch, - ) + self.init_invariant_checker() - self.kv_events_publisher = SchedulerKvEventsPublisher( - kv_events_config=self.server_args.kv_events_config, - ps=self.ps, - attn_tp_rank=self.ps.attn_tp_rank, - attn_cp_rank=self.ps.attn_cp_rank, - attn_dp_rank=self.ps.attn_dp_rank, - dp_rank=self.ps.dp_rank, - tree_cache=self.tree_cache, - send_metrics_from_scheduler=self.ipc_channels.send_metrics_from_scheduler, - max_running_requests=self.max_running_requests, - max_total_num_tokens=self.max_total_num_tokens, - get_stats=lambda: self.metrics_reporter.stats, - ) + self.init_kv_events_publisher() - self.load_inquirer = SchedulerLoadInquirer( - disaggregation_mode=self.disaggregation_mode, - ps=self.ps, - server_args=self.server_args, - max_total_num_tokens=self.max_total_num_tokens, - max_running_requests=self.max_running_requests, - pool_stats_observer=self.pool_stats_observer, - tp_worker=self.tp_worker, - token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - spec_algorithm=self.spec_algorithm, - get_running_batch=lambda: self.running_batch, - get_waiting_queue=lambda: self.waiting_queue, - get_stats=lambda: self.metrics_reporter.stats, - get_chunked_req=lambda: self.chunked_req, - get_disagg_prefill_bootstrap_queue=lambda: self.disagg_prefill_bootstrap_queue, - get_disagg_prefill_inflight_queue=lambda: self.disagg_prefill_inflight_queue, - get_disagg_decode_prealloc_queue=lambda: self.disagg_decode_prealloc_queue, - get_disagg_decode_transfer_queue=lambda: self.disagg_decode_transfer_queue, - get_spec_total_num_accept_tokens=lambda: self.metrics_reporter.spec_total_num_accept_tokens, - get_spec_total_num_forward_ct=lambda: self.metrics_reporter.spec_total_num_forward_ct, - ) + self.init_load_inquirer() - self.output_streamer = SchedulerOutputStreamer( - send_to_detokenizer=self.ipc_channels.send_to_detokenizer, - tree_cache=self.tree_cache, - ps=self.ps, - server_args=self.server_args, - is_generation=self.is_generation, - spec_algorithm=self.spec_algorithm, - disaggregation_mode=self.disaggregation_mode, - enable_hicache_storage=lambda: self.enable_hicache_storage, - load_inquirer_get_loads=lambda req: self.load_inquirer.get_loads(req), - # --- BEGIN ENGRAM: send_to_tokenizer wiring for M3 stateful-generate output routing --- - send_to_tokenizer=self.ipc_channels.send_to_tokenizer, - # --- END ENGRAM --- - ) + self.init_output_streamer() - self.batch_result_processor = SchedulerBatchResultProcessor( - is_generation=self.is_generation, - disaggregation_mode=self.disaggregation_mode, - enable_overlap=self.enable_overlap, - enable_overlap_mlx=self.enable_overlap_mlx, - server_args=self.server_args, - model_config=self.model_config, - token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - tree_cache=self.tree_cache, - hisparse_coordinator=self.hisparse_coordinator, - req_to_token_pool=self.req_to_token_pool, - decode_offload_manager=self.decode_offload_manager, - metrics_collector=self.metrics_collector, - metrics_reporter=self.metrics_reporter, - draft_worker=self.draft_worker, - model_worker=self.model_worker, - logprob_result_processor=SchedulerLogprobResultProcessor( - server_args=self.server_args, model_config=self.model_config - ), - output_streamer=self.output_streamer, - abort_request=self.abort_request, - # --- BEGIN ENGRAM: snapshot_hook_manager wiring for M2 hook --- - snapshot_hook_manager=self.snapshot_hook_manager, - # --- END ENGRAM --- - ) + self.init_batch_result_processor() self.is_initializing = False @@ -1001,6 +850,7 @@ def init_model_worker(self): ) self.dp_tp_cpu_group = self.dp_tp_group.cpu_group + # TODO(Jialin): Migrate pad_input_ids implementations to return array. self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func() set_random_seed(self.random_seed) @@ -1743,6 +1593,196 @@ def process_input_requests(self, recv_reqs: List): if self.external_corpus_manager is not None: self.external_corpus_manager.check_pending_load() + def init_profiler(self) -> None: + self.profiler_manager = SchedulerProfilerManager( + ps=self.ps, + dp_tp_cpu_group=self.dp_tp_cpu_group, + get_forward_ct=lambda: self.forward_ct, + ) + + def init_weight_updater(self) -> None: + self.weight_updater = SchedulerWeightUpdaterManager( + tp_worker=self.tp_worker, + draft_worker=self.draft_worker, + tp_cpu_group=self.tp_cpu_group, + memory_saver_adapter=self.memory_saver_adapter, + flush_cache=self.flush_cache, + is_fully_idle=self.is_fully_idle, + ) + + def init_lora_drainer(self) -> None: + if self.server_args.lora_drain_wait_threshold > 0.0: + self.lora_drainer = LoRADrainer( + self.server_args.max_loras_per_batch, + self.server_args.lora_drain_wait_threshold, + ) + else: + self.lora_drainer = None + + def init_lora_overlap_loader(self) -> None: + if self.enable_lora_overlap_loading: + self.lora_overlap_loader = LoRAOverlapLoader( + self.tp_worker.model_runner.lora_manager + ) + + def init_grammar_manager(self) -> None: + self.grammar_manager = GrammarManager(self) + + def init_request_receiver(self) -> None: + self.request_receiver = SchedulerRequestReceiver( + recv_from_tokenizer=self.ipc_channels.recv_from_tokenizer, + recv_from_rpc=self.ipc_channels.recv_from_rpc, + recv_skipper=self.recv_skipper, + input_blocker=self.input_blocker, + mm_receiver=self.mm_receiver, + ps=self.ps, + tp_group=self.tp_group, + tp_cpu_group=self.tp_cpu_group, + attn_tp_group=self.attn_tp_group, + attn_tp_cpu_group=self.attn_tp_cpu_group, + attn_cp_group=self.attn_cp_group, + attn_cp_cpu_group=self.attn_cp_cpu_group, + world_group=self.world_group, + server_args=self.server_args, + model_config=self.model_config, + max_recv_per_poll=self.max_recv_per_poll, + stream_output=lambda *a, **kw: self.output_streamer.stream_output(*a, **kw), + get_last_forward_mode=lambda: ( + self.last_batch.forward_mode if self.last_batch is not None else None + ), + ) + + def init_dp_attn_adapter(self) -> None: + self.dp_attn_adapter = SchedulerDPAttnAdapter( + tp_group=self.tp_group, + req_to_token_pool=self.req_to_token_pool, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + tree_cache=self.tree_cache, + offload_tags=self.weight_updater.offload_tags, + ps=self.ps, + server_args=self.server_args, + model_config=self.model_config, + enable_overlap=self.enable_overlap, + spec_algorithm=self.spec_algorithm, + get_require_mlp_sync=lambda: self.require_mlp_sync, + ) + + def init_pool_stats_observer(self) -> None: + self.pool_stats_observer = SchedulerPoolStatsObserver( + tree_cache=self.tree_cache, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + req_to_token_pool=self.req_to_token_pool, + session_controller=self.session_controller, + hisparse_coordinator=self.hisparse_coordinator, + is_hybrid_swa=self.is_hybrid_swa, + is_hybrid_ssm=self.is_hybrid_ssm, + enable_hisparse=self.enable_hisparse, + full_tokens_per_layer=self.full_tokens_per_layer, + swa_tokens_per_layer=self.swa_tokens_per_layer, + max_total_num_tokens=self.max_total_num_tokens, + get_last_batch=lambda: self.last_batch, + get_running_batch=lambda: self.running_batch, + ) + + def init_invariant_checker(self) -> None: + self.invariant_checker = SchedulerInvariantChecker( + is_hybrid_swa=self.is_hybrid_swa, + is_hybrid_ssm=self.is_hybrid_ssm, + disaggregation_mode=self.disaggregation_mode, + page_size=self.page_size, + full_tokens_per_layer=self.full_tokens_per_layer, + swa_tokens_per_layer=self.swa_tokens_per_layer, + max_total_num_tokens=self.max_total_num_tokens, + server_args=self.server_args, + tree_cache=self.tree_cache, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + req_to_token_pool=self.req_to_token_pool, + pool_stats_observer=self.pool_stats_observer, + get_last_batch=lambda: self.last_batch, + get_running_batch=lambda: self.running_batch, + ) + + def init_kv_events_publisher(self) -> None: + self.kv_events_publisher = SchedulerKvEventsPublisher( + kv_events_config=self.server_args.kv_events_config, + ps=self.ps, + attn_tp_rank=self.ps.attn_tp_rank, + attn_cp_rank=self.ps.attn_cp_rank, + attn_dp_rank=self.ps.attn_dp_rank, + dp_rank=self.ps.dp_rank, + tree_cache=self.tree_cache, + send_metrics_from_scheduler=self.ipc_channels.send_metrics_from_scheduler, + max_running_requests=self.max_running_requests, + max_total_num_tokens=self.max_total_num_tokens, + get_stats=lambda: self.metrics_reporter.stats, + ) + + def init_load_inquirer(self) -> None: + self.load_inquirer = SchedulerLoadInquirer( + disaggregation_mode=self.disaggregation_mode, + ps=self.ps, + server_args=self.server_args, + max_total_num_tokens=self.max_total_num_tokens, + max_running_requests=self.max_running_requests, + pool_stats_observer=self.pool_stats_observer, + tp_worker=self.tp_worker, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + spec_algorithm=self.spec_algorithm, + get_running_batch=lambda: self.running_batch, + get_waiting_queue=lambda: self.waiting_queue, + get_stats=lambda: self.metrics_reporter.stats, + get_chunked_req=lambda: self.chunked_req, + get_disagg_prefill_bootstrap_queue=lambda: self.disagg_prefill_bootstrap_queue, + get_disagg_prefill_inflight_queue=lambda: self.disagg_prefill_inflight_queue, + get_disagg_decode_prealloc_queue=lambda: self.disagg_decode_prealloc_queue, + get_disagg_decode_transfer_queue=lambda: self.disagg_decode_transfer_queue, + get_spec_total_num_accept_tokens=lambda: self.metrics_reporter.spec_total_num_accept_tokens, + get_spec_total_num_forward_ct=lambda: self.metrics_reporter.spec_total_num_forward_ct, + ) + + def init_output_streamer(self) -> None: + self.output_streamer = SchedulerOutputStreamer( + send_to_detokenizer=self.ipc_channels.send_to_detokenizer, + tree_cache=self.tree_cache, + ps=self.ps, + server_args=self.server_args, + is_generation=self.is_generation, + spec_algorithm=self.spec_algorithm, + disaggregation_mode=self.disaggregation_mode, + enable_hicache_storage=lambda: self.enable_hicache_storage, + load_inquirer_get_loads=lambda req: self.load_inquirer.get_loads(req), + # --- BEGIN ENGRAM: send_to_tokenizer wiring for M3 stateful-generate output routing --- + send_to_tokenizer=self.ipc_channels.send_to_tokenizer, + # --- END ENGRAM --- + ) + + def init_batch_result_processor(self) -> None: + self.batch_result_processor = SchedulerBatchResultProcessor( + is_generation=self.is_generation, + disaggregation_mode=self.disaggregation_mode, + enable_overlap=self.enable_overlap, + enable_overlap_mlx=self.enable_overlap_mlx, + server_args=self.server_args, + model_config=self.model_config, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + tree_cache=self.tree_cache, + hisparse_coordinator=self.hisparse_coordinator, + req_to_token_pool=self.req_to_token_pool, + decode_offload_manager=self.decode_offload_manager, + metrics_collector=self.metrics_collector, + metrics_reporter=self.metrics_reporter, + draft_worker=self.draft_worker, + model_worker=self.model_worker, + logprob_result_processor=SchedulerLogprobResultProcessor( + server_args=self.server_args, model_config=self.model_config + ), + output_streamer=self.output_streamer, + abort_request=self.abort_request, + # --- BEGIN ENGRAM: snapshot_hook_manager wiring for M2 hook --- + snapshot_hook_manager=self.snapshot_hook_manager, + # --- END ENGRAM --- + ) + def init_req_max_new_tokens(self, req): input_len = len(req.origin_input_ids) # Keep this bound consistent with PrefillAdder's admission budget: @@ -1838,6 +1878,29 @@ def _get_multimodal_inputs(self, mm_inputs_dict): else: return MultimodalInputs.from_processor_output(mm_inputs_dict) + @staticmethod + def _try_apply_padded_mm_input_ids(recv_req, req, image_inputs) -> bool: + """setup origin_input_ids with trying to reuse existing MultimodalInputs.padded_input_ids first, + if absent, call pad_input_ids_func""" + padded_input_ids = image_inputs.padded_input_ids + if padded_input_ids is None or recv_req.input_ids is None: + return False + + recv_input_len = len(recv_req.input_ids) + if len(padded_input_ids) != recv_input_len: + return False + + prefix_len = len(req.origin_input_ids) - recv_input_len + if prefix_len < 0: + return False + + padded_input_ids = array("q", padded_input_ids) + if prefix_len == 0: + req.origin_input_ids = padded_input_ids + else: + req.origin_input_ids = req.origin_input_ids[:prefix_len] + padded_input_ids + return True + def _maybe_compute_mrope_positions(self, req) -> None: """Compute M-RoPE positions when they are missing (e.g. gRPC preprocessed path).""" if self._mm_processor is None: @@ -1880,8 +1943,7 @@ def handle_generate_request( if recv_req.input_embeds is not None: # Generate fake input_ids based on the length of input_embeds seq_length = len(recv_req.input_embeds) - fake_input_ids = [1] * seq_length - recv_req.input_ids = fake_input_ids + recv_req.input_ids = array("q", [1]) * seq_length if recv_req.bootstrap_port is None: # Use default bootstrap port @@ -2009,9 +2071,12 @@ def handle_generate_request( # The following steps are already fast, execute locally on each rank. # Expand a single image token into multiple dummy tokens for receiving image embeddings. # The pad function is model-specific and can be None for some backends. - if self.pad_input_ids_func: - req.origin_input_ids = self.pad_input_ids_func( - req.origin_input_ids, image_inputs + if ( + not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs) + and self.pad_input_ids_func + ): + req.origin_input_ids = array( + "q", self.pad_input_ids_func(req.origin_input_ids, image_inputs) ) req.extend_image_inputs(image_inputs) self._maybe_compute_mrope_positions(req) @@ -2305,9 +2370,13 @@ def handle_embedding_request( # The `pad_input_ids_func` is model-specific and may be None for # embedding models or models not requiring special padding. # If None, `req.origin_input_ids` is expected to be correctly populated already. - if self.pad_input_ids_func: - req.origin_input_ids = self.pad_input_ids_func( - req.origin_input_ids, image_inputs + if ( + not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs) + and self.pad_input_ids_func + ): + # See companion call site above for the array.array wrap rationale. + req.origin_input_ids = array( + "q", self.pad_input_ids_func(req.origin_input_ids, image_inputs) ) req.extend_image_inputs(image_inputs) @@ -2728,6 +2797,11 @@ def _get_new_batch_prefill_raw( self.spec_algorithm, chunked_req=self.chunked_req, ) + + new_batch.contains_last_prefill_chunk = ( + self.chunked_req is None or len(can_run_list) != 1 + ) + self.max_prefill_bs = max(self.max_prefill_bs, len(can_run_list)) if self.enable_hierarchical_cache: # todo (zhiqiang): disable cuda graph execution if hicache loading triggered @@ -2965,25 +3039,18 @@ def run_batch( # Run forward if self.is_generation: if self.enable_overlap: - # Spec v2 pre-isolation CPU mirror prep: D2H new_seq_lens_buf - # into batch.seq_lens_cpu + set seq_lens_sum. For non-spec_v2, - # ForwardBatch.init_new lazily computes the sum. - if batch.is_spec_v2: - # FIXME: make this optional to different backends. - self.future_map.resolve_seq_lens_cpu(batch) + # Self-gates on batch.spec_info.future_indices; non-spec_v2 + # no-ops (ForwardBatch.init_new lazily computes the sum). + self.future_map.resolve_seq_lens_cpu(batch) with self._overlap_forward_isolation(batch): - future_indices = FutureIndices(indices=batch.req_pool_indices) + future_indices = batch.req_pool_indices - # Spec_v2 worker fires this between sample-end and - # draft_extend; publish moves the fence to verify-end so - # schedule prep can overlap with draft_extend. + # Spec_v2 fires on_publish mid-worker (between verify and + # draft_extend) so schedule prep can overlap with draft_extend. + # Non-spec has no later work — scheduler publishes after return. fwd_kwargs = ( - { - "on_verify_complete": partial( - self.future_map.publish, future_indices - ) - } + {"on_publish": partial(self.future_map.publish, future_indices)} if batch.is_spec_v2 else {} ) @@ -2995,6 +3062,8 @@ def run_batch( batch_result = self.model_worker.forward_batch_generation( batch, **fwd_kwargs ) + if not batch.is_spec_v2: + self.future_map.publish(future_indices, batch.seq_lens + 1) # Park any refs the worker wants kept alive 2 iters # (cross-stream tensor lifetime; pinned in the same # ring slot as the SB attr snapshot). @@ -3018,16 +3087,11 @@ def run_batch( else: batch_result.future_indices = future_indices - # Placeholder for next iter's resolve_future to look up the - # real token from token_ids_buf via the negated indices. - batch.input_ids = -future_indices.indices + self.future_map.set_input_ids_sentinel(batch, future_indices) if batch.is_spec_v2: batch.spec_info = batch_result.next_draft_input batch.spec_info.future_indices = future_indices - # Schedule-stream sentinel between iters; next iter's - # resolve_future reassigns batch.seq_lens from new_seq_lens_buf. - batch.seq_lens = -future_indices.indices elif self.enable_pdmux and batch.forward_mode.is_split_prefill(): batch_result = self.tp_worker.forward_batch_split_prefill(batch) if isinstance(batch_result.next_token_ids, torch.Tensor): @@ -3987,6 +4051,13 @@ def run_scheduler_process( traceback = get_exception_traceback() logger.error(f"Scheduler hit an exception: {traceback}") parent_process.send_signal(signal.SIGQUIT) + # Opt-in: SIGKILL the pgroup so sibling ranks don't spew thousands + # of NCCL/TCPStore tracebacks before they finally die. + if envs.SGLANG_KILLPG_ON_SCHEDULER_EXCEPTION.get(): + try: + os.killpg(os.getpgrp(), signal.SIGKILL) + except Exception: + pass finally: if scheduler is not None: # FPM has a background ZMQ publisher thread that needs explicit diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index 35dd5370b664..5c7b8be88ed3 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -125,25 +125,27 @@ def _maybe_collect_routed_experts(self, req: Req): if capturer is None: return start_len = req.routed_experts_start_len + seqlen = len(req.origin_input_ids) + len(req.output_ids_through_stop) req.routed_experts = capturer.get_topk( req_pool_idx=req.req_pool_idx, - seqlen=req.seqlen, + seqlen=seqlen, req_to_token_pool=self.req_to_token_pool, start_len=start_len, ) - expected_rows = max(0, req.seqlen - 1 - start_len) + expected_rows = max(0, seqlen - 1 - start_len) if ( req.routed_experts is not None and req.routed_experts.shape[0] != expected_rows ): logger.warning( - "routed_experts row-count mismatch for req %s: got %d, " - "expected %d (seqlen=%d, cached_tokens=%d, start_len=%s). " + "routed_experts row-count mismatch for req %s: got %d, expected %d " + "(seqlen=%d, raw_seqlen=%d, cached_tokens=%d, start_len=%s). " "This indicates a silent bug.", req.rid, req.routed_experts.shape[0], expected_rows, + seqlen, req.seqlen, req.cached_tokens, req.routed_experts_start_len, @@ -153,9 +155,10 @@ def _maybe_collect_indexer_topk(self, req: Req): capturer = get_global_indexer_capturer() if capturer is None: return + seqlen = len(req.origin_input_ids) + len(req.output_ids_through_stop) req.indexer_topk = capturer.get_topk( req_pool_idx=req.req_pool_idx, - seqlen=req.seqlen, + seqlen=seqlen, req_to_token_pool=self.req_to_token_pool, ) @@ -213,6 +216,8 @@ def process_batch_result_prefill( next_token_ids = next_token_ids.tolist() self._move_logprobs_to_cpu(batch=batch, logits_output=logits_output) + self._validate_pp_skip_output_comm(batch, result) + hidden_state_offset = 0 # Check finish conditions @@ -420,6 +425,47 @@ def _apply_prefill_logprobs( logprob_pt += num_input_logprobs return logprob_pt + @staticmethod + def _validate_pp_skip_output_comm( + batch: ScheduleBatch, + result: Union[GenerationBatchResult, EmbeddingBatchResult], + ): + """Validate PP skip output comm correctness. + + - When skip=True: all reqs must be middle chunks (inflight_middle_chunks > 0) + so placeholder zeros are never consumed via req.output_ids.append(). + - When skip=False: at least one req should consume next_token_ids + (inflight_middle_chunks <= 0), otherwise warn. + """ + if not envs.SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM.get(): + return + + if not getattr(result, "skipped_output_comm", False): + if batch.forward_mode.is_extend() and not batch.forward_mode.is_prebuilt(): + has_consumed_output = any( + req.inflight_middle_chunks <= 0 + for req in batch.reqs + if not req.finished() and not req.is_retracted + ) + if not has_consumed_output and len(batch.reqs) > 0: + chunks = list([r.inflight_middle_chunks for r in batch.reqs]) + logger.warning( + f"PP non-skip output comm: no req consumed next_token_ids. " + f"contains_last_prefill_chunk={batch.contains_last_prefill_chunk}, " + f"num_reqs={len(batch.reqs)}, all inflight_middle_chunks={chunks}" + ) + return + + for req in batch.reqs: + if not req.finished() and not req.is_retracted: + assert req.inflight_middle_chunks > 0, ( + f"PP skip output comm invariant violated: req {req.rid} " + f"has inflight_middle_chunks={req.inflight_middle_chunks} " + f"but output was skipped (contains_last_prefill_chunk=" + f"{batch.contains_last_prefill_chunk}). " + f"Placeholder zeros would be appended to output_ids." + ) + def _append_prefill_hidden_states( self, *, diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index bdda2e05819f..b032bd71aee3 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -147,7 +147,11 @@ def prepare_mlp_sync_batch_raw( offload_tags: set[str], ): # Check if other DP workers have running batches - if local_batch is None or local_batch.forward_mode.is_prebuilt(): + if ( + local_batch is None + or local_batch.forward_mode.is_prebuilt() + or local_batch.forward_mode.is_idle() + ): num_tokens = 0 num_tokens_for_logprob = 0 elif local_batch.forward_mode.is_decode(): diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index e04ca73a68d8..b802dda4303a 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -3,6 +3,7 @@ import logging import math import time +from array import array from collections import defaultdict, deque from dataclasses import dataclass from typing import TYPE_CHECKING, Dict, List, Optional, Tuple @@ -28,7 +29,11 @@ get_logprob_dict_from_result, get_logprob_from_pp_outputs, ) -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.model_executor.forward_batch_info import ( + ForwardBatch, + ForwardMode, + PPProxyTensors, +) from sglang.srt.observability.req_time_stats import set_time_batch from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.utils import DynamicGradMode, broadcast_pyobj, point_to_point_pyobj @@ -40,6 +45,18 @@ from sglang.srt.managers.scheduler import Scheduler +def _pp_can_skip_output_comm(batch: ScheduleBatch) -> bool: + """Check if output send/recv can be skipped for this batch.""" + return ( + envs.SGLANG_PP_SKIP_PURE_CHUNKED_OUTPUT_COMM.get() + and batch is not None + and batch.forward_mode == ForwardMode.EXTEND + and len(batch.reqs) == 1 + and not batch.contains_last_prefill_chunk + and not batch.return_logprob + ) + + @dataclass class PPBatchMetadata: can_run_cuda_graph: bool @@ -556,7 +573,7 @@ def profile_and_init_predictor(self: Scheduler): if self.pp_group.is_first_rank: model_runner = self.tp_worker.model_runner model_config = model_runner.model_config - input_ids_list = [] + input_ids_list: List[array[int]] = [] for i in range(128): chunk_size = int( self.chunked_prefill_size * 1.25 @@ -564,9 +581,12 @@ def profile_and_init_predictor(self: Scheduler): ) if chunk_size <= 0: break - input_ids = np.random.randint( - 0, 10000, size=chunk_size, dtype=np.int64 - ).tolist() + input_ids = array( + "q", + np.random.randint( + 0, 10000, size=chunk_size, dtype=np.int64 + ).tobytes(), + ) input_ids_list.append(input_ids) sampling_params = SamplingParams( @@ -1028,6 +1048,30 @@ def _pp_recv_dict_from_prev_stage( ), ) + def _pp_make_skip_output_result( + self: Scheduler, + batch: ScheduleBatch, + mb_metadata: Optional[PPBatchMetadata], + ): + bs = len(batch.reqs) + placeholder = torch.zeros(bs, dtype=torch.int64, device=self.device) + # next_pp_outputs = None so non-last ranks skip forwarding + # (pp_outputs is None gate). Placeholder carried in + # batch_result.next_token_ids for process_batch_result_prefill. + batch.output_ids = placeholder + batch_result = GenerationBatchResult( + logits_output=None, + pp_hidden_states_proxy_tensors=None, + next_token_ids=placeholder, + can_run_cuda_graph=( + mb_metadata.can_run_cuda_graph if mb_metadata else False + ), + skipped_output_comm=True, + ) + d2h_event = self.device_module.Event() + d2h_event.record(self.device_module.current_stream()) + return None, batch_result, d2h_event + def _pp_prep_batch_result( self: Scheduler, batch: ScheduleBatch, @@ -1072,9 +1116,13 @@ def _pp_send_output_to_next_stage( send_output_work = [] if self.pp_group.is_last_rank: # send ready PP output to rank 0 - if mbs[next_first_rank_mb_id] is not None: + target = mbs[next_first_rank_mb_id] + if target is not None: q_event, pp_outputs_to_send = last_rank_comm_queue.popleft() - if not mbs[next_first_rank_mb_id].forward_mode.is_prebuilt(): + if ( + not target.forward_mode.is_prebuilt() + and not _pp_can_skip_output_comm(target) + ): self.device_module.current_stream().wait_event(q_event) with torch.profiler.record_function("send_res_dict_to_next_stage"): send_output_work = self._pp_send_dict_to_next_stage( @@ -1135,14 +1183,20 @@ def _do_send(): def _do_recv(): nonlocal next_pp_outputs, batch_result, d2h_event - if mbs[next_mb_id] is None or mbs[next_mb_id].forward_mode.is_prebuilt(): + target = mbs[next_mb_id] + if target is None or target.forward_mode.is_prebuilt(): + return + if _pp_can_skip_output_comm(target): + next_pp_outputs, batch_result, d2h_event = ( + self._pp_make_skip_output_result(target, mb_metadata[next_mb_id]) + ) return with torch.profiler.record_function("recv_res_dict_from_prev_stage"): next_pp_outputs = PPProxyTensors(self._pp_recv_dict_from_prev_stage()) with self.copy_stream_ctx: self.copy_stream.wait_stream(self.schedule_stream) batch_result = self._pp_prep_batch_result( - mbs[next_mb_id], mb_metadata[next_mb_id], next_pp_outputs + target, mb_metadata[next_mb_id], next_pp_outputs ) d2h_event = self.device_module.Event() d2h_event.record(self.device_module.current_stream()) diff --git a/python/sglang/srt/managers/template_detection.py b/python/sglang/srt/managers/template_detection.py index 0198e75f0ca3..190cec67dbd9 100644 --- a/python/sglang/srt/managers/template_detection.py +++ b/python/sglang/srt/managers/template_detection.py @@ -219,6 +219,12 @@ def _is_minimax(ctx): return ctx.has_text("") +def _is_minicpm5(ctx): + if ctx.has_vocab(" Union[TokenizedGenerateReqInput, TokenizedEmbeddingReqInput]: """Create a tokenized request object from common parameters.""" + input_ids_arr: Optional[array[int]] = ( + array("q", input_ids) if input_ids is not None else None + ) # Parse sampling parameters # Note: if there are preferred sampling params, we use them if they are not # explicitly passed in sampling_params @@ -1069,7 +1084,7 @@ def _create_tokenized_object( tokenized_obj = TokenizedGenerateReqInput( input_text, - input_ids, + input_ids_arr, mm_inputs, sampling_params, obj.return_logprob, @@ -1114,12 +1129,12 @@ def _create_tokenized_object( and obj.embed_override_token_id is not None ): positional_embed_overrides = self._resolve_embed_overrides( - input_ids, obj.embed_override_token_id, obj.embed_overrides + input_ids_arr, obj.embed_override_token_id, obj.embed_overrides ) tokenized_obj = TokenizedEmbeddingReqInput( input_text, - input_ids, + input_ids_arr, mm_inputs, token_type_ids, sampling_params, @@ -1140,7 +1155,7 @@ def _create_tokenized_object( @staticmethod def _resolve_embed_overrides( - input_ids: List[int], + input_ids: array[int], token_id: int, embeds: List[torch.Tensor], ) -> PositionalEmbeds: @@ -1877,7 +1892,7 @@ async def _handle_batch_output( self.server_args.incremental_streaming_output and is_stream ) delta_text = recv_obj.output_strs[i] - delta_output_ids = recv_obj.output_ids[i] + delta_output_ids = list(recv_obj.output_ids[i]) output_offset = state.last_output_offset state.append_text(delta_text) state.output_ids.extend(delta_output_ids) @@ -1920,7 +1935,7 @@ async def _handle_batch_output( incremental = ( self.server_args.incremental_streaming_output and is_stream ) - delta_output_ids = recv_obj.output_ids[i] + delta_output_ids = list(recv_obj.output_ids[i]) output_offset = state.last_output_offset state.output_ids.extend(delta_output_ids) diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index 773e61f67726..f552102b8641 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -484,7 +484,7 @@ def forward_batch_generation( ) if is_verify: - # Skip sampling and return logits for target forward + # Skip sampling; spec_v2 worker fires its own publish post-verify. return batch_result if ( diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 33db3942b107..67de785964bb 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -10,7 +10,6 @@ from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX from sglang.srt.eplb.expert_distribution import ExpertDistributionMetrics from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.managers.overlap_utils import FutureIndices from sglang.srt.managers.schedule_batch import Req from sglang.srt.model_executor.forward_batch_info import PPProxyTensors from sglang.srt.server_args import ServerArgs @@ -33,6 +32,11 @@ class GenerationBatchResult: num_correct_drafts_per_req_cpu: Optional[List[int]] = None can_run_cuda_graph: bool = False + # PP skip output comm: True when output send/recv was skipped and + # next_token_ids are placeholder zeros. Used by process_batch_result_prefill + # to validate that skipped output is never consumed. + skipped_output_comm: bool = False + # For output processing extend_input_len_per_req: Optional[List[int]] = None extend_logprob_start_len_per_req: Optional[List[int]] = None @@ -40,13 +44,16 @@ class GenerationBatchResult: # For overlap scheduling copy_done: Optional[torch.cuda.Event] = None delay_sample_func: Optional[callable] = None - future_indices: Optional[FutureIndices] = None + future_indices: Optional[torch.Tensor] = None speculative_num_draft_tokens: Optional[int] = None # FIXME(lsyin): maybe move to a better place? # sync path: forward stream -> output processor accept_lens: Optional[torch.Tensor] = None + # Next-iter seq_lens; published via on_publish. + new_seq_lens: Optional[torch.Tensor] = None + # relay path: forward stream -> next step forward next_draft_input: Optional[EagleDraftInput] = None diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 802cf8c66f70..e0ba5b4a925c 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -17,7 +17,11 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.memory_pool import ReqToTokenPool -from sglang.srt.observability.metrics_collector import RadixCacheMetricsCollector +from sglang.srt.observability.metrics_collector import ( + STAT_LOGGER_ROLE_RADIX_CACHE, + RadixCacheMetricsCollector, + resolve_collector_class, +) if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req @@ -71,6 +75,7 @@ class InsertResult: prefix_len: int mamba_exist: bool = False + inserted_host_node: Any = None @dataclasses.dataclass @@ -97,6 +102,7 @@ class IncLockRefResult: delta: Optional[int] = None swa_uuid_for_lock: Optional[int] = None + swa_uuid_for_host_lock: Optional[int] = None # Component nodes that were tombstones at acquire time. Replaying this set # at release prevents a short-lived lock from consuming a later load-back or # request lock after that tombstone becomes a valid device value. @@ -108,6 +114,7 @@ def to_dec_params(self) -> "DecLockRefParams": """Convert to the corresponding DecLockRefParams for dec_lock_ref.""" return DecLockRefParams( swa_uuid_for_lock=self.swa_uuid_for_lock, + swa_uuid_for_host_lock=self.swa_uuid_for_host_lock, skip_lock_node_ids={ component_type: set(node_ids) for component_type, node_ids in self.skip_lock_node_ids.items() @@ -120,6 +127,7 @@ class DecLockRefParams: """Parameters for dec_lock_ref operation.""" swa_uuid_for_lock: Optional[int] = None + swa_uuid_for_host_lock: Optional[int] = None skip_lock_node_ids: dict[ComponentType, set[int]] = dataclasses.field( default_factory=dict ) @@ -207,7 +215,12 @@ def init_metrics_collector(self): labels = {"cache_type": self.__class__.__name__} if server_args.extra_metric_labels: labels.update(server_args.extra_metric_labels) - self.metrics_collector = RadixCacheMetricsCollector(labels=labels) + radix_cache_cls = resolve_collector_class( + server_args, + STAT_LOGGER_ROLE_RADIX_CACHE, + RadixCacheMetricsCollector, + ) + self.metrics_collector = radix_cache_cls(labels=labels) def update_eviction_metrics(self, num_evicted: int, start_time: float): if self.metrics_collector is not None and num_evicted > 0: diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 56fccddc9491..94d31c6c38b0 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -531,7 +531,8 @@ def alloc_for_decode(batch: ScheduleBatch, token_per_req: int) -> torch.Tensor: batch.maybe_evict_swa() - bs = batch.seq_lens.shape[0] + seq_lens_gpu = batch.seq_lens + bs = seq_lens_gpu.shape[0] if batch.tree_cache.page_size == 1: # Non-paged allocation @@ -539,9 +540,9 @@ def alloc_for_decode(batch: ScheduleBatch, token_per_req: int) -> torch.Tensor: else: # Paged allocation last_loc = batch.req_to_token_pool.req_to_token[ - batch.req_pool_indices, batch.seq_lens - 1 + batch.req_pool_indices, seq_lens_gpu - 1 ] - seq_lens_next = batch.seq_lens + token_per_req + seq_lens_next = seq_lens_gpu + token_per_req out_cache_loc = alloc_paged_token_slots_decode( tree_cache=batch.tree_cache, seq_lens=seq_lens_next, @@ -552,9 +553,9 @@ def alloc_for_decode(batch: ScheduleBatch, token_per_req: int) -> torch.Tensor: # Write to req_to_token_pool if batch.model_config.is_encoder_decoder: - locs = batch.encoder_lens + batch.seq_lens + locs = batch.encoder_lens + seq_lens_gpu else: - locs = batch.seq_lens.clone() + locs = seq_lens_gpu.clone() batch.req_to_token_pool.write( (batch.req_pool_indices, locs), out_cache_loc.to(torch.int32) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 5e77fa61411a..b44388f37808 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -470,8 +470,13 @@ def __init__( enable_memory_saver, ) + indexer_size = ( + self.c4_logical_size + if (not _is_hip or envs.SGLANG_OPT_USE_COMPRESSOR_V2.get()) + else c4_size + ) self.c4_indexer_kv_pool = DeepSeekV4IndexerPool( - self.c4_logical_size if not _is_hip else c4_size, + indexer_size, c4_page_size, dtype, indexer_head_dim, @@ -578,6 +583,7 @@ def _init_paged_compress_states(self, enable_memory_saver: bool): enable_memory_saver=enable_memory_saver, ratio=ratio, online=(ratio == 128 and ONLINE_C128), + swa_page_size=self.swa_page_size, ) if ratio == 4: diff --git a/python/sglang/srt/mem_cache/events.py b/python/sglang/srt/mem_cache/events.py index 354ea1daae0d..d659268d1a01 100644 --- a/python/sglang/srt/mem_cache/events.py +++ b/python/sglang/srt/mem_cache/events.py @@ -65,7 +65,7 @@ def _record_store_event(self, node: Any, medium=None): if is_bigram: page_tokens = [(raw[j], raw[j + 1]) for j in range(start, end)] else: - page_tokens = raw[start:end] + page_tokens = list(raw[start:end]) block_hash = hash_str_to_int64(node.hash_value[page_index]) diff --git a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py index 76a5426aa21d..c3e4c7a80405 100644 --- a/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py +++ b/python/sglang/srt/mem_cache/hi_mamba_radix_cache.py @@ -46,7 +46,11 @@ RadixKey, ) from sglang.srt.mem_cache.utils import compute_node_hash_values, split_node_hash_value -from sglang.srt.observability.metrics_collector import StorageMetricsCollector +from sglang.srt.observability.metrics_collector import ( + STAT_LOGGER_ROLE_STORAGE, + StorageMetricsCollector, + resolve_collector_class, +) if TYPE_CHECKING: from sglang.srt.mem_cache.cache_init_params import CacheInitParams @@ -1252,7 +1256,14 @@ def _apply_storage_runtime_config( } if extra_metric_labels: labels.update(extra_metric_labels) - storage_metrics_collector = StorageMetricsCollector(labels=labels) + from sglang.srt.server_args import get_global_server_args + + storage_cls = resolve_collector_class( + get_global_server_args(), + STAT_LOGGER_ROLE_STORAGE, + StorageMetricsCollector, + ) + storage_metrics_collector = storage_cls(labels=labels) self.enable_storage = enable_storage self.prefetch_threshold = prefetch_threshold diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 136d6e5145ba..c8cadf0b4686 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -56,7 +56,11 @@ compute_node_hash_values, split_node_hash_value, ) -from sglang.srt.observability.metrics_collector import StorageMetricsCollector +from sglang.srt.observability.metrics_collector import ( + STAT_LOGGER_ROLE_STORAGE, + StorageMetricsCollector, + resolve_collector_class, +) if TYPE_CHECKING: from sglang.srt.mem_cache.cache_init_params import CacheInitParams @@ -248,7 +252,14 @@ def _apply_storage_runtime_config( labels.update(extra_metric_labels) existing_collector = getattr(self, "storage_metrics_collector", None) if existing_collector is None: - self.storage_metrics_collector = StorageMetricsCollector(labels=labels) + from sglang.srt.server_args import get_global_server_args + + storage_cls = resolve_collector_class( + get_global_server_args(), + STAT_LOGGER_ROLE_STORAGE, + StorageMetricsCollector, + ) + self.storage_metrics_collector = storage_cls(labels=labels) elif set(existing_collector.labels.keys()) == set(labels.keys()): existing_collector.labels = labels else: @@ -804,10 +815,6 @@ def loading_check(self): def evictable_size(self): return self.evictable_size_ - def _to_radix_key(self, token_ids: List[int]) -> RadixKey: - """Convert raw token_ids to a RadixKey; must be list (not tuple) for paged match.""" - return RadixKey(token_ids=list(token_ids)) - def inc_lock_ref(self, node: TreeNode) -> IncLockRefResult: if self.disable: return IncLockRefResult(delta=0) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index a339eef5dd2a..1b09a5b96aab 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -1,8 +1,11 @@ from __future__ import annotations +import json import logging +import os import threading import time +from queue import Queue from typing import TYPE_CHECKING, Any, Callable, List, Optional import torch @@ -171,6 +174,7 @@ def __init__( enable_storage_metrics: bool = False, ): startup_storage_backend = storage_backend + self.extra_host_mem_release_queues: dict[PoolName, Queue] = {} super().__init__( token_to_kv_pool_allocator=token_to_kv_pool_allocator, mem_pool_host=mem_pool_host, @@ -204,6 +208,10 @@ def __init__( host_pools=getattr(mem_pool_host, "entries", None), ) + def _start_storage_threads(self): + super()._start_storage_threads() + self._init_extra_host_mem_release_queues() + def attach_storage_backend( self, storage_backend: str, @@ -222,10 +230,133 @@ def attach_storage_backend( for entry in host_pools or []: self.storage_backend.register_mem_host_pool_v2(entry.host_pool, entry.name) + @staticmethod + def parse_storage_backend_extra_config( + storage_backend_extra_config: Optional[str], + ) -> tuple[dict, int, float, float, bool]: + extra_config = {} + if storage_backend_extra_config: + if storage_backend_extra_config.startswith("@"): + path = storage_backend_extra_config[1:] + ext = os.path.splitext(path)[1].lower() + with open(path, "rb" if ext == ".toml" else "r") as f: + if ext == ".json": + extra_config = json.load(f) + elif ext == ".toml": + import tomllib + + extra_config = tomllib.load(f) + elif ext in (".yaml", ".yml"): + import yaml + + extra_config = yaml.safe_load(f) + else: + raise ValueError( + f"Unsupported config file {path} (config format: {ext})" + ) + else: + extra_config = json.loads(storage_backend_extra_config) + + prefetch_threshold = extra_config.pop("prefetch_threshold", 256) + prefetch_timeout_base = extra_config.pop("prefetch_timeout_base", 1) + prefetch_timeout_per_ki_token = extra_config.pop( + "prefetch_timeout_per_ki_token", 0.25 + ) + hicache_storage_pass_prefix_keys = extra_config.pop( + "hicache_storage_pass_prefix_keys", False + ) + + if not isinstance(prefetch_threshold, int): + raise ValueError( + f"prefetch_threshold must be int, got {type(prefetch_threshold).__name__}" + ) + if not isinstance(prefetch_timeout_base, (int, float)): + raise ValueError( + f"prefetch_timeout_base must be number, got {type(prefetch_timeout_base).__name__}" + ) + if not isinstance(prefetch_timeout_per_ki_token, (int, float)): + raise ValueError( + "prefetch_timeout_per_ki_token must be number, got " + f"{type(prefetch_timeout_per_ki_token).__name__}" + ) + if not isinstance(hicache_storage_pass_prefix_keys, bool): + raise ValueError( + "hicache_storage_pass_prefix_keys must be bool, got " + f"{type(hicache_storage_pass_prefix_keys).__name__}" + ) + + return ( + extra_config, + prefetch_threshold, + float(prefetch_timeout_base), + float(prefetch_timeout_per_ki_token), + hicache_storage_pass_prefix_keys, + ) + + def clear_storage_backend(self) -> bool: + if not self.enable_storage: + logger.warning("Hierarchical cache storage backend is not enabled.") + return False + if not hasattr(self.storage_backend, "clear"): + logger.warning( + "Storage backend %s does not support clear operation.", + type(self.storage_backend).__name__, + ) + return False + self.storage_backend.clear() + return True + + def _init_extra_host_mem_release_queues(self) -> None: + self.extra_host_mem_release_queues = {} + entries = getattr(self.mem_pool_host, "entries", None) or [] + anchor_entry = getattr(self.mem_pool_host, "anchor_entry", None) + for entry in entries: + if entry is anchor_entry or entry.is_primary_index_anchor: + continue + self.extra_host_mem_release_queues[entry.name] = Queue() + + def _append_host_mem_release_pages( + self, release_queue: Queue, host_indices: torch.Tensor, page_size: int + ) -> None: + if host_indices.numel() == 0: + return + for page in host_indices.split(page_size): + release_queue.put(page) + + def append_host_mem_release( + self, + host_indices: Optional[torch.Tensor] = None, + extra_pools: Optional[list[PoolTransfer]] = None, + ): + if host_indices is not None: + self._append_host_mem_release_pages( + self.host_mem_release_queue, + host_indices, + self.mem_pool_host.page_size, + ) + for transfer in extra_pools or []: + if transfer.host_indices is None or transfer.host_indices.numel() == 0: + continue + entry = self.mem_pool_host.entry_map.get(transfer.name) + if ( + entry is None + or entry.is_primary_index_anchor + or transfer.indices_from_pool is not None + ): + continue + release_queue = self.extra_host_mem_release_queues.get(transfer.name) + if release_queue is None: + continue + self._append_host_mem_release_pages( + release_queue, transfer.host_indices, entry.host_pool.page_size + ) + def reset(self): super().reset() if self.enable_storage: self.host_mem_release_queue.queue.clear() + for release_queue in self.extra_host_mem_release_queues.values(): + release_queue.queue.clear() self.prefetch_tokens_occupied = 0 def write( diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index 63cb96c4f516..e12c9d350372 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Callable, Optional from sglang.srt.mem_cache.hicache_storage import PoolName, SidecarPoolSpec @@ -18,6 +19,7 @@ MLATokenToKVPoolHost, PoolEntry, ) +from sglang.srt.mem_cache.unified_cache_components import ComponentType if TYPE_CHECKING: import torch @@ -643,269 +645,492 @@ def build_anchor_sidecar_stack( return host_pool_group, cache_controller -def attach_hybrid_pool_to_unified_cache( - cache: UnifiedRadixCache, - params: CacheInitParams, - server_args: ServerArgs, - *, - load_cache_event, - attn_cp_group: Optional[torch.distributed.ProcessGroup] = None, - attn_tp_group: Optional[torch.distributed.ProcessGroup] = None, -) -> None: - """Attach HostPoolGroup + HybridCacheController to UnifiedRadixCache.""" - from sglang.srt.mem_cache.base_prefix_cache import EvictParams - from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool - from sglang.srt.mem_cache.memory_pool import ( - DSATokenToKVPool, - HybridLinearKVPool, - MLATokenToKVPool, - ) - from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool - from sglang.srt.mem_cache.unified_cache_components import ComponentType +_COMPONENT_HOST_ATTR: dict[ComponentType, tuple[str, str]] = { + ComponentType.FULL: ("full_kv_pool_host", "_full_kv_pool_host"), + ComponentType.SWA: ("swa_kv_pool_host", "_swa_kv_pool_host"), + ComponentType.MAMBA: ("mamba_pool_host", "_mamba_pool_host"), +} - try: - kvcache = params.token_to_kv_pool_allocator.get_kvcache() - swa_stack = isinstance(kvcache, SWAKVPool) - mamba_stack = isinstance(kvcache, HybridLinearKVPool) - dsa_stack = isinstance(kvcache, DSATokenToKVPool) - deepseek_v4_stack = isinstance(kvcache, DeepSeekV4TokenToKVPool) - - if deepseek_v4_stack: - use_mla = False - assert set(cache.components.keys()) == { - ComponentType.FULL, - ComponentType.SWA, - }, "DeepSeekV4TokenToKVPool requires FULL + SWA in UnifiedRadixCache." - elif mamba_stack: - full_kv_pool = kvcache.full_kv_pool - use_mla = kvcache.use_mla - assert set(cache.components.keys()) == { - ComponentType.FULL, - ComponentType.MAMBA, - }, "HybridLinearKVPool currently only supports FULL + MAMBA in UnifiedRadixCache." - elif swa_stack: - full_kv_pool = kvcache.full_kv_pool - use_mla = False - assert set(cache.components.keys()) == { - ComponentType.FULL, - ComponentType.SWA, - }, "SWAKVPool currently only supports FULL + SWA in UnifiedRadixCache." - else: - full_kv_pool = kvcache - use_mla = isinstance(kvcache, MLATokenToKVPool) - assert set(cache.components.keys()) == { - ComponentType.FULL - }, "Non-hybrid KV pool currently only supports FULL-only UnifiedRadixCache." - - if deepseek_v4_stack: - host_pool_group, cache_controller = build_deepseek_v4_hicache_stack( - params=params, - server_args=server_args, - kvcache=kvcache, - page_size=cache.page_size, - tp_group=params.tp_cache_group, - load_cache_event=load_cache_event, - attn_cp_group=attn_cp_group, - attn_tp_group=attn_tp_group, - storage_backend=None, - host_swa_evict_fn=lambda n: cache.evict_host(n, ComponentType.SWA), - device_swa_evict_fn=lambda n: cache.evict( - EvictParams(swa_num_tokens=n) - ), - pp_rank=params.pp_rank, - pp_size=params.pp_size, - ) - cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV) - cache.host_pool_group = host_pool_group - cache.cache_controller = cache_controller - cache.components[ComponentType.FULL]._full_kv_pool_host = ( - cache.full_kv_pool_host - ) - cache.swa_kv_pool_host = host_pool_group.get_pool(PoolName.SWA) - cache.components[ComponentType.SWA]._swa_kv_pool_host = ( - cache.swa_kv_pool_host - ) - for pool_name, indices_from_pool in ( + +@dataclass +class StackBuildResult: + host_pool_group: HostPoolGroup + cache_controller: HybridCacheController + component_host_pools: dict[ComponentType, Any] + sidecars: list[SidecarPoolSpec] = field(default_factory=list) + # Mamba state lives in req_to_token_pool, not in kvcache, so its + # layer_transfer_counter has to be wired separately. + register_req_to_token_counter: bool = False + transfer_layer_num: int = 0 + pools_desc: str = "" + + +class StackStrategy: + def matches(self, kvcache: Any, components: set[ComponentType]) -> bool: + raise NotImplementedError + + def build( + self, + *, + cache: UnifiedRadixCache, + kvcache: Any, + params: CacheInitParams, + server_args: ServerArgs, + load_cache_event, + attn_cp_group: Optional[torch.distributed.ProcessGroup] = None, + attn_tp_group: Optional[torch.distributed.ProcessGroup] = None, + storage_backend: Optional[str] = None, + storage_backend_extra_config: Optional[dict] = None, + prefetch_threshold: int = 256, + model_name: Optional[str] = None, + enable_storage_metrics: bool = False, + ) -> StackBuildResult: + raise NotImplementedError + + +class _DeepSeekV4Strategy(StackStrategy): + def matches(self, kvcache, components): + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DeepSeekV4TokenToKVPool, + ) + + return isinstance(kvcache, DeepSeekV4TokenToKVPool) and components == { + ComponentType.FULL, + ComponentType.SWA, + } + + def build( + self, + *, + cache, + kvcache, + params, + server_args, + load_cache_event, + attn_cp_group=None, + attn_tp_group=None, + storage_backend=None, + storage_backend_extra_config=None, + prefetch_threshold=256, + model_name=None, + enable_storage_metrics=False, + ): + from sglang.srt.mem_cache.base_prefix_cache import EvictParams + + host_pool_group, cache_controller = build_deepseek_v4_hicache_stack( + params=params, + server_args=server_args, + kvcache=kvcache, + page_size=cache.page_size, + tp_group=params.tp_cache_group, + load_cache_event=load_cache_event, + attn_cp_group=attn_cp_group, + attn_tp_group=attn_tp_group, + storage_backend=storage_backend, + host_swa_evict_fn=lambda n: cache.evict_host(n, ComponentType.SWA), + device_swa_evict_fn=lambda n: cache.evict(EvictParams(swa_num_tokens=n)), + prefetch_threshold=prefetch_threshold, + model_name=model_name, + storage_backend_extra_config=storage_backend_extra_config, + pp_rank=params.pp_rank, + pp_size=params.pp_size, + enable_storage_metrics=enable_storage_metrics, + ) + sidecars = [ + SidecarPoolSpec(pool_name=name, indices_from_pool=src) + for name, src in ( (PoolName.DEEPSEEK_V4_C4, PoolName.KV), (PoolName.DEEPSEEK_V4_C4_INDEXER, PoolName.KV), (PoolName.DEEPSEEK_V4_C128, PoolName.KV), (PoolName.DEEPSEEK_V4_C4_STATE, PoolName.SWA), (PoolName.DEEPSEEK_V4_C4_INDEXER_STATE, PoolName.SWA), (PoolName.DEEPSEEK_V4_C128_STATE, PoolName.SWA), - ): - if pool_name in host_pool_group.entry_map: - cache.register_sidecar_pool( - SidecarPoolSpec( - pool_name=pool_name, - indices_from_pool=indices_from_pool, - ) - ) - transfer_layer_num = kvcache.end_layer - kvcache.start_layer - elif mamba_stack: - full_layer_mapping = dict(kvcache.full_attention_layer_id_mapping) - mamba_layer_mapping = dict(params.req_to_token_pool.mamba_map) - host_pool_group, cache_controller = build_hybrid_mamba_stack( - params=params, - server_args=server_args, - kv_pool=full_kv_pool, - mamba_pool=params.req_to_token_pool.mamba_pool, - full_layer_mapping=full_layer_mapping, - mamba_layer_mapping=mamba_layer_mapping, - page_size=cache.page_size, - tp_group=params.tp_cache_group, - load_cache_event=load_cache_event, - attn_cp_group=attn_cp_group, - attn_tp_group=attn_tp_group, - storage_backend=None, - use_mla=use_mla, - host_mamba_evict_fn=lambda n: cache.evict_host(n, ComponentType.MAMBA), - device_mamba_evict_fn=lambda n: cache.evict(EvictParams(mamba_num=n)), - pp_rank=params.pp_rank, - pp_size=params.pp_size, - ) - cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV) - cache.host_pool_group = host_pool_group - cache.cache_controller = cache_controller - cache.components[ComponentType.FULL]._full_kv_pool_host = ( - cache.full_kv_pool_host - ) - cache.mamba_pool_host = host_pool_group.get_pool(PoolName.MAMBA) - cache.components[ComponentType.MAMBA]._mamba_pool_host = ( - cache.mamba_pool_host - ) - params.req_to_token_pool.register_layer_transfer_counter( - cache_controller.layer_done_counter - ) - transfer_layer_num = len(full_layer_mapping | mamba_layer_mapping) - elif swa_stack: - full_layer_mapping = { - global_id: local_id - for global_id, (local_id, is_swa) in kvcache.layers_mapping.items() - if not is_swa - } - swa_layer_mapping = { - global_id: local_id - for global_id, (local_id, is_swa) in kvcache.layers_mapping.items() - if is_swa - } - host_pool_group, cache_controller = build_hybrid_swa_stack( - params=params, - server_args=server_args, - full_kv_pool=full_kv_pool, - swa_kv_pool=kvcache.swa_kv_pool, - full_layer_mapping=full_layer_mapping, - swa_layer_mapping=swa_layer_mapping, - page_size=cache.page_size, - tp_group=params.tp_cache_group, - load_cache_event=load_cache_event, - attn_cp_group=attn_cp_group, - attn_tp_group=attn_tp_group, - storage_backend=None, - use_mla=False, - host_swa_evict_fn=lambda n: cache.evict_host(n, ComponentType.SWA), - device_swa_evict_fn=lambda n: cache.evict( - EvictParams(swa_num_tokens=n) - ), - pp_rank=params.pp_rank, - pp_size=params.pp_size, - ) - cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV) - cache.host_pool_group = host_pool_group - cache.cache_controller = cache_controller - cache.components[ComponentType.FULL]._full_kv_pool_host = ( - cache.full_kv_pool_host - ) - cache.swa_kv_pool_host = host_pool_group.get_pool(PoolName.SWA) - cache.components[ComponentType.SWA]._swa_kv_pool_host = ( - cache.swa_kv_pool_host - ) - transfer_layer_num = len(full_layer_mapping | swa_layer_mapping) - elif dsa_stack: - full_layer_mapping = { - layer_id: layer_id for layer_id in range(full_kv_pool.layer_num) - } - host_pool_group, cache_controller = build_anchor_sidecar_stack( - params=params, - server_args=server_args, - kv_pool=full_kv_pool, - sidecar_pool_name=PoolName.INDEXER, - full_layer_mapping=full_layer_mapping, - page_size=cache.page_size, - tp_group=params.tp_cache_group, - load_cache_event=load_cache_event, - attn_cp_group=attn_cp_group, - attn_tp_group=attn_tp_group, - storage_backend=None, - use_mla=use_mla, - override_kv_cache_dim=full_kv_pool.kv_cache_dim, - sidecar_host_pool_factory=lambda kv_host_pool: DSAIndexerPoolHost( - full_kv_pool, - kv_host_pool, - server_args.hicache_mem_layout, - allocator_type=server_args.hicache_storage_backend, - ), - pp_rank=params.pp_rank, - pp_size=params.pp_size, ) - cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV) - cache.host_pool_group = host_pool_group - cache.cache_controller = cache_controller - cache.register_sidecar_pool( + if name in host_pool_group.entry_map + ] + return StackBuildResult( + host_pool_group=host_pool_group, + cache_controller=cache_controller, + component_host_pools={ + ComponentType.FULL: host_pool_group.get_pool(PoolName.KV), + ComponentType.SWA: host_pool_group.get_pool(PoolName.SWA), + }, + sidecars=sidecars, + transfer_layer_num=kvcache.end_layer - kvcache.start_layer, + pools_desc="KV + SWA + DeepSeekV4 sidecars", + ) + + +class _MambaStrategy(StackStrategy): + def matches(self, kvcache, components): + from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool + + return isinstance(kvcache, HybridLinearKVPool) and components == { + ComponentType.FULL, + ComponentType.MAMBA, + } + + def build( + self, + *, + cache, + kvcache, + params, + server_args, + load_cache_event, + attn_cp_group=None, + attn_tp_group=None, + storage_backend=None, + storage_backend_extra_config=None, + prefetch_threshold=256, + model_name=None, + enable_storage_metrics=False, + ): + from sglang.srt.mem_cache.base_prefix_cache import EvictParams + + full_layer_mapping = dict(kvcache.full_attention_layer_id_mapping) + mamba_layer_mapping = dict(params.req_to_token_pool.mamba_map) + host_pool_group, cache_controller = build_hybrid_mamba_stack( + params=params, + server_args=server_args, + kv_pool=kvcache.full_kv_pool, + mamba_pool=params.req_to_token_pool.mamba_pool, + full_layer_mapping=full_layer_mapping, + mamba_layer_mapping=mamba_layer_mapping, + page_size=cache.page_size, + tp_group=params.tp_cache_group, + load_cache_event=load_cache_event, + attn_cp_group=attn_cp_group, + attn_tp_group=attn_tp_group, + storage_backend=storage_backend, + use_mla=kvcache.use_mla, + host_mamba_evict_fn=lambda n: cache.evict_host(n, ComponentType.MAMBA), + device_mamba_evict_fn=lambda n: cache.evict(EvictParams(mamba_num=n)), + prefetch_threshold=prefetch_threshold, + model_name=model_name, + storage_backend_extra_config=storage_backend_extra_config, + pp_rank=params.pp_rank, + pp_size=params.pp_size, + enable_storage_metrics=enable_storage_metrics, + ) + return StackBuildResult( + host_pool_group=host_pool_group, + cache_controller=cache_controller, + component_host_pools={ + ComponentType.FULL: host_pool_group.get_pool(PoolName.KV), + ComponentType.MAMBA: host_pool_group.get_pool(PoolName.MAMBA), + }, + register_req_to_token_counter=True, + transfer_layer_num=len(full_layer_mapping | mamba_layer_mapping), + pools_desc="KV + MAMBA", + ) + + +def _swa_layer_mappings(kvcache) -> tuple[dict[int, int], dict[int, int]]: + full = { + gid: lid for gid, (lid, is_swa) in kvcache.layers_mapping.items() if not is_swa + } + swa = {gid: lid for gid, (lid, is_swa) in kvcache.layers_mapping.items() if is_swa} + return full, swa + + +class _SwaStrategy(StackStrategy): + def matches(self, kvcache, components): + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DeepSeekV4TokenToKVPool, + ) + from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool + + return ( + isinstance(kvcache, SWAKVPool) + and not isinstance(kvcache, DeepSeekV4TokenToKVPool) + and components == {ComponentType.FULL, ComponentType.SWA} + ) + + def build( + self, + *, + cache, + kvcache, + params, + server_args, + load_cache_event, + attn_cp_group=None, + attn_tp_group=None, + storage_backend=None, + storage_backend_extra_config=None, + prefetch_threshold=256, + model_name=None, + enable_storage_metrics=False, + ): + from sglang.srt.mem_cache.base_prefix_cache import EvictParams + + full_layer_mapping, swa_layer_mapping = _swa_layer_mappings(kvcache) + host_pool_group, cache_controller = build_hybrid_swa_stack( + params=params, + server_args=server_args, + full_kv_pool=kvcache.full_kv_pool, + swa_kv_pool=kvcache.swa_kv_pool, + full_layer_mapping=full_layer_mapping, + swa_layer_mapping=swa_layer_mapping, + page_size=cache.page_size, + tp_group=params.tp_cache_group, + load_cache_event=load_cache_event, + attn_cp_group=attn_cp_group, + attn_tp_group=attn_tp_group, + storage_backend=storage_backend, + use_mla=False, + host_swa_evict_fn=lambda n: cache.evict_host(n, ComponentType.SWA), + device_swa_evict_fn=lambda n: cache.evict(EvictParams(swa_num_tokens=n)), + prefetch_threshold=prefetch_threshold, + model_name=model_name, + storage_backend_extra_config=storage_backend_extra_config, + pp_rank=params.pp_rank, + pp_size=params.pp_size, + enable_storage_metrics=enable_storage_metrics, + ) + return StackBuildResult( + host_pool_group=host_pool_group, + cache_controller=cache_controller, + component_host_pools={ + ComponentType.FULL: host_pool_group.get_pool(PoolName.KV), + ComponentType.SWA: host_pool_group.get_pool(PoolName.SWA), + }, + transfer_layer_num=len(full_layer_mapping | swa_layer_mapping), + pools_desc="KV + SWA", + ) + + +class _DsaStrategy(StackStrategy): + def matches(self, kvcache, components): + from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool + + return isinstance(kvcache, DSATokenToKVPool) and components == { + ComponentType.FULL + } + + def build( + self, + *, + cache, + kvcache, + params, + server_args, + load_cache_event, + attn_cp_group=None, + attn_tp_group=None, + storage_backend=None, + storage_backend_extra_config=None, + prefetch_threshold=256, + model_name=None, + enable_storage_metrics=False, + ): + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + + full_kv_pool = kvcache + use_mla = isinstance(kvcache, MLATokenToKVPool) + full_layer_mapping = {i: i for i in range(full_kv_pool.layer_num)} + host_pool_group, cache_controller = build_anchor_sidecar_stack( + params=params, + server_args=server_args, + kv_pool=full_kv_pool, + sidecar_pool_name=PoolName.INDEXER, + full_layer_mapping=full_layer_mapping, + page_size=cache.page_size, + tp_group=params.tp_cache_group, + load_cache_event=load_cache_event, + attn_cp_group=attn_cp_group, + attn_tp_group=attn_tp_group, + storage_backend=storage_backend, + use_mla=use_mla, + override_kv_cache_dim=full_kv_pool.kv_cache_dim, + sidecar_host_pool_factory=lambda kv_host_pool: DSAIndexerPoolHost( + full_kv_pool, + kv_host_pool, + server_args.hicache_mem_layout, + allocator_type=server_args.hicache_storage_backend, + ), + prefetch_threshold=prefetch_threshold, + model_name=model_name, + storage_backend_extra_config=storage_backend_extra_config, + pp_rank=params.pp_rank, + pp_size=params.pp_size, + enable_storage_metrics=enable_storage_metrics, + ) + return StackBuildResult( + host_pool_group=host_pool_group, + cache_controller=cache_controller, + component_host_pools={ + ComponentType.FULL: host_pool_group.get_pool(PoolName.KV), + }, + sidecars=[ SidecarPoolSpec( pool_name=PoolName.INDEXER, indices_from_pool=PoolName.KV, - ) - ) - cache.components[ComponentType.FULL]._full_kv_pool_host = ( - cache.full_kv_pool_host - ) - transfer_layer_num = len(full_layer_mapping) - else: - full_layer_mapping = { - layer_id: layer_id for layer_id in range(full_kv_pool.layer_num) - } - host_pool_group, cache_controller = build_kv_only_stack( - params=params, - server_args=server_args, - kv_pool=full_kv_pool, - full_layer_mapping=full_layer_mapping, - page_size=cache.page_size, - tp_group=params.tp_cache_group, - load_cache_event=load_cache_event, - attn_cp_group=attn_cp_group, - attn_tp_group=attn_tp_group, - storage_backend=None, - use_mla=use_mla, - pp_rank=params.pp_rank, - pp_size=params.pp_size, - ) - cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV) - cache.host_pool_group = host_pool_group - cache.cache_controller = cache_controller - cache.components[ComponentType.FULL]._full_kv_pool_host = ( - cache.full_kv_pool_host - ) - transfer_layer_num = len(full_layer_mapping) + ), + ], + transfer_layer_num=len(full_layer_mapping), + pools_desc="KV + INDEXER", + ) + - kvcache.register_layer_transfer_counter( - cache.cache_controller.layer_done_counter +class _PlainKvStrategy(StackStrategy): + def matches(self, kvcache, components): + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DeepSeekV4TokenToKVPool, + ) + from sglang.srt.mem_cache.memory_pool import ( + DSATokenToKVPool, + HybridLinearKVPool, ) + from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool - if deepseek_v4_stack: - pools_desc = "KV + SWA + DeepSeekV4 sidecars" - elif mamba_stack: - pools_desc = "KV + MAMBA" - elif swa_stack: - pools_desc = "KV + SWA" - elif dsa_stack: - pools_desc = "KV + INDEXER" - else: - pools_desc = "KV" - logger.info( - "Attached hybrid pool stack to UnifiedRadixCache: pools=%s, transfer_layer_num=%s", - pools_desc, - transfer_layer_num, + if isinstance( + kvcache, + (SWAKVPool, HybridLinearKVPool, DSATokenToKVPool, DeepSeekV4TokenToKVPool), + ): + return False + return components == {ComponentType.FULL} + + def build( + self, + *, + cache, + kvcache, + params, + server_args, + load_cache_event, + attn_cp_group=None, + attn_tp_group=None, + storage_backend=None, + storage_backend_extra_config=None, + prefetch_threshold=256, + model_name=None, + enable_storage_metrics=False, + ): + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + + full_kv_pool = kvcache + use_mla = isinstance(kvcache, MLATokenToKVPool) + full_layer_mapping = {i: i for i in range(full_kv_pool.layer_num)} + host_pool_group, cache_controller = build_kv_only_stack( + params=params, + server_args=server_args, + kv_pool=full_kv_pool, + full_layer_mapping=full_layer_mapping, + page_size=cache.page_size, + tp_group=params.tp_cache_group, + load_cache_event=load_cache_event, + attn_cp_group=attn_cp_group, + attn_tp_group=attn_tp_group, + storage_backend=storage_backend, + use_mla=use_mla, + prefetch_threshold=prefetch_threshold, + model_name=model_name, + storage_backend_extra_config=storage_backend_extra_config, + pp_rank=params.pp_rank, + pp_size=params.pp_size, + enable_storage_metrics=enable_storage_metrics, + ) + return StackBuildResult( + host_pool_group=host_pool_group, + cache_controller=cache_controller, + component_host_pools={ + ComponentType.FULL: host_pool_group.get_pool(PoolName.KV), + }, + transfer_layer_num=len(full_layer_mapping), + pools_desc="KV", + ) + + +# Resolved first-to-last; _PlainKvStrategy is the catch-all fallback. +_STRATEGIES: list[StackStrategy] = [ + _DeepSeekV4Strategy(), + _MambaStrategy(), + _SwaStrategy(), + _DsaStrategy(), + _PlainKvStrategy(), +] + + +def register_stack_strategy(strategy: StackStrategy) -> None: + """Prepend a strategy so downstream forks can plug in (kvcache, components) + combinations not in the built-in list.""" + _STRATEGIES.insert(0, strategy) + + +def _select_strategy(kvcache: Any, components: set[ComponentType]) -> StackStrategy: + for strategy in _STRATEGIES: + if strategy.matches(kvcache, components): + return strategy + raise AssertionError( + f"No matching HiCache strategy for kvcache={type(kvcache).__name__}, " + f"components={sorted(c.name for c in components)}" + ) + + +def _apply_stack_result( + cache: UnifiedRadixCache, + kvcache: Any, + params: CacheInitParams, + result: StackBuildResult, +) -> None: + cache.host_pool_group = result.host_pool_group + cache.cache_controller = result.cache_controller + + for ct, host_pool in result.component_host_pools.items(): + cache_attr, component_attr = _COMPONENT_HOST_ATTR[ct] + setattr(cache, cache_attr, host_pool) + setattr(cache.components[ct], component_attr, host_pool) + + for sidecar in result.sidecars: + cache.register_sidecar_pool(sidecar) + + kvcache.register_layer_transfer_counter(result.cache_controller.layer_done_counter) + if result.register_req_to_token_counter: + params.req_to_token_pool.register_layer_transfer_counter( + result.cache_controller.layer_done_counter + ) + + logger.info( + "Attached hybrid pool stack to UnifiedRadixCache: pools=%s, transfer_layer_num=%s", + result.pools_desc, + result.transfer_layer_num, + ) + + +def attach_hybrid_pool_to_unified_cache( + cache: UnifiedRadixCache, + params: CacheInitParams, + server_args: ServerArgs, + *, + load_cache_event, + attn_cp_group: Optional[torch.distributed.ProcessGroup] = None, + attn_tp_group: Optional[torch.distributed.ProcessGroup] = None, + storage_backend: Optional[str] = None, + storage_extra_config: Optional[dict] = None, + storage_prefetch_threshold: int = 256, +) -> None: + """Attach HostPoolGroup + HybridCacheController to UnifiedRadixCache.""" + try: + kvcache = params.token_to_kv_pool_allocator.get_kvcache() + components = set(cache.components.keys()) + strategy = _select_strategy(kvcache, components) + result = strategy.build( + cache=cache, + kvcache=kvcache, + params=params, + server_args=server_args, + load_cache_event=load_cache_event, + attn_cp_group=attn_cp_group, + attn_tp_group=attn_tp_group, + storage_backend=storage_backend, + storage_backend_extra_config=storage_extra_config, + prefetch_threshold=storage_prefetch_threshold, + model_name=server_args.served_model_name, + enable_storage_metrics=cache._enable_metrics_flag, ) + _apply_stack_result(cache, kvcache, params, result) except Exception: logger.exception("attach_hybrid_pool_to_unified_cache failed") raise diff --git a/python/sglang/srt/mem_cache/mamba_radix_cache.py b/python/sglang/srt/mem_cache/mamba_radix_cache.py index 8ca4fa8a5050..1284a3059950 100644 --- a/python/sglang/srt/mem_cache/mamba_radix_cache.py +++ b/python/sglang/srt/mem_cache/mamba_radix_cache.py @@ -22,15 +22,14 @@ # ENGRAM_MODIFIED — Mamba radix cache extensions import heapq +from array import array from collections import defaultdict -from functools import lru_cache from typing import TYPE_CHECKING, List, Optional, Tuple import torch from numpy import float64 from sglang.srt.distributed import get_tensor_model_parallel_rank -from sglang.srt.layers.attention.fla.chunk_delta_h import CHUNK_SIZE as FLA_CHUNK_SIZE from sglang.srt.mem_cache.allocator import ( PagedTokenToKVPoolAllocator, TokenToKVPoolAllocator, @@ -149,7 +148,6 @@ def get_last_hash_value(self) -> Optional[str]: return None return self.hash_value[-1] - @lru_cache(maxsize=1) def get_prefix_hash_values(self, node: "TreeNode") -> List[str]: if node is None or node.hash_value is None: return [] @@ -429,6 +427,7 @@ def __init__(self, params: CacheInitParams): ) or isinstance(params.token_to_kv_pool_allocator, PagedTokenToKVPoolAllocator) self.req_to_token_pool: HybridReqToTokenPool = params.req_to_token_pool self.token_to_kv_pool_allocator = params.token_to_kv_pool_allocator + self.mamba_cache_chunk_size = get_global_server_args().mamba_cache_chunk_size self.page_size = params.page_size self.disable = params.disable @@ -458,7 +457,7 @@ def supports_mamba(self) -> bool: def reset(self) -> None: self.root_node = TreeNode() - self.root_node.key = RadixKey([], None) + self.root_node.key = RadixKey(array("q"), None) self.root_node.value = [] self.root_node.hash_value = [] self.root_node.full_lock_ref = 1 @@ -650,7 +649,7 @@ def _skip_cache_unfinished_req(req: Req) -> None: assert page_aligned_len == len( kv_indices - ), f"page_aligned_len != len(kv_indices), {page_aligned_len=}, {len(kv_indices)=}, {cache_len=}, {self.page_size=}, {FLA_CHUNK_SIZE=}" + ), f"page_aligned_len != len(kv_indices), {page_aligned_len=}, {len(kv_indices)=}, {cache_len=}, {self.page_size=}, {self.mamba_cache_chunk_size=}" page_aligned_token_ids = token_ids[:page_aligned_len] @@ -1057,14 +1056,11 @@ def _match_post_processor( # Calculate the branching point. It is defined as the last aligned position that # does not have a mamba value. if len(value) > best_value_len: - mamba_cache_chunk_size = get_global_server_args().mamba_cache_chunk_size - mamba_cache_chunk_aligned_seqlen = ( - sum(len(v) for v in value) // mamba_cache_chunk_size - ) * mamba_cache_chunk_size + chunk_aligned_seqlen = ( + sum(len(v) for v in value) // self.mamba_cache_chunk_size + ) * self.mamba_cache_chunk_size mamba_branching_seqlen = ( - mamba_cache_chunk_aligned_seqlen - if mamba_cache_chunk_aligned_seqlen > 0 - else None + chunk_aligned_seqlen if chunk_aligned_seqlen > 0 else None ) else: mamba_branching_seqlen = None diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 2f54531d10bb..07446353fd93 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -66,6 +66,7 @@ is_npu, next_power_of_2, ) +from sglang.srt.utils.async_probe import maybe_detect_oob from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter if TYPE_CHECKING: @@ -113,6 +114,16 @@ def _set_kv_buffer_impl( row_bytes=row_bytes, ) + if _is_cpu and _cpu_has_amx_support: + return torch.ops.sgl_kernel.store_cache_cpu( + k, + v, + k_cache, + v_cache, + indices, + row_dim, + ) + from sglang.srt.model_executor.cuda_graph_runner import get_is_capture_mode if get_is_capture_mode() and alt_stream is not None: @@ -747,13 +758,13 @@ def _finalize_allocation_log(self, num_tokens: int): k_size_GB = k_size / GB v_size_GB = v_size / GB logger.info( - f"KV Cache is allocated. #tokens: {num_tokens}, K size: {k_size_GB:.2f} GB, V size: {v_size_GB:.2f} GB" + f"KV Cache is allocated. dtype: {self.dtype}, #tokens: {num_tokens}, K size: {k_size_GB:.2f} GB, V size: {v_size_GB:.2f} GB" ) self.mem_usage = k_size_GB + v_size_GB else: kv_size_GB = kv_size_bytes / GB logger.info( - f"KV Cache is allocated. #tokens: {num_tokens}, KV size: {kv_size_GB:.2f} GB" + f"KV Cache is allocated. dtype: {self.dtype}, #tokens: {num_tokens}, KV size: {kv_size_GB:.2f} GB" ) self.mem_usage = kv_size_GB @@ -1090,6 +1101,11 @@ def set_kv_buffer( ) def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor): + # Catch stale indices here instead of as illegal-addr or silent KV corruption. + size_limit = self.size + self.page_size + maybe_detect_oob(tgt_loc, 0, size_limit, "move_kv_cache tgt_loc") + maybe_detect_oob(src_loc, 0, size_limit, "move_kv_cache src_loc") + if envs.SGLANG_NATIVE_MOVE_KV_CACHE.get(): move_kv_cache_native(self.k_buffer, self.v_buffer, tgt_loc, src_loc) return diff --git a/python/sglang/srt/mem_cache/memory_pool_host.py b/python/sglang/srt/mem_cache/memory_pool_host.py index a872e24eaed8..d5d9d5655808 100644 --- a/python/sglang/srt/mem_cache/memory_pool_host.py +++ b/python/sglang/srt/mem_cache/memory_pool_host.py @@ -187,9 +187,11 @@ def alloc_with_host_register( """ buffer = allocator.allocate(dims, dtype=dtype, device=device) if pin_memory: - torch.cuda.cudart().cudaHostRegister( + ret = torch.cuda.cudart().cudaHostRegister( buffer.data_ptr(), buffer.numel() * buffer.element_size(), 0 ) + if ret != 0: + raise RuntimeError(f"cudaHostRegister failed with error code {ret}") return buffer diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 5f8a256f6674..6e35d1a313f2 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -26,8 +26,8 @@ import logging import sys import time +from array import array from collections import defaultdict -from functools import lru_cache from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Tuple, Union import torch @@ -70,7 +70,7 @@ class RadixKey: def __init__( self, - token_ids: List[int], + token_ids: array[int], extra_key: Optional[str] = None, is_bigram: bool = False, ): @@ -87,6 +87,7 @@ def __len__(self) -> int: return n - 1 if n > 0 else 0 return len(self.token_ids) + # TODO(Jialin): vectorize with numpy without PyLong boxing def __iter__(self) -> Iterator: if self.is_bigram: t = self.token_ids @@ -110,7 +111,7 @@ def __getitem__(self, idx: Union[int, slice]) -> "RadixKey": if self.is_bigram: # bigrams [start, stop) span raw tokens [start, stop + 1); # empty slice -> empty raw tokens (not a dangling boundary token). - raw = self.token_ids[start : stop + 1] if stop > start else [] + raw = self.token_ids[start : stop + 1] if stop > start else array("q") return RadixKey(raw, self.extra_key, is_bigram=True) return RadixKey(self.token_ids[start:stop], self.extra_key) @@ -144,6 +145,7 @@ def _check_compatible(self, other: "RadixKey") -> None: f"{self.extra_key=} != {other.extra_key=}" ) + # TODO(Jialin): replace zip with numpy to skip per-element PyLong boxing def match(self, other: "RadixKey", page_size: int = 1) -> int: """Logical-unit prefix length shared with ``other``. Result is rounded down to ``page_size``.""" self._check_compatible(other) @@ -255,7 +257,6 @@ def get_last_hash_value(self) -> Optional[str]: return None return self.hash_value[-1] - @lru_cache(maxsize=1) def get_prefix_hash_values(self, node: TreeNode) -> List[str]: if node is None or node.hash_value is None: return [] @@ -337,7 +338,7 @@ def create_simulated( def reset(self): # Initialize root with minimum priority so any real priority overrides it self.root_node = TreeNode(priority=-sys.maxsize) - self.root_node.key = RadixKey(token_ids=[], extra_key=None) + self.root_node.key = RadixKey(token_ids=array("q"), extra_key=None) self.root_node.value = [] self.root_node.host_value = [] self.root_node.lock_ref = 1 @@ -811,20 +812,15 @@ def _total_size_helper(self): if __name__ == "__main__": tree = RadixCache.create_simulated() - # Example token id sequences (as lists of ints) - tree.insert(InsertParams(key=RadixKey(token_ids=[1, 2, 3], extra_key=None))) - tree.insert(InsertParams(key=RadixKey(token_ids=[1, 2, 3], extra_key=None))) - tree.insert(InsertParams(key=RadixKey(token_ids=[1, 2, 4, 5], extra_key=None))) - tree.insert( - InsertParams(key=RadixKey(token_ids=[1, 2, 4, 5, 6, 7], extra_key=None)) - ) - tree.insert( - InsertParams(key=RadixKey(token_ids=[8, 9, 10, 11, 12], extra_key=None)) - ) + tree.insert(InsertParams(key=RadixKey(token_ids=array("q", [1, 2, 3])))) + tree.insert(InsertParams(key=RadixKey(token_ids=array("q", [1, 2, 3])))) + tree.insert(InsertParams(key=RadixKey(token_ids=array("q", [1, 2, 4, 5])))) + tree.insert(InsertParams(key=RadixKey(token_ids=array("q", [1, 2, 4, 5, 6, 7])))) + tree.insert(InsertParams(key=RadixKey(token_ids=array("q", [8, 9, 10, 11, 12])))) tree.pretty_print() print( tree.match_prefix( - MatchPrefixParams(key=RadixKey(token_ids=[1, 2, 3, 13, 14], extra_key=None)) + MatchPrefixParams(key=RadixKey(token_ids=array("q", [1, 2, 3, 13, 14]))) ) ) diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py index bd1205708351..21c9f692b21e 100644 --- a/python/sglang/srt/mem_cache/swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/swa_memory_pool.py @@ -173,7 +173,7 @@ def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor) -> torch.Tens key = (kv_indices.data_ptr(), kv_indices.numel()) if key != self._cached_loc_key: if self._cached_loc_key is not None: - logger.warning( + logger.debug( "translate_loc_from_full_to_swa: loc tensor changed mid-forward " "without invalidate_loc_cache() — possible missing call site" ) @@ -561,6 +561,8 @@ def alloc_extend_swa_tail( self.full_to_swa_index_mapping[alloc_full_indices[-swa_tail_len:]] = ( alloc_swa_indices ) + if swa_tail_len < extend_num_tokens: + self.full_to_swa_index_mapping[alloc_full_indices[:-swa_tail_len]] = 0 return alloc_full_indices def alloc_decode( diff --git a/python/sglang/srt/mem_cache/unified_cache_components/full_component.py b/python/sglang/srt/mem_cache/unified_cache_components/full_component.py index bccb866a7c07..5ed89e3b40bd 100644 --- a/python/sglang/srt/mem_cache/unified_cache_components/full_component.py +++ b/python/sglang/srt/mem_cache/unified_cache_components/full_component.py @@ -65,7 +65,7 @@ def finalize_match_result( # last_device_node, summing host_value lengths of evicted nodes. ct = self.component_type kv_host_hit = 0 - node = result.last_host_node + node = result.best_match_node root_node = self.cache.root_node while node is not result.last_device_node and node is not root_node: full_host = node.component_data[ct].host_value @@ -155,9 +155,22 @@ def drive_host_eviction( heapq.heappush(heap, (x.parent.last_access_time, x.parent)) def acquire_component_lock( - self, node: UnifiedTreeNode, result: IncLockRefResult + self, + node: UnifiedTreeNode, + result: IncLockRefResult, + lock_host: bool = False, ) -> IncLockRefResult: ct = self.component_type + + # Only the last host node needs to be protected. + if lock_host: + cd = node.component_data[ct] + if cd.host_value is None: + return result + cd.host_lock_ref += 1 + self.cache._update_evictable_leaf_sets(node) + return result + root = self.cache.root_node cur = node @@ -185,9 +198,20 @@ def acquire_component_lock( return result def release_component_lock( - self, node: UnifiedTreeNode, params: Optional[DecLockRefParams] + self, + node: UnifiedTreeNode, + params: Optional[DecLockRefParams], + lock_host: bool = False, ) -> None: ct = self.component_type + if lock_host: + cd = node.component_data[ct] + if cd.host_value is None or cd.host_lock_ref == 0: + return + cd.host_lock_ref -= 1 + self.cache._update_evictable_leaf_sets(node) + return + root = self.cache.root_node skip_lock_node_ids = params.skip_lock_node_ids.get(ct, ()) if params else () cur = node @@ -255,6 +279,7 @@ def commit_hicache_transfer( node: UnifiedTreeNode, phase: CacheTransferPhase, transfers: list[PoolTransfer] = (), + **kw, ) -> None: ct = self.component_type diff --git a/python/sglang/srt/mem_cache/unified_cache_components/mamba_component.py b/python/sglang/srt/mem_cache/unified_cache_components/mamba_component.py index 7ecfec0a27d7..fac322005b32 100644 --- a/python/sglang/srt/mem_cache/unified_cache_components/mamba_component.py +++ b/python/sglang/srt/mem_cache/unified_cache_components/mamba_component.py @@ -13,7 +13,7 @@ MatchPrefixParams, MatchResult, ) -from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer +from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer from sglang.srt.mem_cache.unified_cache_components.tree_component import ( CacheTransferPhase, ComponentType, @@ -213,34 +213,59 @@ def drive_eviction( x = x_next def acquire_component_lock( - self, node: UnifiedTreeNode, result: IncLockRefResult + self, + node: UnifiedTreeNode, + result: IncLockRefResult, + lock_host: bool = False, ) -> IncLockRefResult: ct = self.component_type + if node is self.cache.root_node: + return result cd = node.component_data[ct] - value = cd.value + value = cd.host_value if lock_host else cd.value # A node in skip_lock_node_ids was a tombstone when this lock was acquired. if value is None: result.skip_lock_node_ids.setdefault(ct, set()).add(node.id) return result - if cd.lock_ref == 0: - vlen = len(value) - self.cache.component_evictable_size_[ct] -= vlen - self.cache.component_protected_size_[ct] += vlen - cd.lock_ref += 1 + if lock_host: + if cd.host_lock_ref == 0: + host_lru = self.cache.host_lru_lists[ct] + if host_lru.in_list(node): + host_lru.remove_node(node) + cd.host_lock_ref += 1 + else: + if cd.lock_ref == 0: + vlen = len(value) + self.cache.component_evictable_size_[ct] -= vlen + self.cache.component_protected_size_[ct] += vlen + cd.lock_ref += 1 return result def release_component_lock( - self, node: UnifiedTreeNode, params: Optional[DecLockRefParams] + self, + node: UnifiedTreeNode, + params: Optional[DecLockRefParams], + lock_host: bool = False, ) -> None: ct = self.component_type + if node is self.cache.root_node: + return cd = node.component_data[ct] skip_lock_node_ids = params.skip_lock_node_ids.get(ct, ()) if params else () if node.id in skip_lock_node_ids: return - value = cd.value - if value is not None and cd.lock_ref > 0: + value = cd.host_value if lock_host else cd.value + if lock_host: + cd.host_lock_ref -= 1 + if cd.host_lock_ref == 0 and cd.value is None and cd.host_value is not None: + host_lru = self.cache.host_lru_lists[ct] + if not host_lru.in_list(node): + host_lru.insert_mru(node) + return + + if cd.lock_ref > 0: if cd.lock_ref == 1: vlen = len(value) self.cache.component_evictable_size_[ct] += vlen @@ -392,6 +417,35 @@ def build_hicache_transfers( return transfers if transfers else None + if phase == CacheTransferPhase.BACKUP_STORAGE: + cd = node.component_data[ct] + if cd.host_value is None or not node.hash_value: + return None + return [ + PoolTransfer( + name=PoolName.MAMBA, + host_indices=cd.host_value, + keys=[node.hash_value[-1]], + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + ] + + if phase == CacheTransferPhase.PREFETCH: + host_indices = self._mamba_pool_host.alloc(1) + if host_indices is None: + self.cache.evict_host(1, ComponentType.MAMBA) + host_indices = self._mamba_pool_host.alloc(1) + if host_indices is None: + return [] + return [ + PoolTransfer( + name=PoolName.MAMBA, + host_indices=host_indices, + keys=["__placeholder__"], + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + ] + return None def commit_hicache_transfer( @@ -399,6 +453,7 @@ def commit_hicache_transfer( node: UnifiedTreeNode, phase: CacheTransferPhase, transfers: list[PoolTransfer] = (), + **kw, ) -> None: ct = self.component_type @@ -423,6 +478,41 @@ def commit_hicache_transfer( self.cache.lru_lists[ct].insert_mru(node) self.cache.component_evictable_size_[ct] += count + elif phase == CacheTransferPhase.PREFETCH: + if not transfers: + return + transfer = transfers[0] + host_indices = transfer.host_indices + insert_result = kw.get("insert_result") + pool_storage_result = kw.get("pool_storage_result") + loaded = ( + pool_storage_result is not None + and pool_storage_result.extra_pool_hit_pages.get(PoolName.MAMBA, 0) >= 1 + ) + target_node = ( + insert_result.inserted_host_node if insert_result is not None else None + ) + if ( + host_indices is None + or target_node is None + or not loaded + or target_node.component_data[ct].host_value is not None + ): + self.cache.cache_controller.append_host_mem_release( + extra_pools=[transfer] + ) + if insert_result is not None: + insert_result.mamba_exist = True + return + + target_node.component_data[ct].host_value = host_indices.clone() + if target_node.component_data[ct].value is None: + host_lru = self.cache.host_lru_lists[ct] + if not host_lru.in_list(target_node): + host_lru.insert_mru(target_node) + if insert_result is not None: + insert_result.mamba_exist = False + def drive_host_eviction( self, num_tokens: int, tracker: dict[ComponentType, int] ) -> None: @@ -445,4 +535,5 @@ def drive_host_eviction( x, self, target=EvictLayer.HOST, tracker=tracker ) self.cache._cascade_evict(x, self, tracker, target=EvictLayer.HOST) + self.cache._update_evictable_leaf_sets(x) x = x_next diff --git a/python/sglang/srt/mem_cache/unified_cache_components/swa_component.py b/python/sglang/srt/mem_cache/unified_cache_components/swa_component.py index 63223625e906..0ba006079928 100644 --- a/python/sglang/srt/mem_cache/unified_cache_components/swa_component.py +++ b/python/sglang/srt/mem_cache/unified_cache_components/swa_component.py @@ -350,7 +350,10 @@ def drive_eviction( x = x_next def acquire_component_lock( - self, node: UnifiedTreeNode, result: IncLockRefResult + self, + node: UnifiedTreeNode, + result: IncLockRefResult, + lock_host: bool = False, ) -> IncLockRefResult: ct = self.component_type root = self.cache.root_node @@ -384,7 +387,10 @@ def acquire_component_lock( return result def release_component_lock( - self, node: UnifiedTreeNode, params: Optional[DecLockRefParams] + self, + node: UnifiedTreeNode, + params: Optional[DecLockRefParams], + lock_host: bool = False, ) -> None: ct = self.component_type root = self.cache.root_node @@ -484,6 +490,7 @@ def commit_hicache_transfer( node: UnifiedTreeNode, phase: CacheTransferPhase, transfers: list[PoolTransfer] = (), + **kw, ) -> None: ct = self.component_type diff --git a/python/sglang/srt/mem_cache/unified_cache_components/tree_component.py b/python/sglang/srt/mem_cache/unified_cache_components/tree_component.py index ae6f71167a7b..2b9b03b88b04 100644 --- a/python/sglang/srt/mem_cache/unified_cache_components/tree_component.py +++ b/python/sglang/srt/mem_cache/unified_cache_components/tree_component.py @@ -276,9 +276,12 @@ def drive_eviction( @abstractmethod def acquire_component_lock( - self, node: UnifiedTreeNode, result: IncLockRefResult + self, + node: UnifiedTreeNode, + result: IncLockRefResult, + lock_host: bool = False, ) -> IncLockRefResult: - """Increment lock_ref for this component, protecting nodes from + """Increment component lock refs, protecting nodes from eviction. Updates evictable → protected size on first lock. - Full: path-lock — walks from node up to root, incrementing lock_ref on every ancestor. @@ -286,21 +289,31 @@ def acquire_component_lock( sliding window is filled; records a component_uuid at the boundary for release_component_lock to know where to stop. - Mamba: single-node lock — only increments lock_ref on the - node itself (mamba state is per-leaf, not per-path).""" + node itself (mamba state is per-leaf, not per-path). + + When ``lock_host`` is True, the lock applies to host-side state: + - Full: single-node host lock. + - SWA: host window-lock with a dedicated host UUID boundary. + - Mamba: single-node host lock with host LRU detach.""" ... @abstractmethod def release_component_lock( - self, node: UnifiedTreeNode, params: Optional[DecLockRefParams] + self, + node: UnifiedTreeNode, + params: Optional[DecLockRefParams], + lock_host: bool = False, ) -> None: - """Decrement lock_ref for this component, un-protecting nodes. + """Decrement component lock refs, un-protecting nodes. Updates protected → evictable size when lock_ref drops to 0. - Full: path-unlock — walks from node up to root, decrementing lock_ref on every ancestor. - SWA: path-unlock — walks upward, stopping at the node whose component_uuid matches the one recorded during acquire. - Mamba: single-node unlock — only decrements lock_ref on the - node itself.""" + node itself. + + When ``lock_host`` is True, the inverse host-side semantics apply.""" ... def prepare_for_caching_req( @@ -351,6 +364,7 @@ def commit_hicache_transfer( node: UnifiedTreeNode, phase: CacheTransferPhase, transfers: list[PoolTransfer] = (), + **kw, ) -> None: """Post-transfer bookkeeping: store host indices, update LRU, etc.""" pass diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index d16b41603664..a42c5d0c3907 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -3,8 +3,10 @@ import logging import threading import time +from array import array from collections import defaultdict -from functools import partial +from functools import lru_cache, partial +from queue import Empty from typing import TYPE_CHECKING, Any, Optional import torch @@ -27,6 +29,9 @@ PoolTransfer, SidecarPoolSpec, ) +from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( + HybridCacheController, +) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache_components import ( _NUM_COMPONENT_TYPES, @@ -41,6 +46,8 @@ TreeComponent, get_and_increase_time_counter, ) +from sglang.srt.mem_cache.utils import compute_node_hash_values, split_node_hash_value +from sglang.srt.observability.metrics_collector import StorageMetricsCollector from sglang.srt.session.streaming_session import StreamingSession if TYPE_CHECKING: @@ -92,6 +99,18 @@ def evicted(self) -> bool: def __lt__(self, other: UnifiedTreeNode): return self.last_access_time < other.last_access_time + def get_last_hash_value(self) -> Optional[str]: + if self.hash_value is None or len(self.hash_value) == 0: + return None + return self.hash_value[-1] + + @lru_cache(maxsize=1) + def get_prefix_hash_values(self, node: UnifiedTreeNode) -> list[str]: + if node is None or node.hash_value is None: + return [] + + return node.get_prefix_hash_values(node.parent) + node.hash_value + class UnifiedLRUList: def __init__( @@ -219,6 +238,10 @@ def __init__( if params.enable_metrics: self.init_metrics_collector() + self._enable_metrics_flag = params.enable_metrics + self.enable_storage_metrics = False + self.storage_metrics_collector: Optional[StorageMetricsCollector] = None + self.extra_metric_labels = None assert params.tree_components is not None self.tree_components = tuple(params.tree_components) @@ -247,6 +270,11 @@ def __init__( # HiCache D↔H defaults (overridden by init_hicache) self.cache_controller = None self.write_through_threshold = 256 + self.prefetch_stop_policy = "best_effort" + self.prefetch_threshold = 256 + self.prefetch_timeout_base = 1.0 + self.prefetch_timeout_per_page = 0.25 + self.hicache_storage_pass_prefix_keys = False self.reset() logger.info(f"Init Unified RadixTree with components {self.tree_components}") @@ -257,8 +285,9 @@ def reset(self) -> None: def _reset_full(self) -> None: """Full reset: destroy entire tree and all state.""" self.root_node = UnifiedTreeNode(self.tree_components) - self.root_node.key = RadixKey([], None) + self.root_node.key = RadixKey(array("q"), None) self.root_node.component_data[BASE_COMPONENT_TYPE].value = [] + self.root_node.hash_value = [] for ct in self.tree_components: self.root_node.component_data[ct].lock_ref = 1 self.component_evictable_size_ = {ct: 0 for ct in self.tree_components} @@ -280,12 +309,14 @@ def _reset_full(self) -> None: ] = {} self.ongoing_load_back: dict[int, tuple[UnifiedTreeNode, DecLockRefParams]] = {} self.enable_storage = False + self.prefetch_loaded_tokens_by_reqid: dict[str, int] = {} self.ongoing_prefetch: dict = {} self.ongoing_backup: dict = {} if self.cache_controller is not None: self.cache_controller.reset() self.cache_controller.mem_pool_host.clear() + self.enable_storage = self.cache_controller.enable_storage self._empty_match_result = MatchResult( device_indices=torch.empty( @@ -315,6 +346,26 @@ def init_hicache(self, server_args: ServerArgs, params: CacheInitParams) -> None self.load_cache_event = threading.Event() self.sidecar_pool_specs.clear() + self.extra_metric_labels = server_args.extra_metric_labels + + # Parse storage config once, share with assembler and tree + storage_backend = server_args.hicache_storage_backend + storage_extra_config = None + storage_prefetch_threshold = 256 + prefetch_timeout_base = 1.0 + prefetch_timeout_per_ki_token = 0.25 + hicache_storage_pass_prefix_keys = False + if storage_backend is not None: + ( + storage_extra_config, + storage_prefetch_threshold, + prefetch_timeout_base, + prefetch_timeout_per_ki_token, + hicache_storage_pass_prefix_keys, + ) = HybridCacheController.parse_storage_backend_extra_config( + server_args.hicache_storage_backend_extra_config + ) + attach_hybrid_pool_to_unified_cache( self, params, @@ -322,6 +373,9 @@ def init_hicache(self, server_args: ServerArgs, params: CacheInitParams) -> None load_cache_event=self.load_cache_event, attn_cp_group=params.attn_cp_cache_group, attn_tp_group=params.attn_tp_cache_group, + storage_backend=storage_backend, + storage_extra_config=storage_extra_config, + storage_prefetch_threshold=storage_prefetch_threshold, ) # State initialization @@ -329,14 +383,19 @@ def init_hicache(self, server_args: ServerArgs, params: CacheInitParams) -> None 1 if server_args.hicache_write_policy == "write_through" else 2 ) self.load_back_threshold = 256 - - logger.info( - f"HiCache D\u2194H initialized: " - f"host_pool_size={self.host_pool_group.size}, " - f"write_policy={server_args.hicache_write_policy}, " - f"tp_world_size={self.tp_world_size}, " - f"transfer_layer_num={self.cache_controller.layer_num}" - ) + self.prefetch_stop_policy = server_args.hicache_storage_prefetch_policy + + if storage_backend is not None: + self._apply_storage_runtime_config( + storage_backend=storage_backend, + prefetch_threshold=storage_prefetch_threshold, + prefetch_timeout_base=prefetch_timeout_base, + prefetch_timeout_per_ki_token=prefetch_timeout_per_ki_token, + hicache_storage_pass_prefix_keys=hicache_storage_pass_prefix_keys, + enable_storage=self.cache_controller.enable_storage, + enable_storage_metrics=self._enable_metrics_flag, + extra_metric_labels=self.extra_metric_labels, + ) def register_sidecar_pool(self, spec: SidecarPoolSpec) -> None: self.sidecar_pool_specs.append(spec) @@ -434,6 +493,29 @@ def dec_lock_ref( # TODO: delta is not aggregated from components; no caller uses it yet. return DecLockRefResult() + def inc_host_lock_ref(self, node: Any) -> IncLockRefResult: + if self.disable: + return IncLockRefResult() + result = IncLockRefResult() + for component in self._components_tuple: + result = component.acquire_component_lock( + node=node, result=result, lock_host=True + ) + + self._update_evictable_leaf_sets(node) + return result + + def dec_host_lock_ref( + self, node: Any, params: Optional[DecLockRefParams] = None + ) -> DecLockRefResult: + if self.disable: + return DecLockRefResult() + for component in self._components_tuple: + component.release_component_lock(node=node, params=params, lock_host=True) + + self._update_evictable_leaf_sets(node) + return DecLockRefResult() + def cache_finished_req(self, req: Req, is_insert: bool = True, **kwargs) -> None: if self.session.try_cache_finished_req(req, is_insert=is_insert, **kwargs): return @@ -702,13 +784,15 @@ def _match_post_processor( cur_time -= 0.00001 node_update = node_update.parent - # Walk up to find last_host_node for full component. - if self.cache_controller is None: - last_host_node = best_match_device_node - else: - last_host_node = best_match_node - while last_host_node is not self.root_node and not last_host_node.backuped: - last_host_node = last_host_node.parent + # last_host_node will be used as the starting node for the subsequent + # `prefetch_from_storage` flow. We directly use best_match_node here, + # because best_match_node represents the node where all components + # have reached consensus on both device & host availability. + last_host_node = ( + best_match_node + if self.cache_controller is not None + else best_match_device_node + ) if best_match_device_value_len > 0: device_indices = torch.cat(value[:best_match_device_value_len]) @@ -743,6 +827,9 @@ def _split_node( child.parent = new_node child.key = child.key[split_len:] + new_node.hash_value, child.hash_value = split_node_hash_value( + child.hash_value, split_len, self.page_size + ) for component in self._components_tuple: component.redistribute_on_node_split(new_parent=new_node, child=child) @@ -777,6 +864,8 @@ def _add_new_node( new_node.component_data[BASE_COMPONENT_TYPE].value = value.clone() parent.children[key.child_key(self.page_size)] = new_node self.component_evictable_size_[BASE_COMPONENT_TYPE] += len(value) + if self.enable_storage: + new_node.hash_value = compute_node_hash_values(new_node, self.page_size) self._update_evictable_leaf_sets(new_node) self._update_evictable_leaf_sets(parent) @@ -893,6 +982,58 @@ def _insert_helper( self._inc_hit_count(target_node, params.chunked) return result + def _insert_helper_host( + self, + node: UnifiedTreeNode, + key: RadixKey, + host_value: torch.Tensor, + hash_value: list[str], + ) -> InsertResult: + total_len = len(key) + self._touch_node(node) + if total_len == 0: + return InsertResult(prefix_len=0, mamba_exist=True) + + child_key = key.child_key(self.page_size) + matched_length = 0 + while len(key) > 0 and child_key in node.children: + node = node.children[child_key] + self._touch_node(node) + prefix_len = node.key.match(key, page_size=self.page_size) + + key = key[prefix_len:] + host_value = host_value[prefix_len:] + hash_value = hash_value[prefix_len // self.page_size :] + matched_length += prefix_len + + if prefix_len < len(node.key): + node = self._split_node(node.key, node, prefix_len) + + if len(key): + child_key = key.child_key(self.page_size) + + result = InsertResult( + prefix_len=matched_length, + ) + if len(key) == 0: + if ( + node is not self.root_node + and node.component_data[BASE_COMPONENT_TYPE].host_value is not None + ): + result.inserted_host_node = node + return result + + new_node = UnifiedTreeNode(self.tree_components) + new_node.parent = node + new_node.key = key + new_node.hash_value = hash_value + new_node.component_data[BASE_COMPONENT_TYPE].host_value = host_value.clone() + node.children[child_key] = new_node + self._update_evictable_leaf_sets(new_node) + self._update_evictable_leaf_sets(node) + result.inserted_host_node = new_node + return result + # ---- Evict Helpers ---- def _cascade_evict( @@ -1125,6 +1266,7 @@ def _evict_device_leaf( and self.cache_controller.write_policy == "write_back" ): self.write_backup(node, write_back=True) + self.writing_check(write_back=True) self._evict_to_host(node, tracker) return else: @@ -1169,7 +1311,8 @@ def write_backup(self, node: UnifiedTreeNode, write_back: bool = False) -> int: if not write_back and ( node.parent is not self.root_node and not node.parent.backuped ): - return 0 + if self.write_backup(node.parent) <= 0: + return 0 device_value = node.component_data[BASE_COMPONENT_TYPE].value kv_xfer = PoolTransfer(name=PoolName.KV, device_indices=device_value) @@ -1366,6 +1509,511 @@ def _inc_hit_count(self, node: UnifiedTreeNode, chunked: bool = False) -> None: if not node.backuped and node.hit_count >= self.write_through_threshold: self.write_backup(node) + def write_backup_storage(self, node: UnifiedTreeNode) -> None: + if ( + not self.enable_storage + or self.cache_controller is None + or not node.backuped + ): + return + + prefix_keys = None + if self.hicache_storage_pass_prefix_keys: + prefix_keys = node.get_prefix_hash_values(node.parent) + + comp_xfers: dict[ComponentType, list[PoolTransfer]] = {} + for comp in self._components_tuple: + if comp.component_type == BASE_COMPONENT_TYPE: + continue + transfers = comp.build_hicache_transfers( + node, + CacheTransferPhase.BACKUP_STORAGE, + ) + if transfers: + comp_xfers[comp.component_type] = transfers + + kv_xfer = PoolTransfer( + name=PoolName.KV, + host_indices=node.component_data[BASE_COMPONENT_TYPE].host_value, + keys=node.hash_value, + ) + sidecar_xfers = self._build_sidecar_transfers( + CacheTransferPhase.BACKUP_STORAGE, kv_xfer, comp_xfers + ) + aux_xfers = [x for xfers in comp_xfers.values() for x in xfers] + aux_xfers.extend(sidecar_xfers) + + operation_id = self.cache_controller.write_storage( + node.component_data[BASE_COMPONENT_TYPE].host_value, + node.key.token_ids, + node.hash_value, + prefix_keys, + extra_pools=aux_xfers or None, + ) + self.ongoing_backup[operation_id] = ( + node, + self.inc_host_lock_ref(node).to_dec_params(), + ) + + def prefetch_from_storage( + self, + req_id: str, + last_host_node: UnifiedTreeNode, + new_input_tokens: list[int], + last_hash: Optional[str] = None, + prefix_keys: Optional[list[str]] = None, + ) -> None: + if not self.enable_storage or self.cache_controller is None: + return + + extra_key = last_host_node.key.extra_key if last_host_node.key else None + prefetch_key = RadixKey( + new_input_tokens, + extra_key=extra_key, + is_bigram=self.is_eagle, + ).page_aligned(self.page_size) + prefetch_length = len(prefetch_key) + if ( + prefetch_length < self.prefetch_threshold + or self.cache_controller.prefetch_rate_limited() + ): + return + + anchor_lock_params = self.inc_host_lock_ref(last_host_node).to_dec_params() + host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length) + if host_indices is None: + self.evict_host(prefetch_length) + host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length) + if host_indices is None: + available_size = self.cache_controller.mem_pool_host.available_size() + prefetch_length = available_size - (available_size % self.page_size) + if prefetch_length >= self.prefetch_threshold: + prefetch_key = prefetch_key[:prefetch_length] + host_indices = self.cache_controller.mem_pool_host.alloc( + prefetch_length + ) + else: + self.dec_host_lock_ref(last_host_node, anchor_lock_params) + return + if host_indices is None: + self.dec_host_lock_ref(last_host_node, anchor_lock_params) + return + + comp_xfers: dict[ComponentType, list[PoolTransfer]] = {} + alloc_failed = False + for comp in self._components_tuple: + if comp.component_type == BASE_COMPONENT_TYPE: + continue + transfers = comp.build_hicache_transfers( + last_host_node, + CacheTransferPhase.PREFETCH, + token_ids=prefetch_key.token_ids, + prefetch_tokens=len(prefetch_key), + last_hash=last_hash, + ) + if transfers == []: + alloc_failed = True + break + if transfers: + comp_xfers[comp.component_type] = transfers + kv_xfer = PoolTransfer(name=PoolName.KV, host_indices=host_indices) + sidecar_xfers = self._build_sidecar_transfers( + CacheTransferPhase.PREFETCH, kv_xfer, comp_xfers + ) + if alloc_failed: + self.cache_controller.append_host_mem_release( + host_indices=host_indices, + extra_pools=[x for xfers in comp_xfers.values() for x in xfers], + ) + self.dec_host_lock_ref(last_host_node, anchor_lock_params) + return + + aux_xfers = [x for xfers in comp_xfers.values() for x in xfers] + aux_xfers.extend(sidecar_xfers) + operation = self.cache_controller.prefetch( + req_id, + host_indices, + prefetch_key.token_ids, + last_hash, + prefix_keys, + extra_pools=aux_xfers or None, + ) + self.ongoing_prefetch[req_id] = ( + last_host_node, + prefetch_key, + host_indices, + operation, + anchor_lock_params, + comp_xfers, + ) + self.cache_controller.prefetch_tokens_occupied += len(prefetch_key) + + def _prefetch_timeout_check_linear_func(self, operation) -> bool: + return ( + time.monotonic() - operation.start_time + > self.prefetch_timeout_base + + len(operation.hash_value) * self.prefetch_timeout_per_page + ) + + def can_terminate_prefetch(self, operation) -> bool: + if self.prefetch_stop_policy == "best_effort": + return True + + if len(operation.hash_value) == 0: + completed = False + else: + completed = ( + operation.completed_tokens == len(operation.hash_value) * self.page_size + ) + + if self.prefetch_stop_policy == "wait_complete": + can_terminate = completed + elif self.prefetch_stop_policy == "timeout": + can_terminate = completed or self._prefetch_timeout_check_linear_func( + operation + ) + else: + return True + + operation_terminated = operation.is_terminated() + states = torch.tensor( + [1 - int(can_terminate), int(operation_terminated)], + dtype=torch.int, + ) + if self.tp_world_size > 1: + torch.distributed.all_reduce( + states, op=torch.distributed.ReduceOp.MAX, group=self.tp_group + ) + can_terminate = states[0].item() == 0 + operation_terminated = states[1].item() == 1 + return can_terminate or operation_terminated + + def check_prefetch_progress(self, req_id: str) -> bool: + if req_id not in self.ongoing_prefetch: + return True + + ( + last_host_node, + prefetch_key, + host_indices, + operation, + anchor_lock_params, + comp_xfers, + ) = self.ongoing_prefetch[req_id] + if operation.host_indices is None: + return True + if not self.can_terminate_prefetch(operation): + return False + + completed_tokens, hash_value = self.cache_controller.terminate_prefetch( + operation + ) + min_completed_tokens = completed_tokens + if self.tp_world_size > 1: + completed_tokens_tensor = torch.tensor( + min_completed_tokens, dtype=torch.int + ) + torch.distributed.all_reduce( + completed_tokens_tensor, + op=torch.distributed.ReduceOp.MIN, + group=self.tp_group, + ) + min_completed_tokens = int(completed_tokens_tensor.item()) + + fetched_key = prefetch_key[:min_completed_tokens] + insert_result = self._insert_helper_host( + last_host_node, + fetched_key, + host_indices[:min_completed_tokens], + hash_value[: min_completed_tokens // self.page_size], + ) + + for ct, xfers in comp_xfers.items(): + self.components[ct].commit_hicache_transfer( + last_host_node, + CacheTransferPhase.PREFETCH, + xfers, + insert_result=insert_result, + pool_storage_result=operation.pool_storage_result, + ) + + self.cache_controller.mem_pool_host.free( + host_indices[: insert_result.prefix_len] + ) + self.cache_controller.append_host_mem_release( + host_indices[min_completed_tokens:completed_tokens] + ) + self.dec_host_lock_ref(last_host_node, anchor_lock_params) + del self.ongoing_prefetch[req_id] + self.cache_controller.prefetch_tokens_occupied -= len(prefetch_key) + + loaded_from_storage = min_completed_tokens - insert_result.prefix_len + self.prefetch_loaded_tokens_by_reqid[req_id] = loaded_from_storage + logger.info( + "HiCache prefetch success req=%s completed_local=%d completed_synced=%d matched=%d loaded=%d tail_release=%d occupied=%d", + req_id, + completed_tokens, + min_completed_tokens, + insert_result.prefix_len, + loaded_from_storage, + completed_tokens - min_completed_tokens, + self.cache_controller.prefetch_tokens_occupied, + ) + if self.enable_storage_metrics and self.storage_metrics_collector is not None: + self.storage_metrics_collector.log_prefetched_tokens(loaded_from_storage) + return True + + def terminate_prefetch(self, req_id: str) -> None: + if req_id not in self.ongoing_prefetch: + return + _, _, _, operation, _, _ = self.ongoing_prefetch[req_id] + if operation.host_indices is None: + return + operation.mark_terminate() + + def pop_prefetch_loaded_tokens(self, req_id: str) -> int: + return self.prefetch_loaded_tokens_by_reqid.pop(req_id, 0) + + def release_aborted_request(self, rid: str) -> None: + self.prefetch_loaded_tokens_by_reqid.pop(rid, None) + if rid not in self.ongoing_prefetch: + return + + ( + last_host_node, + prefetch_key, + host_indices, + operation, + anchor_lock_params, + comp_xfers, + ) = self.ongoing_prefetch[rid] + if operation.host_indices is None: + return + + completed_tokens, _ = self.cache_controller.terminate_prefetch(operation) + if self.tp_world_size > 1: + torch.distributed.barrier(group=self.tp_group) + self.dec_host_lock_ref(last_host_node, anchor_lock_params) + del self.ongoing_prefetch[rid] + self.cache_controller.append_host_mem_release( + host_indices=host_indices[:completed_tokens], + extra_pools=[x for xfers in comp_xfers.values() for x in xfers], + ) + self.cache_controller.prefetch_tokens_occupied -= len(prefetch_key) + + def _drain_storage_control_queues_impl( + self, + n_revoke: Optional[int], + n_backup: Optional[int], + n_release: Optional[int], + extra_release_counts: Optional[dict[PoolName, int]], + log_metrics: bool, + ) -> None: + cc = self.cache_controller + + def _drain_queue(q, limit: Optional[int]): + drained = 0 + while limit is None or drained < limit: + try: + item = q.get_nowait() + except Empty: + break + drained += 1 + yield item + + def _drain_revoke(): + drained = 0 + for req_id in _drain_queue(cc.prefetch_revoke_queue, n_revoke): + info = self.ongoing_prefetch.pop(req_id, None) + if info is None: + continue + drained += 1 + ( + last_host_node, + prefetch_key, + _host_indices, + _operation, + anchor_lock_params, + comp_xfers, + ) = info + cc.append_host_mem_release( + extra_pools=[x for xfers in comp_xfers.values() for x in xfers] + ) + self.dec_host_lock_ref(last_host_node, anchor_lock_params) + cc.prefetch_tokens_occupied -= len(prefetch_key) + if cc.prefetch_tokens_occupied < 0: + cc.prefetch_tokens_occupied = 0 + return drained + + def _drain_backup(): + drained = 0 + for operation in _drain_queue(cc.ack_backup_queue, n_backup): + drained += 1 + entry = self.ongoing_backup.pop(operation.id, None) + if entry is not None: + node, lock_params = entry + self.dec_host_lock_ref(node, lock_params) + if ( + log_metrics + and self.enable_storage_metrics + and self.storage_metrics_collector is not None + ): + self.storage_metrics_collector.log_backuped_tokens( + operation.completed_tokens + ) + return drained + + def _drain_release(): + host_indices_list = [] + released_tokens = 0 + for host_indices in _drain_queue(cc.host_mem_release_queue, n_release): + host_indices_list.append(host_indices) + released_tokens += len(host_indices) + if host_indices_list: + cc.mem_pool_host.free(torch.cat(host_indices_list, dim=0)) + return len(host_indices_list), released_tokens + + def _drain_extra_release(): + drained: dict[PoolName, tuple[int, int]] = {} + if not extra_release_counts: + return drained + for pool_name, limit in extra_release_counts.items(): + release_queue = cc.extra_host_mem_release_queues.get(pool_name) + if release_queue is None: + continue + host_indices_list = [] + released_tokens = 0 + for host_indices in _drain_queue(release_queue, limit): + host_indices_list.append(host_indices) + released_tokens += len(host_indices) + if host_indices_list: + entry = cc.mem_pool_host.entry_map.get(pool_name) + if entry is not None: + entry.host_pool.free(torch.cat(host_indices_list, dim=0)) + drained[pool_name] = (len(host_indices_list), released_tokens) + return drained + + _drain_revoke() + _drain_backup() + _drain_release() + _drain_extra_release() + + def drain_storage_control_queues(self) -> None: + cc = self.cache_controller + extra_release_queues = getattr(cc, "extra_host_mem_release_queues", {}) + extra_pool_names = list(extra_release_queues) + local_qsize_list = [ + cc.prefetch_revoke_queue.qsize(), + cc.ack_backup_queue.qsize(), + cc.host_mem_release_queue.qsize(), + *[ + extra_release_queues[pool_name].qsize() + for pool_name in extra_pool_names + ], + ] + qsizes = torch.tensor( + local_qsize_list, + dtype=torch.int, + ) + if self.tp_world_size > 1: + torch.distributed.all_reduce( + qsizes, op=torch.distributed.ReduceOp.MIN, group=self.tp_group + ) + qsize_list = list(map(int, qsizes.tolist())) + n_revoke, n_backup, n_release = qsize_list[:3] + extra_release_counts = { + pool_name: count + for pool_name, count in zip(extra_pool_names, qsize_list[3:]) + } + self._drain_storage_control_queues_impl( + n_revoke=n_revoke, + n_backup=n_backup, + n_release=n_release, + extra_release_counts=extra_release_counts, + log_metrics=True, + ) + + def _apply_storage_runtime_config( + self, + *, + storage_backend: Optional[str], + prefetch_threshold: int, + prefetch_timeout_base: float, + prefetch_timeout_per_ki_token: float, + hicache_storage_pass_prefix_keys: bool, + enable_storage: bool, + enable_storage_metrics: bool, + extra_metric_labels: Optional[dict[str, str]], + ) -> None: + self.enable_storage = enable_storage + self.prefetch_threshold = prefetch_threshold + self.prefetch_timeout_base = prefetch_timeout_base + self.prefetch_timeout_per_page = ( + self.page_size / 1024 * prefetch_timeout_per_ki_token + ) + self.hicache_storage_pass_prefix_keys = hicache_storage_pass_prefix_keys + self.enable_storage_metrics = enable_storage_metrics + + if self.enable_storage_metrics: + attn_cp_rank, attn_cp_size = ( + self.cache_controller.get_attn_cp_rank_and_size() + ) + labels = { + "storage_backend": storage_backend, + "tp_rank": self.cache_controller.tp_rank, + "dp_rank": self.cache_controller.dp_rank, + "pp_rank": self.cache_controller.pp_rank, + "pp_size": self.cache_controller.pp_size, + "attn_cp_rank": attn_cp_rank, + "attn_cp_size": attn_cp_size, + } + if extra_metric_labels: + labels.update(extra_metric_labels) + existing_collector = self.storage_metrics_collector + if existing_collector is None: + self.storage_metrics_collector = StorageMetricsCollector(labels=labels) + elif set(existing_collector.labels.keys()) == set(labels.keys()): + existing_collector.labels = labels + else: + logger.warning( + "Storage metrics labels changed (%s -> %s). Keep existing labels to avoid duplicate metric registration.", + sorted(existing_collector.labels.keys()), + sorted(labels.keys()), + ) + else: + self.storage_metrics_collector = None + + def attach_storage_backend( + self, + storage_backend: str, + storage_backend_extra_config_json: Optional[str] = None, + served_model_name: Optional[str] = None, + hicache_storage_prefetch_policy: Optional[str] = None, + hicache_write_policy: Optional[str] = None, + ) -> tuple[bool, str]: + return ( + False, + "UnifiedRadixCache does not support runtime HiCache storage attach yet. " + "Configure hicache_storage_backend at startup instead.", + ) + + def detach_storage_backend(self) -> tuple[bool, str]: + return ( + False, + "UnifiedRadixCache does not support runtime HiCache storage detach yet. " + "Restart without hicache_storage_backend to disable it.", + ) + + def clear_storage_backend(self) -> bool: + try: + ok = self.cache_controller.clear_storage_backend() + except Exception as e: + logger.error("Failed to clear hierarchical cache storage backend: %s", e) + return False + if ok: + logger.info("Hierarchical cache storage backend cleared successfully!") + return ok + # ---- HiCache: Async Event Management ---- def writing_check(self, write_back: bool = False) -> None: @@ -1385,6 +2033,8 @@ def writing_check(self, write_back: bool = False) -> None: node, params = entry if params is not None: self.dec_lock_ref(node, params) + if self.enable_storage: + self.write_backup_storage(node) cc.ack_write_queue.clear() assert len(self.ongoing_write_through) == 0 return @@ -1413,6 +2063,8 @@ def writing_check(self, write_back: bool = False) -> None: for ack_id in ack_list: node, params = self.ongoing_write_through.pop(ack_id) self.dec_lock_ref(node, params) + if self.enable_storage: + self.write_backup_storage(node) finish_count -= 1 def loading_check(self) -> None: @@ -1482,6 +2134,12 @@ def check_hicache_events(self) -> None: """Called per scheduler step to poll async HiCache events.""" self.writing_check() self.loading_check() + if self.enable_storage: + self.drain_storage_control_queues() + if self.enable_storage_metrics and self.storage_metrics_collector is not None: + self.storage_metrics_collector.log_storage_metrics( + self.cache_controller.storage_backend.get_stats() + ) def flush_write_through_acks(self) -> None: """Flush pending write-through acknowledgements.""" @@ -1881,10 +2539,6 @@ def sanity_check(self): logger.error(msg) self.pretty_print() raise AssertionError(msg) - logger.debug( - f"Sanity check PASSED: {len(all_nodes)} nodes, " - f"{len(self.tree_components)} components" - ) def _check_lru_linked_list( self, diff --git a/python/sglang/srt/model_executor/breakable_cuda_graph_runner.py b/python/sglang/srt/model_executor/breakable_cuda_graph_runner.py index 1365d80fe051..7d2a495c90b9 100644 --- a/python/sglang/srt/model_executor/breakable_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/breakable_cuda_graph_runner.py @@ -57,6 +57,7 @@ CaptureHiddenMode, PPProxyTensors, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.piecewise_cuda_graph_runner import ( PiecewiseCudaGraphRunner, freeze_gc, @@ -292,9 +293,6 @@ def _build_capture_forward_batch(self, num_tokens): next_token_logits_buffer=None, orig_seq_lens=orig_seq_lens, seq_lens_cpu=torch.tensor([num_tokens], device="cpu"), - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, - attn_backend=self.model_runner.attn_backend, out_cache_loc=buffers.out_cache_loc[:num_tokens], seq_lens_sum=num_tokens, mamba_track_indices=None, @@ -329,8 +327,11 @@ def _warmup(self): """Warmup the model with a forward pass.""" num_tokens = self.capture_num_tokens[0] forward_batch = self._build_capture_forward_batch(num_tokens) - self.model_runner.attn_backend.init_forward_metadata(forward_batch) - self._run_forward(forward_batch, num_tokens) + with forward_context( + ForwardContext(attn_backend=self.model_runner.attn_backend) + ): + self.model_runner.attn_backend.init_forward_metadata(forward_batch) + self._run_forward(forward_batch, num_tokens) def _capture_all(self): """Capture breakable CUDA graphs for all token sizes.""" @@ -394,14 +395,17 @@ def run_once(): self.model_runner.token_to_kv_pool.invalidate_loc_cache() return self._run_forward(forward_batch, num_tokens) - for _ in range(2): - self.device_module.synchronize() - self.model_runner.tp_group.barrier() - run_once() - - graph = BreakableCUDAGraph() - with BreakableCUDAGraphCapture(cuda_graph=graph, pool=pool, stream=stream): - output = run_once() + with forward_context( + ForwardContext(attn_backend=self.model_runner.attn_backend) + ): + for _ in range(2): + self.device_module.synchronize() + self.model_runner.tp_group.barrier() + run_once() + + graph = BreakableCUDAGraph() + with BreakableCUDAGraphCapture(cuda_graph=graph, pool=pool, stream=stream): + output = run_once() return graph, output diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index edf0dadcb9c9..4b7c177a1cb6 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -36,6 +36,7 @@ PPProxyTensors, enable_num_token_non_padded, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.utils import ( log_info_on_rank0, require_attn_tp_gather, @@ -679,9 +680,6 @@ def capture_one_batch_size(self, bs: int, forward: Callable): input_ids=input_ids, req_pool_indices=req_pool_indices, seq_lens=seq_lens, - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, - attn_backend=self.model_runner.attn_backend, out_cache_loc=out_cache_loc, seq_lens_sum=seq_lens.sum().item(), return_logprob=False, @@ -693,43 +691,46 @@ def capture_one_batch_size(self, bs: int, forward: Callable): num_token_non_padded=self.num_token_non_padded, global_forward_mode=self.capture_forward_mode, ) - self.model_runner.attn_backend.init_forward_metadata_capture_cpu_graph( - bs, - num_tokens, - req_pool_indices, - seq_lens, - None, - forward_batch.forward_mode, - forward_batch.spec_info, - ) - # Do infernence to avoid setting attr at runtime, e.g., - # self.attn_mha.kv_b_proj = self.kv_b_proj for full graph compile on CPU - with torch.no_grad(): - self.model_runner.tp_group.barrier() - self.model_runner.model.forward( - forward_batch.input_ids, - forward_batch.positions, - forward_batch, + with forward_context( + ForwardContext(attn_backend=self.model_runner.attn_backend) + ): + self.model_runner.attn_backend.init_forward_metadata_capture_cpu_graph( + bs, + num_tokens, + req_pool_indices, + seq_lens, + None, + forward_batch.forward_mode, + forward_batch.spec_info, ) + with torch.no_grad(): + self.model_runner.tp_group.barrier() + self.model_runner.model.forward( + forward_batch.input_ids, + forward_batch.positions, + forward_batch, + ) - # Run and capture - def run_once(): - # Clean intermediate result cache for DP attention - forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = None - logits_output_or_pp_proxy_tensors = forward( - forward_batch.input_ids, - forward_batch.positions, - forward_batch, - ) - return logits_output_or_pp_proxy_tensors + # Run and capture + def run_once(): + # Clean intermediate result cache for DP attention + forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = ( + None + ) + logits_output_or_pp_proxy_tensors = forward( + forward_batch.input_ids, + forward_batch.positions, + forward_batch, + ) + return logits_output_or_pp_proxy_tensors - with torch.no_grad(): - for _ in range(2): - self.model_runner.tp_group.barrier() - out = run_once() - # Save the captured forward_batch - self.captured_forward_batches[bs] = forward_batch - return forward, out + with torch.no_grad(): + for _ in range(2): + self.model_runner.tp_group.barrier() + out = run_once() + # Save the captured forward_batch + self.captured_forward_batches[bs] = forward_batch + return forward, out def recapture_if_needed(self, forward_batch: ForwardBatch): @@ -785,6 +786,9 @@ def prepare_replay( assert captured_forward_batch is not None captured_forward_batch.seq_lens.fill_(self.seq_len_fill_value) captured_forward_batch.out_cache_loc.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init). + captured_forward_batch.req_pool_indices.zero_() captured_forward_batch.input_ids[:raw_num_token].copy_(forward_batch.input_ids) captured_forward_batch.req_pool_indices[:raw_bs].copy_( forward_batch.req_pool_indices diff --git a/python/sglang/srt/model_executor/cuda_graph_runner.py b/python/sglang/srt/model_executor/cuda_graph_runner.py index c2e6d121fcfa..6501b6fe1d68 100644 --- a/python/sglang/srt/model_executor/cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/cuda_graph_runner.py @@ -56,6 +56,7 @@ from sglang.srt.layers.moe.token_dispatcher.deepep import DeepEPBuffer from sglang.srt.layers.moe.utils import get_deepep_mode, get_moe_a2a_backend from sglang.srt.layers.utils import MultiPlatformOp +from sglang.srt.layers.utils.cp_utils import is_mla_prefill_cp_enabled from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, ForwardBatch, @@ -65,6 +66,7 @@ compute_local_num_token_non_padded, enable_num_token_non_padded, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.input_buffers import ForwardInputBuffers from sglang.srt.multiplex.pdmux_context import get_current_stream_idx, get_stream_groups from sglang.srt.utils import ( @@ -284,6 +286,11 @@ def populate_from_forward_batch( if bs != raw_bs: self.seq_lens.fill_(seq_len_fill_value) self.out_cache_loc.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init), + # so dummy attention reads land on slot 0 instead of a stale + # req_to_token row left by an earlier replay. + self.req_pool_indices.zero_() if self.mamba_track_indices is not None: self.mamba_track_indices.zero_() if self.mamba_track_mask is not None: @@ -566,7 +573,15 @@ def __init__( self.attn_tp_size = get_attention_tp_size() self.attn_tp_rank = get_attention_tp_rank() - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() + # True if a DSACPLayerCommunicator-style prefill-CP flavor is active + # (DSA or MLA). These flavors feed a zigzag-split rank-local layout + # into the runner; MHA-arch prefill CP (Qwen3/Qwen2 MoE via PR + # #18233) uses the plain LayerCommunicator with an attn_tp-replicated + # layout and is intentionally excluded so the attn_tp-local + # num_token_non_padded adjustment still runs for it. + self.enable_prefill_cp = ( + is_dsa_enable_prefill_cp() or is_mla_prefill_cp_enabled() + ) self.deepep_adapter = DeepEPCudaGraphRunnerAdapter() @@ -927,7 +942,7 @@ def capture_one_batch_size( if ( enable_num_token_non_padded() and self.require_gathered_buffer - and not self.dsa_enable_prefill_cp + and not self.enable_prefill_cp ): local = compute_local_num_token_non_padded( global_num_token_non_padded=buffers.num_token_non_padded, @@ -1016,9 +1031,6 @@ def capture_one_batch_size( seq_lens_cpu=seq_lens_cpu, next_token_logits_buffer=next_token_logits_buffer, orig_seq_lens=seq_lens, - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, - attn_backend=attn_backend, out_cache_loc=out_cache_loc, seq_lens_sum=seq_lens.sum().item(), mamba_track_indices=mamba_track_indices, @@ -1040,85 +1052,90 @@ def capture_one_batch_size( lora_ids=lora_ids, ) - # HiSparse: set coordinator so the hisparse code path is captured into the graph - forward_batch.hisparse_coordinator = self.model_runner.hisparse_coordinator - if forward_batch.hisparse_coordinator is not None: - forward_batch.hisparse_coordinator.num_real_reqs.fill_(bs) + # Trip the coordinator so the hisparse code path is captured into the + # graph; backends read it from self.model_runner.hisparse_coordinator. + hisparse_coordinator = self.model_runner.hisparse_coordinator + if hisparse_coordinator is not None: + hisparse_coordinator.num_real_reqs.fill_(bs) if buffers.ngram_embedding_info is not None: forward_batch.ngram_embedding_info = buffers.ngram_embedding_info.slice(bs) - self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens) + # All setup hooks below read get_attn_backend() (TboForwardBatchPreparer, + # DeepEP adapter, …) so they must run inside the same ForwardContext + # that wraps the warmup/capture forward. + with forward_context(ForwardContext(attn_backend=attn_backend)): + self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens) - if lora_ids is not None: - self.model_runner.lora_manager.prepare_lora_batch(forward_batch) + if lora_ids is not None: + self.model_runner.lora_manager.prepare_lora_batch(forward_batch) - # Attention backend - attn_backend.init_forward_metadata_capture_cuda_graph( - bs, - num_tokens, - req_pool_indices, - seq_lens, - encoder_lens, - forward_batch.forward_mode, - forward_batch.spec_info, - ) - - # Run and capture - def run_once(): - # Without this, warmup-1 caches the translation; the capture run gets - # a hit, skips the gather, and replay reuses stale SWA locations. - if self.model_runner.is_hybrid_swa: - self.model_runner.token_to_kv_pool.invalidate_loc_cache() - - # Clean intermediate result cache for DP attention - forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = None - set_dp_buffer_len( - global_dp_buffer_len, + attn_backend.init_forward_metadata_capture_cuda_graph( + bs, num_tokens, - forward_batch.dp_padding_mode.is_max_len(), + req_pool_indices, + seq_lens, + encoder_lens, + forward_batch.forward_mode, + forward_batch.spec_info, ) - set_is_extend_in_batch(False) - kwargs = {} - if ( - self.pp_size > 1 - and "pp_proxy_tensors" in inspect.signature(forward).parameters - ): - kwargs["pp_proxy_tensors"] = PPProxyTensors( - {k: v.clone() for k, v in pp_proxy_tensors.tensors.items()} + def run_once(): + # Without this, warmup-1 caches the translation; the capture + # run hits the cache, skips the gather, and replay reuses + # stale SWA locations. + if self.model_runner.is_hybrid_swa: + self.model_runner.token_to_kv_pool.invalidate_loc_cache() + + forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = ( + None ) - if ( - self.model_runner.spec_algorithm.is_dflash() - and self.model_runner.is_draft_worker - and "input_embeds" in inspect.signature(forward).parameters - ): - kwargs["input_embeds"] = buffers.input_embeds[:num_tokens] + set_dp_buffer_len( + global_dp_buffer_len, + num_tokens, + forward_batch.dp_padding_mode.is_max_len(), + ) + set_is_extend_in_batch(False) - logits_output_or_pp_proxy_tensors = forward( - input_ids, - forward_batch.positions, - forward_batch, - **kwargs, - ) - return logits_output_or_pp_proxy_tensors + kwargs = {} + if ( + self.pp_size > 1 + and "pp_proxy_tensors" in inspect.signature(forward).parameters + ): + kwargs["pp_proxy_tensors"] = PPProxyTensors( + {k: v.clone() for k, v in pp_proxy_tensors.tensors.items()} + ) + if ( + self.model_runner.spec_algorithm.is_dflash() + and self.model_runner.is_draft_worker + and "input_embeds" in inspect.signature(forward).parameters + ): + kwargs["input_embeds"] = buffers.input_embeds[:num_tokens] + + logits_output_or_pp_proxy_tensors = forward( + input_ids, + forward_batch.positions, + forward_batch, + **kwargs, + ) + return logits_output_or_pp_proxy_tensors - self.deepep_adapter.capture(is_extend_in_batch=False) + self.deepep_adapter.capture(is_extend_in_batch=False) - for _ in range(2): - self.device_module.synchronize() - self.model_runner.tp_group.barrier() - run_once() - attn_backend.on_after_cuda_graph_warmup() + for _ in range(2): + self.device_module.synchronize() + self.model_runner.tp_group.barrier() + run_once() + attn_backend.on_after_cuda_graph_warmup() - if get_global_graph_memory_pool() is None: - set_global_graph_memory_pool(self.device_module.graph_pool_handle()) - # Set graph pool id globally to be able to use symmetric memory - set_graph_pool_id(get_global_graph_memory_pool()) + if get_global_graph_memory_pool() is None: + set_global_graph_memory_pool(self.device_module.graph_pool_handle()) + # Set graph pool id globally to be able to use symmetric memory + set_graph_pool_id(get_global_graph_memory_pool()) - out = self._capture_graph( - graph, get_global_graph_memory_pool(), stream, run_once - ) + out = self._capture_graph( + graph, get_global_graph_memory_pool(), stream, run_once + ) return graph, out @@ -1188,7 +1205,9 @@ def replay_prepare( seq_len_fill_value=self.seq_len_fill_value, require_gathered_buffer=self.require_gathered_buffer, num_tokens_per_bs=self.num_tokens_per_bs, - dsa_enable_prefill_cp=self.dsa_enable_prefill_cp, + # Parameter name retained for API stability; semantically this is + # "any prefill-CP flavor enabled" (DSA CP or MLA CP). + dsa_enable_prefill_cp=self.enable_prefill_cp, enable_num_token_non_padded_flag=enable_num_token_non_padded(), pp_proxy_tensors=pp_proxy_tensors, ) diff --git a/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py b/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py index 769d3023523a..72b09187d67f 100644 --- a/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py +++ b/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py @@ -9,6 +9,10 @@ from sglang.srt.environ import envs from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.srt.model_executor.forward_context import ( + get_req_to_token_pool, + get_token_to_kv_pool, +) class ForwardBatchDeepSeekMHAMixin: @@ -55,6 +59,7 @@ def set_attn_attend_prefix_cache(self, attn_attend_prefix_cache: bool): def prepare_chunked_kv_indices(self, device: torch.device): self.prefix_chunk_kv_indices = [] + req_to_token = get_req_to_token_pool().req_to_token for idx in range(self.num_prefix_chunks): chunk_starts = self.prefix_chunk_starts[idx] chunk_seq_lens = self.prefix_chunk_seq_lens[idx] @@ -66,13 +71,13 @@ def prepare_chunked_kv_indices(self, device: torch.device): ) create_chunked_prefix_cache_kv_indices[(self.batch_size,)]( - self.req_to_token_pool.req_to_token, + req_to_token, self.req_pool_indices, chunk_starts, chunk_seq_lens, chunk_cu_seq_lens, chunk_kv_indices, - self.req_to_token_pool.req_to_token.shape[1], + req_to_token.shape[1], ) self.prefix_chunk_kv_indices.append(chunk_kv_indices) @@ -108,10 +113,14 @@ def get_prefix_chunk_seq_lens( # Some of the codes are adapted from https://github.com/vllm-project/vllm/blob/main/vllm/v1/attention/backends/mla/common.py def prepare_chunked_prefix_cache_info(self, device: torch.device): - from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + from sglang.srt.mem_cache.memory_pool import ( + HybridLinearKVPool, + MLATokenToKVPool, + ) - assert isinstance( - self.token_to_kv_pool, MLATokenToKVPool + assert isinstance(get_token_to_kv_pool(), MLATokenToKVPool) or ( + isinstance(get_token_to_kv_pool(), HybridLinearKVPool) + and isinstance(get_token_to_kv_pool().full_kv_pool, MLATokenToKVPool) ), "Currently chunked prefix cache can only be used by Deepseek models" if not any(self.extend_prefix_lens_cpu): @@ -191,14 +200,15 @@ def fetch_mha_one_shot_kv_indices(self): device=self.req_pool_indices.device, ) kv_indptr[1:] = torch.cumsum(self.seq_lens, dim=0) + req_to_token = get_req_to_token_pool().req_to_token create_flashinfer_kv_indices_triton[(self.batch_size,)]( - self.req_to_token_pool.req_to_token, + req_to_token, self.req_pool_indices, self.seq_lens, kv_indptr, None, kv_indices, - self.req_to_token_pool.req_to_token.shape[1], + req_to_token.shape[1], ) self.mha_one_shot_kv_indices = kv_indices return kv_indices diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index e0b8447afbc9..d31c91342f25 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -63,11 +63,8 @@ from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: - from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.logits_processor import LogitsProcessorOutput - from sglang.srt.managers.hisparse_coordinator import HiSparseCoordinator from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch - from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm @@ -375,11 +372,6 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Sampling info sampling_info: SamplingBatchInfo = None - # Attention backend - req_to_token_pool: ReqToTokenPool = None - token_to_kv_pool: KVCache = None - attn_backend: AttentionBackend = None - # For DP attention original_global_num_tokens_cpu: Optional[List[int]] = None global_num_tokens_cpu: Optional[List[int]] = None @@ -438,9 +430,6 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Whether to return pooled hidden states (pre-head transformer output) return_pooled_hidden_states: bool = False - # For hisparse - hisparse_coordinator: Optional[HiSparseCoordinator] = None - # For ngram embedding ngram_embedding_info: Optional[NgramEmbeddingInfo] = None @@ -542,9 +531,6 @@ def init_new( multi_item_delimiter_indices=batch.multi_item_delimiter_indices, lora_ids=[req.lora_id for req in batch.reqs], sampling_info=batch.sampling_info, - req_to_token_pool=model_runner.req_to_token_pool, - token_to_kv_pool=model_runner.token_to_kv_pool, - attn_backend=model_runner.attn_backend, spec_algorithm=batch.spec_algorithm, spec_info=batch.spec_info, capture_hidden_mode=capture_hidden_mode, diff --git a/python/sglang/srt/model_executor/forward_context.py b/python/sglang/srt/model_executor/forward_context.py new file mode 100644 index 000000000000..3a3a7e50fc31 --- /dev/null +++ b/python/sglang/srt/model_executor/forward_context.py @@ -0,0 +1,84 @@ +"""Per-forward-call control context. + +Owns ``ForwardContext`` — a frozen dataclass holding control configs the model +layer reads at depth via ``get_forward_context()``. The only mandatory field +today is ``attn_backend``; pool refs are derived from ``attn_backend.*`` +(every backend caches them at ``__init__``), so a published ``ForwardContext`` +is enough to resolve the active pools without a separate global. + +``ModelRunner._forward_raw`` publishes a fresh ``ForwardContext`` for the +duration of each forward; callers that need a per-call override (PDmux +per-stream backend, frozen-KV MTP draft loop, TBO per-child dispatch) use +``dataclasses.replace`` and wrap the override scope with ``forward_context()``. + +Distinct from ``sglang.srt.compilation.piecewise_context_manager.ForwardContext``, +which collects compilation-time refs for the piecewise CUDA graph backend. + +Concurrency: ``_current`` is a plain module-level global, not thread-local. +This matches the ``global_server_args`` precedent and is safe because each +forward runs synchronously on a single Python thread per worker process. If +worker threads ever share a process, migrate to ``contextvars.ContextVar``. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from sglang.srt.layers.attention.base_attn_backend import AttentionBackend + from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool + + +@dataclass(frozen=True, slots=True) +class ForwardContext: + """Per-forward-call control configs. Read via ``get_forward_context()``; + extend by adding fields here. Frozen so accidental mutation raises at + write time — use ``dataclasses.replace`` for per-call overrides.""" + + attn_backend: AttentionBackend + + +_current: Optional[ForwardContext] = None + + +def set_forward_context(ctx: Optional[ForwardContext]) -> Optional[ForwardContext]: + """Set the active context; return the previous one for explicit + save/restore. Prefer the ``forward_context()`` context manager.""" + global _current + prev, _current = _current, ctx + return prev + + +def has_forward_context() -> bool: + return _current is not None + + +def get_forward_context() -> ForwardContext: + assert _current is not None, ( + "no forward context active — call forward_context(...) or set_forward_context(...) " + "before reading get_forward_context()." + ) + return _current + + +def get_attn_backend() -> AttentionBackend: + return get_forward_context().attn_backend + + +def get_token_to_kv_pool() -> KVCache: + return get_attn_backend().token_to_kv_pool + + +def get_req_to_token_pool() -> ReqToTokenPool: + return get_attn_backend().req_to_token_pool + + +@contextmanager +def forward_context(ctx: ForwardContext): + prev = set_forward_context(ctx) + try: + yield + finally: + set_forward_context(prev) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index b88d07d9959d..06f792551c86 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -125,11 +125,13 @@ set_is_extend_in_batch, ) from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.layers.moe.hash_topk import HashTopK from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.pooler import EmbeddingPoolerOutput from sglang.srt.layers.quantization.fp8_kernel import fp8_dtype from sglang.srt.layers.sampler import create_sampler from sglang.srt.layers.torchao_utils import apply_torchao_config_to_model +from sglang.srt.layers.utils.cp_utils import is_mla_prefill_cp_enabled from sglang.srt.lora.lora_manager import LoRAManager from sglang.srt.lora.lora_registry import LoRARef from sglang.srt.managers.schedule_batch import sanity_check_mm_pad_shift_value @@ -150,6 +152,11 @@ ForwardMode, PPProxyTensors, ) +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + forward_context, + has_forward_context, +) from sglang.srt.model_executor.hook_manager import register_forward_hooks from sglang.srt.model_executor.model_runner_kv_cache_mixin import ( ModelRunnerKVCacheMixin, @@ -208,6 +215,7 @@ set_cuda_arch, slow_rank_detector, ) +from sglang.srt.utils.common import ceil_align, require_mlp_sync from sglang.srt.utils.network import NetworkAddress, get_local_ip_auto from sglang.srt.utils.nvtx_pytorch_hooks import PytHooks from sglang.srt.utils.offloader import ( @@ -765,9 +773,6 @@ def initialize(self, pre_model_load_memory: float): if self.device == "cuda" or self.device == "musa": self.init_cublas() - self.init_attention_backend() - self.kernel_warmup() - # Init hisparse coordinator (must happen before CUDA graph capture) if self.enable_hisparse: from sglang.srt.managers.hisparse_coordinator import HiSparseCoordinator from sglang.srt.mem_cache.sparsity import parse_hisparse_config @@ -789,6 +794,8 @@ def initialize(self, pre_model_load_memory: float): ), host_to_device_ratio=hisparse_cfg.host_to_device_ratio, ) + self.init_attention_backend() + self.kernel_warmup() self._pre_initialize_flashinfer_allreduce_workspace() self.init_device_graphs() elif self.device == "cpu": @@ -1435,7 +1442,7 @@ def _prepare_moe_topk(self): num_prepared = 0 num_routed_experts = None for module in self.model.modules(): - if not isinstance(module, TopK): + if not isinstance(module, (TopK, HashTopK)): continue if ( not module.enable_deepep_waterfill @@ -1462,15 +1469,17 @@ def _prepare_moe_topk(self): num_physical_routed_experts = ( num_routed_experts + self.server_args.ep_num_redundant_experts ) + if isinstance(module, TopK): + routed_scaling_factor = module.topk_config.routed_scaling_factor + else: + routed_scaling_factor = module.routed_scaling_factor module.deepep_waterfill_balancer = balancer_cls( num_routed_experts=num_physical_routed_experts, world_size=self.moe_ep_size, rank=self.moe_ep_rank, layer_id=module.layer_id, routed_scaling_factor=( - module.topk_config.routed_scaling_factor - if module.topk_config.routed_scaling_factor is not None - else 1.0 + routed_scaling_factor if routed_scaling_factor is not None else 1.0 ), ) num_prepared += 1 @@ -2221,8 +2230,6 @@ def configure_kv_cache_dtype(self): f"Unsupported kv_cache_dtype: {self.server_args.kv_cache_dtype}." ) - log_info_on_rank0(logger, f"Using KV cache dtype: {self.kv_cache_dtype}") - def init_cublas(self): """We need to run a small matmul to init cublas. Otherwise, it will raise some errors later.""" dtype = torch.float16 @@ -2463,10 +2470,11 @@ def _dummy_run(self, batch_size: int, run_ctx=None): num_tokens = batch_size * num_tokens_per_bs - if require_gathered_buffer(self.server_args): + # Keep warmup aligned with scheduler MLP-sync padding. + if require_mlp_sync(self.server_args): attn_tp_size = get_attention_tp_size() if attn_tp_size > 1 and num_tokens % attn_tp_size != 0: - num_tokens = num_tokens // attn_tp_size * attn_tp_size + num_tokens = ceil_align(num_tokens, attn_tp_size) batch_size = num_tokens // num_tokens_per_bs seq_len_fill_value = self.attn_backend.get_cuda_graph_seq_len_fill_value() @@ -2651,9 +2659,6 @@ def get_spec_info(): seq_lens_cpu=buffers.seq_lens_cpu, next_token_logits_buffer=buffers.next_token_logits_buffer, orig_seq_lens=buffers.seq_lens, - req_to_token_pool=self.req_to_token_pool, - token_to_kv_pool=self.token_to_kv_pool, - attn_backend=self.attn_backend, out_cache_loc=buffers.out_cache_loc, seq_lens_sum=buffers.seq_lens.sum().item(), encoder_lens=buffers.encoder_lens, @@ -2714,8 +2719,9 @@ def run_once(): torch.get_device_module(self.device).synchronize() self.tp_group.barrier() - with torch.inference_mode(), run_ctx or empty_context(): - run_once() + with forward_context(ForwardContext(attn_backend=self.attn_backend)): + with torch.inference_mode(), run_ctx or empty_context(): + run_once() def maybe_init_ngram_embedding(self): self.use_ngram_embedding = self.model_config.use_ngram_embedding @@ -2861,6 +2867,7 @@ def init_piecewise_cuda_graphs(self, force_for_draft_worker: bool = False): self.attention_layers = [] self.moe_layers = [] self.moe_fusions = [] + self.dsa_indexers = [] for layer in layer_model.layers: attn_layer = None if hasattr(layer, "self_attn"): @@ -2913,6 +2920,11 @@ def init_piecewise_cuda_graphs(self, force_for_draft_worker: bool = False): moe_fusion = layer.mixer self.moe_layers.append(moe_block) self.moe_fusions.append(moe_fusion) + # NSA indexers (None for layers without NSA) + dsa_indexer = None + if hasattr(layer, "self_attn") and hasattr(layer.self_attn, "indexer"): + dsa_indexer = layer.self_attn.indexer + self.dsa_indexers.append(dsa_indexer) if len(self.attention_layers) < self.model_config.num_hidden_layers: # TODO(yuwei): support Non-Standard GQA @@ -2992,6 +3004,7 @@ def forward_decode( pp_proxy_tensors=None, ) -> Union[LogitsProcessorOutput, PPProxyTensors]: # Set extra arguments + pdmux_override = False if not skip_attn_backend_init: if hasattr(self.model, "prepare_forward_batch"): # Prepare model-specific attention metadata before planning, @@ -2999,7 +3012,10 @@ def forward_decode( self.model.prepare_forward_batch(forward_batch) if self.server_args.enable_pdmux: self.decode_attn_backend.init_forward_metadata(forward_batch) - forward_batch.attn_backend = self.decode_attn_backend + # PDmux selects a per-stream backend; publish it to model-layer + # readers via the active ForwardContext so RadixAttention etc. + # dispatch against the right backend for this forward. + pdmux_override = True else: self.attn_backend.init_forward_metadata(forward_batch) # FIXME: add pp_proxy_tensors arg to all models @@ -3013,7 +3029,8 @@ def forward_decode( if self.device_timer else contextlib.nullcontext() ) - with ctx: + + def _do_forward(): return self.model.forward( forward_batch.input_ids, forward_batch.positions, @@ -3021,6 +3038,14 @@ def forward_decode( **kwargs, ) + with ctx: + if pdmux_override: + with forward_context( + ForwardContext(attn_backend=self.decode_attn_backend) + ): + return _do_forward() + return _do_forward() + def forward_extend( self, forward_batch: ForwardBatch, @@ -3092,11 +3117,17 @@ def forward_extend( def forward_idle( self, forward_batch: ForwardBatch, pp_proxy_tensors=None ) -> Union[LogitsProcessorOutput, PPProxyTensors]: - # In DP Attention, IDLE batches are padded (batch_size > 0) for MLP sync. - # in this case, we need to reinit the forward metadata, otherwise the stale - # metadata causes batch_size mismatch in attention kernel(e.g. DSA Indexer). + # In DP Attention, IDLE batches may be padded (batch_size > 0) for MLP + # sync. Reinit metadata for the padded case so attention kernels see + # the right batch_size (e.g. DSA Indexer). For the unpadded case + # (batch_size == 0) explicitly drop any stale forward_metadata left + # over from the previous forward — without this, attention layers + # called from the idle path can re-read a prior batch's req_pool + # indices and trigger SWA mapping use-after-free. if forward_batch.batch_size > 0: self.attn_backend.init_forward_metadata(forward_batch) + else: + self.attn_backend.forward_metadata = None kwargs = {} if self.support_pp: @@ -3229,92 +3260,105 @@ def _forward_raw( reinit_attn_backend: bool = False, split_forward_count: int = 1, ) -> ModelRunnerOutput: - # Check whether can run cuda graph - mode_check = ( - forward_batch.forward_mode.is_cpu_graph - if self.device == "cpu" - else forward_batch.forward_mode.is_cuda_graph - ) - can_run_graph = bool( - mode_check() - and self.graph_runner - and self.graph_runner.can_run(forward_batch) - ) - - # Hisparse coordinator - if ( - forward_batch.forward_mode.is_decode() - and self.hisparse_coordinator is not None - ): - forward_batch.hisparse_coordinator = self.hisparse_coordinator - self.hisparse_coordinator.wait_for_pending_backup() - self.hisparse_coordinator.num_real_reqs.fill_(forward_batch.batch_size) - - # Replay cuda graph if applicable - if can_run_graph: - ret = self.graph_runner.replay( - forward_batch, - skip_attn_backend_init=skip_attn_backend_init, - pp_proxy_tensors=pp_proxy_tensors, + # Honor an outer-published context (spec workers wrap each per-step + # draft forward with the i-th child backend); otherwise publish this + # runner's own attn_backend for the forward. + if has_forward_context(): + ctx_mgr = contextlib.nullcontext() + else: + ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend)) + with ctx_mgr: + mode_check = ( + forward_batch.forward_mode.is_cpu_graph + if self.device == "cpu" + else forward_batch.forward_mode.is_cuda_graph + ) + can_run_graph = bool( + mode_check() + and self.graph_runner + and self.graph_runner.can_run(forward_batch) ) - return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph) - # For MLP sync - if forward_batch.global_num_tokens_cpu is not None: - forward_batch.prepare_mlp_sync_batch(self) - else: - forward_batch.prepare_attn_tp_scatter_input(self) + # Hisparse coordinator — backends now read it from self.model_runner. + if ( + forward_batch.forward_mode.is_decode() + and self.hisparse_coordinator is not None + ): + self.hisparse_coordinator.wait_for_pending_backup() + self.hisparse_coordinator.num_real_reqs.fill_(forward_batch.batch_size) - # Normalize num_token_non_padded to be local to this attention TP rank if needed. - if ( - forward_batch.num_token_non_padded is not None - and forward_batch.global_num_tokens_gpu is not None - and require_gathered_buffer(self.server_args) - and not is_dsa_enable_prefill_cp() - ): - forward_batch.adjust_num_token_non_padded_for_attn_tp( - server_args=self.server_args, - ) + if self.is_hybrid_swa: + self.token_to_kv_pool.invalidate_loc_cache() - if self.is_hybrid_swa: - self.token_to_kv_pool.invalidate_loc_cache() + # Replay cuda graph if applicable + if can_run_graph: + ret = self.graph_runner.replay( + forward_batch, + skip_attn_backend_init=skip_attn_backend_init, + pp_proxy_tensors=pp_proxy_tensors, + ) + return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph) - # Hisparse coordinator - forward_batch.hisparse_coordinator = self.hisparse_coordinator - if self.hisparse_coordinator is not None: - self.hisparse_coordinator.num_real_reqs.fill_(forward_batch.batch_size) + # For MLP sync + if forward_batch.global_num_tokens_cpu is not None: + forward_batch.prepare_mlp_sync_batch(self) + else: + forward_batch.prepare_attn_tp_scatter_input(self) + + # Normalize num_token_non_padded to be local to this attention TP rank if needed. + # The skip is scoped to DSACPLayerCommunicator-style CP (DSA, MLA): those + # flavors already feed a zigzag-split rank-local layout whose token count + # should not be further divided by attn_tp_size. MHA-arch prefill CP + # (Qwen3/Qwen2 MoE) keeps the attn_tp-replicated layout and wants the + # adjustment to run — see docs/design/prefill-cp-mla.md §Phase 5. + if ( + forward_batch.num_token_non_padded is not None + and forward_batch.global_num_tokens_gpu is not None + and require_gathered_buffer(self.server_args) + and not is_dsa_enable_prefill_cp() + and not is_mla_prefill_cp_enabled() + ): + forward_batch.adjust_num_token_non_padded_for_attn_tp( + server_args=self.server_args, + ) - # Forward without cuda graph - if forward_batch.forward_mode.is_decode(): - ret = self.forward_decode( - forward_batch, - skip_attn_backend_init=skip_attn_backend_init, - pp_proxy_tensors=pp_proxy_tensors, - ) - elif forward_batch.forward_mode.is_split_prefill(): - ret = self.forward_split_prefill( - forward_batch, - reinit_attn_backend=reinit_attn_backend, - forward_count=split_forward_count, - ) - elif forward_batch.forward_mode.is_extend(include_draft_extend_v2=True): - ret, can_run_graph = self.forward_extend( - forward_batch, - skip_attn_backend_init=skip_attn_backend_init, - pp_proxy_tensors=pp_proxy_tensors, - ) - elif forward_batch.forward_mode.is_idle(): - ret = self.forward_idle(forward_batch, pp_proxy_tensors=pp_proxy_tensors) - else: - raise ValueError(f"Invalid forward mode: {forward_batch.forward_mode}") + # Hisparse coordinator — backends now read it from self.model_runner. + if self.hisparse_coordinator is not None: + self.hisparse_coordinator.num_real_reqs.fill_(forward_batch.batch_size) - if ( - forward_batch.global_num_tokens_cpu is not None - and self.pp_group.is_last_rank - ): - forward_batch.post_forward_mlp_sync_batch(ret) + # Forward without cuda graph + if forward_batch.forward_mode.is_decode(): + ret = self.forward_decode( + forward_batch, + skip_attn_backend_init=skip_attn_backend_init, + pp_proxy_tensors=pp_proxy_tensors, + ) + elif forward_batch.forward_mode.is_split_prefill(): + ret = self.forward_split_prefill( + forward_batch, + reinit_attn_backend=reinit_attn_backend, + forward_count=split_forward_count, + ) + elif forward_batch.forward_mode.is_extend(include_draft_extend_v2=True): + ret, can_run_graph = self.forward_extend( + forward_batch, + skip_attn_backend_init=skip_attn_backend_init, + pp_proxy_tensors=pp_proxy_tensors, + ) + elif forward_batch.forward_mode.is_idle(): + ret = self.forward_idle( + forward_batch, pp_proxy_tensors=pp_proxy_tensors + ) + else: + raise ValueError(f"Invalid forward mode: {forward_batch.forward_mode}") - return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph) + if ( + forward_batch.global_num_tokens_cpu is not None + and self.pp_group.is_last_rank + ): + forward_batch.post_forward_mlp_sync_batch(ret) + + return ModelRunnerOutput(logits_output=ret, can_run_graph=can_run_graph) def _preprocess_logits( self, logits_output: LogitsProcessorOutput, sampling_info: SamplingBatchInfo diff --git a/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py b/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py index 39a516a45992..877cebf2de10 100644 --- a/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py @@ -58,6 +58,7 @@ ForwardMode, PPProxyTensors, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.input_buffers import ForwardInputBuffers from sglang.srt.utils import ( get_available_gpu_memory, @@ -297,6 +298,7 @@ def __init__(self, model_runner: ModelRunner): self.attention_layers = self.model_runner.attention_layers self.moe_layers = self.model_runner.moe_layers self.moe_fusions = self.model_runner.moe_fusions + self.dsa_indexers = getattr(self.model_runner, "dsa_indexers", None) if get_global_graph_memory_pool() is None: set_global_graph_memory_pool(self.device_module.graph_pool_handle()) @@ -387,9 +389,6 @@ def warmup_compile(self, num_tokens: int): next_token_logits_buffer=None, orig_seq_lens=torch.tensor([num_tokens], device=self.device), seq_lens_cpu=torch.tensor([num_tokens], device="cpu"), - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, - attn_backend=self.model_runner.attn_backend, out_cache_loc=out_cache_loc, seq_lens_sum=num_tokens, mamba_track_indices=mamba_track_indices, @@ -425,18 +424,22 @@ def warmup_compile(self, num_tokens: int): forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = None set_dp_buffer_len(None, num_tokens, forward_batch.dp_padding_mode.is_max_len()) set_is_extend_in_batch(False) - with set_forward_context( - forward_batch, - self.attention_layers, - self.quant_config, - self.moe_layers, - self.moe_fusions, + with forward_context( + ForwardContext(attn_backend=self.model_runner.attn_backend) ): - _ = self.model_runner.model.forward( - forward_batch.input_ids, - forward_batch.positions, + with set_forward_context( forward_batch, - ) + self.attention_layers, + self.quant_config, + self.moe_layers, + self.moe_fusions, + dsa_indexers=self.dsa_indexers, + ): + _ = self.model_runner.model.forward( + forward_batch.input_ids, + forward_batch.positions, + forward_batch, + ) def _cache_loc_dtype(self): return torch.int64 if not is_npu() else torch.int32 @@ -554,9 +557,6 @@ def capture_one_batch_size(self, num_tokens: int): next_token_logits_buffer=None, orig_seq_lens=torch.tensor([num_tokens], device=self.device), seq_lens_cpu=torch.tensor([num_tokens], device="cpu"), - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, - attn_backend=self.model_runner.attn_backend, out_cache_loc=out_cache_loc, seq_lens_sum=num_tokens, mamba_track_indices=mamba_track_indices, @@ -586,52 +586,60 @@ def capture_one_batch_size(self, num_tokens: int): lora_ids=None, return_pooled_hidden_states=self.capture_return_pooled_hidden_states, ) + # Setup hooks below read get_attn_backend() and must run inside the + # same ForwardContext as the warmup/capture forward. + with forward_context( + ForwardContext(attn_backend=self.model_runner.attn_backend) + ): self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens) - if lora_ids is not None: - self.model_runner.lora_manager.prepare_lora_batch(forward_batch) + if lora_ids is not None: + self.model_runner.lora_manager.prepare_lora_batch(forward_batch) - self.model_runner.attn_backend.init_forward_metadata(forward_batch) + self.model_runner.attn_backend.init_forward_metadata(forward_batch) - # Run and capture - def run_once(): - # Invalidate SWA loc cache — same fix as in cuda_graph_runner.run_once. - if self.model_runner.is_hybrid_swa: - self.model_runner.token_to_kv_pool.invalidate_loc_cache() - - # Clean intermediate result cache for DP attention - forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = None - set_dp_buffer_len( - global_dp_buffer_len, - num_tokens, - forward_batch.dp_padding_mode.is_max_len(), - ) - # FIXME: the implementation is hacky. `is_extend_in_batch`` is for determining the deepep mode. - # It is True in this context but we need to set it to use low latency deepep mode. - set_is_extend_in_batch(False) + # Run and capture + def run_once(): + # Invalidate SWA loc cache — same fix as in cuda_graph_runner.run_once. + if self.model_runner.is_hybrid_swa: + self.model_runner.token_to_kv_pool.invalidate_loc_cache() - kwargs = {} - with set_forward_context( - forward_batch, - self.attention_layers, - self.quant_config, - self.moe_layers, - self.moe_fusions, - ): - self.model_runner.model.forward( - forward_batch.input_ids, - forward_batch.positions, - forward_batch, - **kwargs, + # Clean intermediate result cache for DP attention + forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = ( + None + ) + set_dp_buffer_len( + global_dp_buffer_len, + num_tokens, + forward_batch.dp_padding_mode.is_max_len(), ) - return + # FIXME: the implementation is hacky. `is_extend_in_batch`` is for determining the deepep mode. + # It is True in this context but we need to set it to use low latency deepep mode. + set_is_extend_in_batch(False) - # run twice for warmup at the first time and cuda graph capture at the second time - # detail lies in sglang/python/sglang/srt/compilation/cuda_piecewise_backend.py - for _ in range(2): - self.device_module.synchronize() - self.model_runner.tp_group.barrier() - run_once() + kwargs = {} + with set_forward_context( + forward_batch, + self.attention_layers, + self.quant_config, + self.moe_layers, + self.moe_fusions, + dsa_indexers=self.dsa_indexers, + ): + self.model_runner.model.forward( + forward_batch.input_ids, + forward_batch.positions, + forward_batch, + **kwargs, + ) + return + + # run twice for warmup at the first time and cuda graph capture at the second time + # detail lies in sglang/python/sglang/srt/compilation/cuda_piecewise_backend.py + for _ in range(2): + self.device_module.synchronize() + self.model_runner.tp_group.barrier() + run_once() return @@ -733,9 +741,6 @@ def replay_prepare( next_token_logits_buffer=next_token_logits_buffer, orig_seq_lens=forward_batch.orig_seq_lens, seq_lens_cpu=forward_batch.seq_lens_cpu, - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, - attn_backend=self.model_runner.attn_backend, out_cache_loc=out_cache_loc, seq_lens_sum=forward_batch.seq_lens_sum, mamba_track_indices=mamba_track_indices, @@ -793,6 +798,7 @@ def replay( self.quant_config, self.moe_layers, self.moe_fusions, + dsa_indexers=self.dsa_indexers, ): # Due to the dispatch kernel for MLA model, we init the metadata with original forward_batch self.model_runner.attn_backend.init_forward_metadata(forward_batch) diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index 84c57c1c7aae..3373f2512293 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -685,7 +685,13 @@ def maybe_add_mtp_safetensors( getattr(hf_config, "num_nextn_predict_layers", 0), ) if not ( - arch in ["Glm4MoeForCausalLM", "Glm4MoeForCausalLMNextN"] + arch + in [ + "Glm4MoeForCausalLM", + "Glm4MoeForCausalLMNextN", + "Glm4MoeLiteForCausalLM", + "Glm4MoeLiteForCausalLMNextN", + ] and num_nextn_layers > 0 ): return hf_weights_files diff --git a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py index de8c6b322afb..6dcf9bc79ea9 100644 --- a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py +++ b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py @@ -1,5 +1,7 @@ from sglang.srt.compilation.piecewise_context_manager import is_in_piecewise_cuda_graph from sglang.srt.layers.attention.tbo_backend import TboAttnBackend +from sglang.srt.layers.utils.cp_utils import mla_use_prefill_cp +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.models.deepseek_common.attention_forward_methods.forward_methods import ( AttnForwardMethod, ) @@ -73,6 +75,12 @@ def _handle_attention_backend(attn, forward_batch, backend_name): if is_in_piecewise_cuda_graph(): return AttnForwardMethod.MLA + # MLA prefill CP forces absorbed MLA regardless of prefix length: the + # CP path gathers latent KV via rebuild_cp_kv_cache and feeds the + # backend's absorbed-MLA kernel. + if mla_use_prefill_cp(forward_batch): + return _dispatch_mla_subtype(attn, forward_batch) + sum_extend_prefix_lens = _get_sum_extend_prefix_lens(forward_batch) disable_ragged = ( backend_name in ["flashinfer", "flashmla"] @@ -153,7 +161,7 @@ def handle_attention_dsa(attn, forward_batch): in init_forward_metadata. Read the decision from backend.use_mha. """ - backend = forward_batch.attn_backend + backend = get_attn_backend() if isinstance(backend, TboAttnBackend): # if enable tbo, get primary backend backend = backend.primary if hasattr(backend, "use_mha") and backend.use_mha: diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py index 75019ba11d52..dbcb3ee0fa84 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py @@ -10,6 +10,10 @@ from sglang.srt.layers.attention.utils import concat_and_cast_mha_k_triton from sglang.srt.layers.communicator import get_attn_tp_context from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_token_to_kv_pool, +) from sglang.srt.models.deepseek_common.utils import ( _is_cuda, _is_hip, @@ -28,7 +32,11 @@ from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA if _is_cuda: - from sgl_kernel import concat_mla_k, merge_state_v2 + from sgl_kernel import merge_state_v2 + + from sglang.jit_kernel.concat_mla import concat_mla_k +elif _is_musa: + from sgl_kernel import concat_mla_k if _use_aiter_gfx95: from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant @@ -38,7 +46,7 @@ def _resolve_attn_backend(forward_batch: ForwardBatch): - backend = forward_batch.attn_backend + backend = get_attn_backend() if isinstance(backend, TboAttnBackend): backend = backend.primary return backend @@ -334,8 +342,8 @@ def forward_normal_chunked_kv_core( # Only initialize the info once if has_extend_prefix and forward_batch.num_prefix_chunks is None: forward_batch.prepare_chunked_prefix_cache_info(q.device) - if hasattr(forward_batch.attn_backend, "init_mha_chunk_metadata"): - forward_batch.attn_backend.init_mha_chunk_metadata(forward_batch) + if hasattr(get_attn_backend(), "init_mha_chunk_metadata"): + get_attn_backend().init_mha_chunk_metadata(forward_batch) forward_batch.mha_return_lse = has_extend_prefix # Do mha for extended part without prefix @@ -380,8 +388,8 @@ def forward_normal_one_shot_core( # Only initialize the info once if has_extend_prefix and forward_batch.num_prefix_chunks is None: forward_batch.num_prefix_chunks = 0 - if hasattr(forward_batch.attn_backend, "init_mha_chunk_metadata"): - forward_batch.attn_backend.init_mha_chunk_metadata(forward_batch) + if hasattr(get_attn_backend(), "init_mha_chunk_metadata"): + get_attn_backend().init_mha_chunk_metadata(forward_batch) forward_batch.mha_return_lse = False # Do mha for extended part without prefix forward_batch.set_attn_attend_prefix_cache(False) @@ -449,12 +457,12 @@ def _set_mla_kv_buffer( ): if _is_cuda or _use_aiter_gfx95: # Save latent cache - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + get_token_to_kv_pool().set_mla_kv_buffer( self.attn_mha, forward_batch.out_cache_loc, kv_a.unsqueeze(1), k_pe ) elif _is_npu: # To reduce a time-costing split operation - forward_batch.token_to_kv_pool.set_kv_buffer( + get_token_to_kv_pool().set_kv_buffer( self.attn_mha, forward_batch.out_cache_loc, kv_a.unsqueeze(1), k_pe ) else: @@ -462,7 +470,7 @@ def _set_mla_kv_buffer( latent_cache[:, :, self.kv_lora_rank :] = k_pe.clone() # Save latent cache - forward_batch.token_to_kv_pool.set_kv_buffer( + get_token_to_kv_pool().set_kv_buffer( self.attn_mha, forward_batch.out_cache_loc, latent_cache, None ) @@ -473,12 +481,12 @@ def _get_mla_kv_buffer( forward_batch: ForwardBatch, ): if _is_cuda or _use_aiter_gfx95: - kv_a, k_pe = forward_batch.token_to_kv_pool.get_mla_kv_buffer( + kv_a, k_pe = get_token_to_kv_pool().get_mla_kv_buffer( self.attn_mha, kv_indices, dst_dtype ) kv_a = kv_a.squeeze(1) else: - latent_cache_buf = forward_batch.token_to_kv_pool.get_key_buffer( + latent_cache_buf = get_token_to_kv_pool().get_key_buffer( self.attn_mha.layer_id ) latent_cache = latent_cache_buf[kv_indices].contiguous().to(dst_dtype) @@ -498,7 +506,7 @@ def _get_mla_kv_buffer_from_fp8_for_dsa( Returns: (kv_a, k_pe) both in BF16 """ - backend = forward_batch.attn_backend + backend = get_attn_backend() if isinstance(backend, TboAttnBackend): # if enable tbo, get primary backend backend = backend.primary kv_indices = backend.forward_metadata.page_table_1_flattened @@ -506,9 +514,7 @@ def _get_mla_kv_buffer_from_fp8_for_dsa( kv_indices is not None ), "page_table_1_flattened should have been generated for FP8 MHA path" - kv_cache_fp8 = forward_batch.token_to_kv_pool.get_key_buffer( - self.attn_mha.layer_id - ) + kv_cache_fp8 = get_token_to_kv_pool().get_key_buffer(self.attn_mha.layer_id) kv_latent_bf16 = dequantize_k_cache_paged(kv_cache_fp8, kv_indices) @@ -544,7 +550,7 @@ def _concat_and_cast_mha_k( self.current_attention_backend == "fa3" and self.kv_cache_dtype != "auto" ): - attn_dtype = forward_batch.token_to_kv_pool.dtype + attn_dtype = get_token_to_kv_pool().dtype else: attn_dtype = k_nope.dtype k = k_nope.new_empty(*k_shape, dtype=attn_dtype) diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index e0ad07511b85..364f065b4d06 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -13,6 +13,7 @@ per_tensor_quant_mla_fp8, per_token_group_quant_mla_deep_gemm_masked_fp8, ) +from sglang.srt.layers.utils.cp_utils import mla_use_prefill_cp from sglang.srt.lora.deepseek_mla_correction import ( apply_q_correction as apply_kv_b_lora_q_correction, ) @@ -23,6 +24,10 @@ is_kv_b_lora_active, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_token_to_kv_pool, +) from sglang.srt.models.deepseek_common.utils import ( FORWARD_ABSORB_CORE_ATTENTION_BACKENDS, _is_cpu, @@ -376,7 +381,7 @@ def forward_absorb_prepare( ): q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) - if dsa_use_prefill_cp(forward_batch): + if dsa_use_prefill_cp(forward_batch) or mla_use_prefill_cp(forward_batch): # support allgather+rerrange k_nope, k_pe = self.rebuild_cp_kv_cache( latent_cache, forward_batch, k_nope, k_pe @@ -420,9 +425,7 @@ def forward_absorb_core( q_pe, k_nope, k_pe, - forward_batch.token_to_kv_pool.get_key_buffer( - self.attn_mqa.layer_id - ), + get_token_to_kv_pool().get_key_buffer(self.attn_mqa.layer_id), forward_batch.out_cache_loc, positions, cos, @@ -516,9 +519,7 @@ def forward_absorb_core( q_pe, k_nope, k_pe, - forward_batch.token_to_kv_pool.get_key_buffer( - self.attn_mqa.layer_id - ), + get_token_to_kv_pool().get_key_buffer(self.attn_mqa.layer_id), forward_batch.out_cache_loc, positions, cos, @@ -694,7 +695,7 @@ def _fuse_rope_for_trtllm_mla( return ( get_global_server_args().dsa_decode_backend == "trtllm" or get_global_server_args().dsa_prefill_backend == "trtllm" - ) and forward_batch.attn_backend.kv_cache_dtype == torch.float8_e4m3fn + ) and get_attn_backend().kv_cache_dtype == torch.float8_e4m3fn return ( self.current_attention_backend in ("trtllm_mla", "tokenspeed_mla") @@ -702,7 +703,7 @@ def _fuse_rope_for_trtllm_mla( forward_batch.forward_mode.is_decode_or_idle() or forward_batch.forward_mode.is_target_verify() ) - and forward_batch.attn_backend.data_type == torch.float8_e4m3fn + and get_attn_backend().data_type == torch.float8_e4m3fn ) def _skip_rope_for_dsa_tilelang_fused(self: DeepseekV2AttentionMLA) -> bool: diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_rocm.py index 8868897af2b8..65545069eb81 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_rocm.py @@ -7,6 +7,10 @@ from sglang.srt.layers.quantization.fp8_kernel import per_tensor_quant_mla_fp8 from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_token_to_kv_pool, +) from sglang.srt.models.deepseek_common.utils import ( _is_cuda, _is_hip, @@ -108,10 +112,10 @@ def forward_absorb_fused_mla_rope_prepare( device=q.device, ) attn_logits, _, kv_indptr, kv_indices, _, _, _ = ( - forward_batch.attn_backend.forward_metadata + get_attn_backend().forward_metadata ) cos_sin_cache = self.rotary_emb.cos_sin_cache - num_kv_split = forward_batch.attn_backend.num_kv_splits + num_kv_split = get_attn_backend().num_kv_splits sm_scale = self.attn_mqa.scaling if attn_logits is None: attn_logits = torch.empty( @@ -126,12 +130,10 @@ def forward_absorb_fused_mla_rope_prepare( ) # save current latent cache. - forward_batch.token_to_kv_pool.set_kv_buffer( + get_token_to_kv_pool().set_kv_buffer( self.attn_mqa, forward_batch.out_cache_loc, k_input, None ) - key_cache_buf = forward_batch.token_to_kv_pool.get_key_buffer( - self.attn_mqa.layer_id - ) + key_cache_buf = get_token_to_kv_pool().get_key_buffer(self.attn_mqa.layer_id) val_cache_buf = key_cache_buf[..., : self.kv_lora_rank] return ( @@ -194,7 +196,7 @@ def forward_absorb_fused_mla_rope_core( if enable_rope_fusion: k_input[..., self.kv_lora_rank :] = k_pe_output - forward_batch.token_to_kv_pool.set_kv_buffer( + get_token_to_kv_pool().set_kv_buffer( self.attn_mqa, forward_batch.out_cache_loc, k_input, None ) diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 59d26282e6c8..1e16972521b5 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -43,9 +43,12 @@ from sglang.srt.layers.quantization import Fp8Config from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils.cp_utils import ( + can_cp_split, cp_all_gather_rerange_output, cp_split_and_rebuild_data, cp_split_and_rebuild_position, + is_mla_prefill_cp_enabled, + mla_use_prefill_cp, prepare_context_parallel_metadata, ) from sglang.srt.layers.vocab_parallel_embedding import ( @@ -136,6 +139,14 @@ def __init__( layer_name = "layers." + str(config.num_hidden_layers) self.quant_config = quant_config + self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() + self.mla_enable_prefill_cp = ( + is_mla_prefill_cp_enabled() and not is_deepseek_dsa(config) + ) + if self.dsa_enable_prefill_cp or self.mla_enable_prefill_cp: + self.cp_size = get_attention_cp_size() + else: + self.cp_size = None self.decoder = DeepseekV2DecoderLayer( config, 0, @@ -144,15 +155,12 @@ def __init__( is_nextn=True, prefix=add_prefix(layer_name, prefix), alt_stream=self.alt_stream, + dsa_enable_prefill_cp=self.dsa_enable_prefill_cp, + mla_enable_prefill_cp=self.mla_enable_prefill_cp, ) self.shared_head = nn.Module() self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() - if self.dsa_enable_prefill_cp: - self.cp_size = get_attention_cp_size() - else: - self.cp_size = None def forward( self, @@ -193,7 +201,9 @@ def forward( else: hidden_states = self.eh_proj(eh_input) - if dsa_use_prefill_cp(forward_batch, self.dsa_enable_prefill_cp): + if dsa_use_prefill_cp( + forward_batch, self.dsa_enable_prefill_cp + ) or mla_use_prefill_cp(forward_batch, self.mla_enable_prefill_cp): hidden_states = cp_split_and_rebuild_data(forward_batch, hidden_states) positions = cp_split_and_rebuild_position(forward_batch, positions) residual = None @@ -212,7 +222,9 @@ def forward( else: hidden_states = self.shared_head.norm(hidden_states) - if dsa_use_prefill_cp(forward_batch, self.dsa_enable_prefill_cp): + if dsa_use_prefill_cp( + forward_batch, self.dsa_enable_prefill_cp + ) or mla_use_prefill_cp(forward_batch, self.mla_enable_prefill_cp): # allgather + rerrange hidden_states = cp_all_gather_rerange_output( hidden_states, @@ -250,7 +262,8 @@ def __init__( self.determine_num_fused_shared_experts("DeepseekV3ForCausalLMNextN") self.use_dsa = is_deepseek_dsa(config) self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() - if self.dsa_enable_prefill_cp: + self.mla_enable_prefill_cp = is_mla_prefill_cp_enabled() and not self.use_dsa + if self.dsa_enable_prefill_cp or self.mla_enable_prefill_cp: self.cp_rank = get_attention_cp_rank() self.cp_size = get_attention_cp_size() else: @@ -298,6 +311,16 @@ def forward( self.cp_rank, self.cp_size, forward_batch.seq_lens_cpu.tolist(), + extend_lens=forward_batch.extend_seq_lens_cpu, + ) + elif self.mla_enable_prefill_cp: + if can_cp_split(len(input_ids), self.cp_size, forward_batch): + forward_batch.attn_cp_metadata = prepare_context_parallel_metadata( + len(input_ids), + self.cp_rank, + self.cp_size, + forward_batch.seq_lens_cpu.tolist(), + extend_lens=forward_batch.extend_seq_lens_cpu, ) hidden_states = self.model(input_ids, positions, forward_batch) return self.logits_processor( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 98944726fe32..8b254348a44f 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -123,9 +123,12 @@ from sglang.srt.layers.rotary_embedding import get_rope_wrapper from sglang.srt.layers.utils import PPMissingLayer from sglang.srt.layers.utils.cp_utils import ( + can_cp_split, cp_all_gather_rerange_output, cp_split_and_rebuild_data, cp_split_and_rebuild_position, + is_prefill_context_parallel_enabled, + mla_use_prefill_cp, prepare_context_parallel_metadata, ) from sglang.srt.layers.vocab_parallel_embedding import ( @@ -255,6 +258,11 @@ def __init__( "Only silu is supported for now." ) self.act_fn = SiluAndMul() + self.use_fused_clamp_act_mul = ( + _is_hip and envs.SGLANG_OPT_USE_FUSED_CLAMP_ACT_MUL.get() + ) + self._fused_clamp_fp8_checked = False + self._fused_clamp_use_fp8 = False def forward( self, @@ -316,8 +324,41 @@ def forward( down_output, ) return down_output + + if self.use_fused_clamp_act_mul and self.swiglu_limit is not None: + from aiter.ops.triton.fusions.fused_clamp_act_mul import ( + fused_clamp_act_mul, + ) + + if not self._fused_clamp_fp8_checked: + from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod + + qm = getattr(self.down_proj, "quant_method", None) + self._fused_clamp_use_fp8 = ( + isinstance(qm, Fp8LinearMethod) and qm.block_quant + ) + self._fused_clamp_fp8_checked = True + + if self._fused_clamp_use_fp8: + from aiter import dtypes + + x_fp8, x_scale = fused_clamp_act_mul( + gate_up, + swiglu_limit=self.swiglu_limit, + activation="silu", + dtype_quant=dtypes.fp8, + transpose_scale=False, + ) + x = (x_fp8, x_scale) + else: + x = fused_clamp_act_mul( + gate_up, + swiglu_limit=self.swiglu_limit, + activation="silu", + ) + # Fallback: fused silu+clamp kernel (still faster than unfused) - if self.swiglu_limit is not None: + elif self.swiglu_limit is not None: M, N = gate_up.shape x = gate_up.new_empty((M, N // 2)) silu_and_mul_clamp(gate_up, x, float(self.swiglu_limit)) @@ -339,6 +380,8 @@ def __init__( is_nextn: bool = False, is_hash_moe: bool = False, is_deepseek_v4: bool = False, + dsa_enable_prefill_cp: bool = False, + mla_enable_prefill_cp: bool = False, ): super().__init__() self.is_nextn = is_nextn @@ -368,7 +411,9 @@ def __init__( self.e_score_correction_bias = None if _is_cpu and _is_cpu_amx_available: self.quant_method = PackWeightMethod(weight_names=["weight"]) - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() + self.use_dsa = is_deepseek_dsa(config) + self.dsa_enable_prefill_cp = dsa_enable_prefill_cp + self.mla_enable_prefill_cp = mla_enable_prefill_cp def forward( self, @@ -390,7 +435,10 @@ def forward( if ( not self.is_deepseek_v4 and forward_batch is not None - and dsa_use_prefill_cp(forward_batch) + and ( + dsa_use_prefill_cp(forward_batch, self.dsa_enable_prefill_cp) + or mla_use_prefill_cp(forward_batch, self.mla_enable_prefill_cp) + ) ): logits = F.linear(hidden_states, self.weight, None) else: @@ -403,6 +451,7 @@ def forward( and _device_sm >= 90 ): if _device_sm in [100, 103] and self.weight.shape[0] == 256: + # TODO: will check the dtype to be bf16 # router gemm output float32 logits = torch.empty( hidden_states.shape[0], @@ -441,6 +490,8 @@ def __init__( alt_stream: Optional[torch.cuda.Stream] = None, is_nextn: bool = False, is_deepseek_v4: bool = False, + dsa_enable_prefill_cp: bool = False, + mla_enable_prefill_cp: bool = False, ): super().__init__() self.tp_size = get_tensor_model_parallel_world_size() @@ -502,6 +553,8 @@ def __init__( is_nextn=is_nextn, is_hash_moe=self.is_hash, is_deepseek_v4=is_deepseek_v4, + dsa_enable_prefill_cp=dsa_enable_prefill_cp, + mla_enable_prefill_cp=mla_enable_prefill_cp, ) # scaling factor for fused shared experts on AMD-platform. @@ -787,8 +840,11 @@ def forward_normal_dual_stream( **topk_kwargs, ) final_hidden_states = self.experts(hidden_states, topk_output) - if not (_is_cuda or _is_musa) or isinstance( - self.experts.quant_method, KTEPWrapperMethod + if ( + not _is_cuda + and not _is_musa + and not _use_aiter + or isinstance(self.experts.quant_method, KTEPWrapperMethod) ): final_hidden_states *= self.routed_scaling_factor @@ -1339,6 +1395,8 @@ def __init__( alt_stream: Optional[torch.cuda.Stream] = None, skip_rope: bool = False, is_nextn: bool = False, + dsa_enable_prefill_cp: bool = False, + mla_enable_prefill_cp: bool = False, ) -> None: super().__init__() self.layer_id = layer_id @@ -1353,11 +1411,14 @@ def __init__( attn_tp_rank = get_attention_tp_rank() attn_tp_size = get_attention_tp_size() self.use_dsa = is_deepseek_dsa(config) - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() + self.dsa_enable_prefill_cp = dsa_enable_prefill_cp + self.mla_enable_prefill_cp = mla_enable_prefill_cp if self.dsa_enable_prefill_cp: assert self.use_dsa, "CP currently only supports deepseek v3.2 model" - # cp reuse the attn_tp comm group but need to duplicate the weights - if self.dsa_enable_prefill_cp and self.use_dsa: + # cp reuses the attn_tp comm group but needs to duplicate the weights; + # store cp_size whenever either CP flavor is active so rebuild_cp_kv_cache + # and the FA3 MLA wrapper can reach it on the dense MLA path too. + if self.dsa_enable_prefill_cp or self.mla_enable_prefill_cp: self.cp_size = get_attention_cp_size() self.num_heads = num_heads assert num_heads % attn_tp_size == 0 @@ -1792,6 +1853,8 @@ def __init__( is_nextn: bool = False, prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, + dsa_enable_prefill_cp: bool = False, + mla_enable_prefill_cp: bool = False, ) -> None: super().__init__() self.hidden_size = config.hidden_size @@ -1808,7 +1871,8 @@ def __init__( self.speculative_algorithm = SpeculativeAlgorithm.from_string( get_global_server_args().speculative_algorithm ) - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() + self.dsa_enable_prefill_cp = dsa_enable_prefill_cp + self.mla_enable_prefill_cp = mla_enable_prefill_cp self.layer_id = layer_id self.is_nextn = is_nextn self.self_attn = DeepseekV2AttentionMLA( @@ -1831,6 +1895,8 @@ def __init__( prefix=add_prefix("self_attn", prefix), alt_stream=alt_stream, is_nextn=is_nextn, + dsa_enable_prefill_cp=dsa_enable_prefill_cp, + mla_enable_prefill_cp=mla_enable_prefill_cp, ) if not hasattr(config, "q_lora_rank") and envs.SGLANG_USE_AG_AFTER_QLORA.get(): raise ValueError( @@ -1857,6 +1923,8 @@ def __init__( layer_id=self.layer_id, alt_stream=alt_stream, is_nextn=is_nextn, + dsa_enable_prefill_cp=dsa_enable_prefill_cp, + mla_enable_prefill_cp=mla_enable_prefill_cp, ) else: if enable_moe_dense_fully_dp(): @@ -1881,7 +1949,10 @@ def __init__( self._gfx95_quant_format = self._detect_gfx95_quant_format() - if self.dsa_enable_prefill_cp: + if self.dsa_enable_prefill_cp or self.mla_enable_prefill_cp: + # DSACPLayerCommunicator is flavor-agnostic; its internal gates + # read both dsa_use_prefill_cp and mla_use_prefill_cp. The rename + # to CPLayerCommunicator is deferred to a cleanup PR. self.layer_communicator = DSACPLayerCommunicator( layer_scatter_modes=self.layer_scatter_modes, input_layernorm=self.input_layernorm, @@ -1980,6 +2051,7 @@ def forward( if ( isinstance(self.mlp, DeepseekV2MoE) and not self.mlp.experts.moe_runner_config.inplace + and not torch.compiler.is_compiling() ): from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx @@ -1996,7 +2068,10 @@ def forward( gemm_output_zero_allocator, ) - if not self.dsa_enable_prefill_cp and should_allreduce_fusion: + if ( + not (self.dsa_enable_prefill_cp or self.mla_enable_prefill_cp) + and should_allreduce_fusion + ): hidden_states._sglang_needs_allreduce_fusion = True if not should_allreduce_fusion: @@ -2094,7 +2169,10 @@ def __init__( self.first_k_dense_replace = config.first_k_dense_replace self.pp_group = get_pp_group() self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() - if self.dsa_enable_prefill_cp: + self.mla_enable_prefill_cp = ( + is_prefill_context_parallel_enabled() and not is_deepseek_dsa(config) + ) + if self.dsa_enable_prefill_cp or self.mla_enable_prefill_cp: self.cp_size = get_attention_cp_size() else: self.cp_size = None @@ -2127,6 +2205,8 @@ def __init__( quant_config=quant_config, prefix=prefix, alt_stream=self.alt_stream, + dsa_enable_prefill_cp=self.dsa_enable_prefill_cp, + mla_enable_prefill_cp=self.mla_enable_prefill_cp, ), pp_rank=self.pp_group.rank_in_group, pp_size=self.pp_group.world_size, @@ -2253,7 +2333,9 @@ def forward( else None ) - if dsa_use_prefill_cp(forward_batch): + if dsa_use_prefill_cp( + forward_batch, self.dsa_enable_prefill_cp + ) or mla_use_prefill_cp(forward_batch, self.mla_enable_prefill_cp): if self.pp_group.is_first_rank: hidden_states = cp_split_and_rebuild_data(forward_batch, hidden_states) positions = cp_split_and_rebuild_position(forward_batch, positions) @@ -2338,7 +2420,10 @@ def forward( else: hidden_states, _ = self.norm(hidden_states, residual) - if self.pp_group.is_last_rank and dsa_use_prefill_cp(forward_batch): + if self.pp_group.is_last_rank and ( + dsa_use_prefill_cp(forward_batch, self.dsa_enable_prefill_cp) + or mla_use_prefill_cp(forward_batch, self.mla_enable_prefill_cp) + ): # allgather + rerrange hidden_states = cp_all_gather_rerange_output( hidden_states, @@ -2416,7 +2501,10 @@ def __init__( self.capture_aux_hidden_states = False self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() - if self.dsa_enable_prefill_cp: + self.mla_enable_prefill_cp = ( + is_prefill_context_parallel_enabled() and not is_deepseek_dsa(config) + ) + if self.dsa_enable_prefill_cp or self.mla_enable_prefill_cp: self.cp_rank = get_attention_cp_rank() self.cp_size = get_attention_cp_size() else: @@ -2450,9 +2538,19 @@ def determine_num_fused_shared_experts( # Allow-list of n_routed_experts values that have been validated # for shared-experts fusion under this code path. Currently: # 256 -> DeepSeek-V3 / R1 - # 384 -> Kimi-K2.5 (text_config wraps DeepseekV3ForCausalLM) + # 384 -> Kimi-K2.5, only when the checkpoint is Quark MXFP4 + # (amd/Kimi-K2.5-MXFP4); the standard + # moonshotai/Kimi-K2.5 (compressed-tensors) checkpoint + # stores the shared expert loose and is NOT pre-fused, + # so the fused path silently mis-loads it. or self.config.n_routed_experts not in (256, 384) or self.config.n_shared_experts != 1 + or ( + self.config.n_routed_experts == 384 + and ( + self.quant_config is None or self.quant_config.get_name() != "quark" + ) + ) ): disable_reason = "Config does not support fused shared expert(s)." elif ( @@ -2498,15 +2596,29 @@ def forward( input_embeds: torch.Tensor = None, pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> torch.Tensor: + # Minor fix for multi-modal model: input_ids is None + len_input_ids = ( + input_ids.shape[0] if input_ids is not None else input_embeds.shape[0] + ) if self.dsa_enable_prefill_cp: if can_dsa_cp_split( - len(input_ids), self.cp_size, self.use_dsa, forward_batch + len_input_ids, self.cp_size, self.use_dsa, forward_batch ): forward_batch.attn_cp_metadata = prepare_context_parallel_metadata( - len(input_ids), + len_input_ids, + self.cp_rank, + self.cp_size, + forward_batch.seq_lens_cpu.tolist(), + extend_lens=forward_batch.extend_seq_lens_cpu, + ) + elif self.mla_enable_prefill_cp: + if can_cp_split(len_input_ids, self.cp_size, forward_batch): + forward_batch.attn_cp_metadata = prepare_context_parallel_metadata( + len_input_ids, self.cp_rank, self.cp_size, forward_batch.seq_lens_cpu.tolist(), + extend_lens=forward_batch.extend_seq_lens_cpu, ) with get_attn_tp_context().maybe_input_scattered(forward_batch): diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index fdf6d557ca45..2a89d22f49cc 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3,6 +3,7 @@ import concurrent.futures import logging import time +from contextlib import nullcontext from typing import ( TYPE_CHECKING, Iterable, @@ -33,6 +34,7 @@ get_tp_group, ) from sglang.srt.environ import envs +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.dsa.utils import ( can_dsa_cp_split, @@ -78,6 +80,10 @@ get_is_capture_mode, ) from sglang.srt.model_executor.forward_batch_info import PPProxyTensors +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_token_to_kv_pool, +) from sglang.srt.model_loader.utils import maybe_executor_submit, should_async_load from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.dbrx import ReplicatedLinear @@ -92,6 +98,8 @@ from sglang.srt.utils import ( LazyValue, add_prefix, + get_bool_env_var, + is_gfx95_supported, log_info_on_rank0, make_layers, ) @@ -101,6 +109,29 @@ _FP8_WO_A_GEMM = envs.SGLANG_OPT_FP8_WO_A_GEMM.get() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +_is_gfx95_supported = is_gfx95_supported() + +if _use_aiter: + if _is_gfx95_supported: + from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_group_quant + + +def _fused_rmsnorm_fp8_quant(hidden_states, weight, eps): + x_quant, x_bf16, _, _ = fused_rms_fp8_group_quant( + hidden_states, + weight, + eps, + inp2=None, + inp2_weight=None, + inp2_epsilon=None, + group_size=128, + dtype_quant=torch.float8_e4m3fn, + res1=None, + output_unquantized_inp1=True, + ) + return x_quant, x_bf16 + if TYPE_CHECKING: from sglang.srt.layers.attention.deepseek_v4_backend import ( @@ -245,6 +276,12 @@ def __init__( self.register_buffer("freqs_cis", freqs_cis, persistent=False) self.freqs_cis: torch.Tensor + if _is_hip: + cos_cache = freqs_cis.real.to(torch.bfloat16).unsqueeze(-2).unsqueeze(-2) + sin_cache = freqs_cis.imag.to(torch.bfloat16).unsqueeze(-2).unsqueeze(-2) + self.register_buffer("cos_cache", cos_cache, persistent=False) + self.register_buffer("sin_cache", sin_cache, persistent=False) + if envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get() and alt_streams is not None: self.alt_streams = alt_streams[:3] self.alt_streams_indexer = alt_streams[-2:] @@ -353,6 +390,10 @@ def __init__( prefix=add_prefix("attn_mqa", prefix), ) + self.use_fused_qk_norm_rope = ( + _is_hip and envs.SGLANG_OPT_USE_FUSED_QK_NORM_ROPE.get() + ) + # KV cache write is always fused into the K kernel # (`_compute_kv_to_cache`), so the legacy "overlap store cache" flag # has no effect here -- the fused path is on by default. @@ -398,7 +439,7 @@ def _compute_kv_to_cache( kv = qkv_a[..., self.q_lora_rank :] else: kv, _ = self.wkv(x) - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = get_token_to_kv_pool() if TYPE_CHECKING: assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) token_to_kv_pool.set_swa_key_buffer_radix_fused_norm_rope( @@ -439,6 +480,7 @@ def _forward_prepare_multi_stream( forward_batch: ForwardBatch, attn_backend, q_out: Optional[torch.Tensor] = None, + x_quant=None, ) -> torch.Tensor: assert self.alt_streams is not None assert len(self.alt_streams) >= 3 @@ -452,13 +494,14 @@ def _forward_prepare_multi_stream( stream_compressor.wait_stream(current_stream) stream_indexer.wait_stream(current_stream) + x_linear = x_quant if x_quant is not None else x qkv_a: Optional[torch.Tensor] = None qkv_a_ready: Optional[torch.cuda.Event] = None if self.fuse_wqa_wkv: - qkv_a, _ = self.wqkv_a(x) + qkv_a, _ = self.wqkv_a(x_linear) qkv_a_ready = current_stream.record_event() - q_lora = self._compute_q_a(x, qkv_a=qkv_a) + q_lora = self._compute_q_a(x_linear, qkv_a=qkv_a) q_lora_ready = current_stream.record_event() if self.indexer is not None: @@ -467,6 +510,7 @@ def _forward_prepare_multi_stream( x=x, q_lora=q_lora, forward_batch=forward_batch, + attn_backend=attn_backend, enable_multi_stream=True, q_lora_ready=q_lora_ready, ) @@ -475,7 +519,7 @@ def _forward_prepare_multi_stream( if qkv_a_ready is not None: stream_kv.wait_event(qkv_a_ready) # Fused norm + rope + cache write -- no bf16 KV intermediate. - self._compute_kv_to_cache(x, positions, forward_batch, qkv_a=qkv_a) + self._compute_kv_to_cache(x_linear, positions, forward_batch, qkv_a=qkv_a) del qkv_a @@ -492,6 +536,118 @@ def _forward_prepare_multi_stream( return q + def _forward_prepare_multi_stream_hip( + self, + x: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + attn_backend, + q_out: Optional[torch.Tensor] = None, + x_quant=None, + ) -> torch.Tensor: + """ATOM-style ROCm path: overlap compressors, keep Q/KV on main stream.""" + assert self.alt_streams is not None + assert len(self.alt_streams) >= 1 + + current_stream = torch.cuda.current_stream() + stream_compressor = self.alt_streams[0] + stream_indexer_compressor = ( + self.alt_streams[1] if len(self.alt_streams) > 1 else None + ) + + if self.compressor is not None: + stream_compressor.wait_stream(current_stream) + with torch.cuda.stream(stream_compressor): + attn_backend.forward_core_compressor( + x, forward_batch, self.layer_id, self.compressor + ) + + if self.indexer is not None and stream_indexer_compressor is not None: + stream_indexer_compressor.wait_stream(current_stream) + with torch.cuda.stream(stream_indexer_compressor): + attn_backend.forward_indexer_compressor( + x=x, + forward_batch=forward_batch, + layer_id=self.indexer.layer_id, + compressor=self.indexer.compressor, + ) + + x_linear = x_quant if x_quant is not None else x + if self.fuse_wqa_wkv: + qkv_a, _ = self.wqkv_a(x_linear) + q_lora = qkv_a[..., : self.q_lora_rank] + else: + q_lora, _ = self.wq_a(x_linear) + qkv_a = None + + if self.use_fused_qk_norm_rope: + if _is_gfx95_supported: + q_for_wqb, q_lora = _fused_rmsnorm_fp8_quant( + q_lora, + self.q_norm.weight, + self.q_norm.variance_epsilon, + ) + q, _ = self.wq_b(q_for_wqb) + else: + q_lora = self.q_norm(q_lora) + q, _ = self.wq_b(q_lora) + + kv = ( + qkv_a[..., self.q_lora_rank :] + if qkv_a is not None + else self.wkv(x_linear)[0] + ) + + from sglang.srt.layers.fused_qk_norm_rope_store import ( + fused_qk_norm_rope_swa_store, + ) + + token_to_kv_pool = get_token_to_kv_pool() + swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa( + forward_batch.out_cache_loc + ) + swa_cache = token_to_kv_pool.swa_kv_pool.kv_buffer[self.layer_id] + swa_page_size = token_to_kv_pool.swa_kv_pool.page_size + + q = fused_qk_norm_rope_swa_store( + q=q, + kv=kv, + q_norm_weight=None, + kv_norm_weight=self.kv_norm.weight, + q_rms_eps=self.eps, + kv_rms_eps=self.eps, + rope_head_dim=self.qk_rope_head_dim, + cos_cache=self.cos_cache, + sin_cache=self.sin_cache, + positions=positions, + swa_cache=swa_cache, + swa_loc=swa_loc, + swa_page_size=swa_page_size, + q_out=q_out, + dtype=x.dtype, + ) + else: + q_lora = self.q_norm(q_lora) + q = self._compute_q_b(q_lora, positions, q_out) + self._compute_kv_to_cache(x_linear, positions, forward_batch, qkv_a=qkv_a) + + del qkv_a + + if self.indexer is not None: + current_stream.wait_stream(stream_compressor) + if stream_indexer_compressor is not None: + current_stream.wait_stream(stream_indexer_compressor) + self.indexer( + x=x, + q_lora=q_lora, + forward_batch=forward_batch, + skip_compressor=True, + ) + elif self.compressor is not None: + current_stream.wait_stream(stream_compressor) + + return q + def _forward_prepare( self, x: torch.Tensor, @@ -499,41 +655,110 @@ def _forward_prepare( forward_batch: ForwardBatch, attn_backend, q_out: Optional[torch.Tensor] = None, + x_quant=None, ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + x_linear = x_quant if x_quant is not None else x if self.fuse_wqa_wkv: - qkv_a, _ = self.wqkv_a(x) + qkv_a, _ = self.wqkv_a(x_linear) q_lora = qkv_a[..., : self.q_lora_rank] else: - q_lora, _ = self.wq_a(x) + q_lora, _ = self.wq_a(x_linear) qkv_a = None - q_lora = self.q_norm(q_lora) - q = self._compute_q_b(q_lora, positions, q_out) use_cp = self.dsa_enable_prefill_cp and dsa_use_prefill_cp(forward_batch) kv: Optional[torch.Tensor] - if use_cp: - # DSA CP: keep bf16 kv around for the cross-rank all-gather, then - # write to the FlashMLA cache after gather. - kv = self._compute_kv_bf16(x, positions, qkv_a=qkv_a) - kv = cp_all_gather_rerange_output( - kv.contiguous(), - self.cp_size, - forward_batch, - torch.cuda.current_stream(), + + if self.use_fused_qk_norm_rope: + + if _is_gfx95_supported: + q_for_wqb, q_lora = _fused_rmsnorm_fp8_quant( + q_lora, + self.q_norm.weight, + self.q_norm.variance_epsilon, + ) + q, _ = self.wq_b(q_for_wqb) + else: + q_lora = self.q_norm(q_lora) + q, _ = self.wq_b(q_lora) + + kv = ( + qkv_a[..., self.q_lora_rank :] + if qkv_a is not None + else self.wkv(x_linear)[0] ) - attn_backend.store_cache( - layer_id=self.layer_id, - swa_k=kv, - forward_batch=forward_batch, + + from sglang.srt.layers.fused_qk_norm_rope_store import ( + fused_qk_norm_rope_swa_store, + ) + + token_to_kv_pool = get_token_to_kv_pool() + swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa( + forward_batch.out_cache_loc ) + swa_cache = token_to_kv_pool.swa_kv_pool.kv_buffer[self.layer_id] + swa_page_size = token_to_kv_pool.swa_kv_pool.page_size + + q = fused_qk_norm_rope_swa_store( + q=q, + kv=kv, + q_norm_weight=None, + kv_norm_weight=self.kv_norm.weight, + q_rms_eps=self.eps, + kv_rms_eps=self.eps, + rope_head_dim=self.qk_rope_head_dim, + cos_cache=self.cos_cache, + sin_cache=self.sin_cache, + positions=positions, + swa_cache=swa_cache, + swa_loc=swa_loc, + swa_page_size=swa_page_size, + q_out=q_out, + dtype=x.dtype, + ) + + if use_cp: + # DSA CP: keep bf16 kv around for the cross-rank all-gather, then + # write to the FlashMLA cache after gather. + kv = self._compute_kv_bf16(x, positions, qkv_a=qkv_a) + kv = cp_all_gather_rerange_output( + kv.contiguous(), + self.cp_size, + forward_batch, + torch.cuda.current_stream(), + ) else: - self._compute_kv_to_cache(x, positions, forward_batch, qkv_a=qkv_a) - kv = None + q_lora = self.q_norm(q_lora) + q = self._compute_q_b(q_lora, positions, q_out) + if use_cp: + # NSA CP: keep bf16 kv around for the cross-rank all-gather, then + # write to the FlashMLA cache after gather. + kv = self._compute_kv_bf16(x_linear, positions, qkv_a=qkv_a) + kv = cp_all_gather_rerange_output( + kv.contiguous(), + self.cp_size, + forward_batch, + torch.cuda.current_stream(), + ) + attn_backend.store_cache( + layer_id=self.layer_id, + swa_k=kv, + forward_batch=forward_batch, + ) + else: + self._compute_kv_to_cache( + x_linear, positions, forward_batch, qkv_a=qkv_a + ) + kv = None del qkv_a if self.indexer is not None: - self.indexer(x=x, q_lora=q_lora, forward_batch=forward_batch) + self.indexer( + x=x, + q_lora=q_lora, + forward_batch=forward_batch, + attn_backend=attn_backend, + ) if self.compressor is not None: attn_backend.forward_core_compressor( x, @@ -549,6 +774,7 @@ def forward( x: torch.Tensor, positions: torch.Tensor, forward_batch: ForwardBatch, + x_quant=None, ) -> torch.Tensor: if not get_attn_tp_context().input_scattered and x.shape[0] == 0: assert ( @@ -556,7 +782,7 @@ def forward( ), "short-circuiting allreduce will lead to hangs" return x - attn_backend = forward_batch.attn_backend + attn_backend = get_attn_backend() if TYPE_CHECKING: assert isinstance( attn_backend, @@ -581,13 +807,33 @@ def forward( if enable_multi_stream: # Multi-stream path always fuses cache write into the K kernel, # so the bf16 KV intermediate is gone. - q = self._forward_prepare_multi_stream( - x, positions, forward_batch, attn_backend, q_out - ) + if _is_hip: + q = self._forward_prepare_multi_stream_hip( + x, + positions, + forward_batch, + attn_backend, + q_out, + x_quant=x_quant, + ) + else: + q = self._forward_prepare_multi_stream( + x, + positions, + forward_batch, + attn_backend, + q_out, + x_quant=x_quant, + ) kv = None else: q, kv = self._forward_prepare( - x, positions, forward_batch, attn_backend, q_out + x, + positions, + forward_batch, + attn_backend, + q_out, + x_quant=x_quant, ) # The cache write is always fused / already done by _forward_prepare* -- @@ -668,12 +914,20 @@ def __init__( alt_streams=alt_streams, compress_ratio_override=compress_ratio_override, ) + moe_alt_stream = ( + alt_streams[0] + if ( + alt_streams is not None + and (_is_cuda or envs.SGLANG_ROCM_USE_MULTI_STREAM.get()) + ) + else None + ) self.mlp = deepseek_v2.DeepseekV2MoE( config=config, quant_config=moe_quant_config_override or quant_config, prefix=add_prefix("mlp", prefix), layer_id=self.layer_id, - alt_stream=alt_streams[0] if alt_streams is not None else None, + alt_stream=moe_alt_stream, is_nextn=is_nextn, is_deepseek_v4=True, ) @@ -914,12 +1168,23 @@ def forward( norm=self.input_layernorm, ) if not norm_fused: - hidden_states = self.input_layernorm(hidden_states) + if _use_aiter and _is_gfx95_supported: + x_quant, hidden_states = _fused_rmsnorm_fp8_quant( + hidden_states, + self.input_layernorm.weight, + self.rms_norm_eps, + ) + else: + hidden_states = self.input_layernorm(hidden_states) + x_quant = None + else: + x_quant = None hidden_states = self.self_attn( x=hidden_states, positions=positions, forward_batch=forward_batch, + x_quant=x_quant, ) hidden_states = self.hc_post(hidden_states, residual, post, comb) @@ -1012,8 +1277,18 @@ def __init__( else: self.embed_tokens = PPMissingLayer() self.rms_norm_eps = config.rms_norm_eps + use_stream_pool = _is_cuda or ( + _is_hip + and ( + envs.SGLANG_ROCM_USE_MULTI_STREAM.get() + or envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get() + ) + ) + num_alt_streams = 5 if _is_cuda else 2 self.alt_streams = ( - [torch.cuda.Stream() for _ in range(5)] if (_is_cuda or _is_hip) else None + [torch.cuda.Stream() for _ in range(num_alt_streams)] + if use_stream_pool + else None ) self.layers, self.start_layer, self.end_layer = make_layers( config.num_hidden_layers, @@ -1128,19 +1403,29 @@ def forward( hidden_states = cp_split_and_rebuild_data(forward_batch, hidden_states) positions = cp_split_and_rebuild_position(forward_batch, positions) + # Reset Compressor's per-step freqs_cis cache from any previous step. + for _attr in ("freqs_cis_c4", "freqs_cis_c128"): + if hasattr(forward_batch, _attr): + delattr(forward_batch, _attr) # Upgrade lazy raw metadata on the main stream once before any layer # forks alt-streams; later per-layer calls become no-ops. - forward_batch.attn_backend._maybe_upgrade_forward_metadata() + get_attn_backend()._maybe_upgrade_forward_metadata() for i in range(self.start_layer, self.end_layer): layer = self.layers[i] - hidden_states = layer( - positions=positions, - hidden_states=hidden_states, - forward_batch=forward_batch, - input_ids=input_ids, - input_ids_global=input_ids_global, + ctx = ( + nullcontext() + if not get_global_server_args().disable_piecewise_cuda_graph + else get_global_expert_distribution_recorder().with_current_layer(i) ) + with ctx: + hidden_states = layer( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + input_ids=input_ids, + input_ids_global=input_ids_global, + ) # CP all-gather only on the last PP rank; PP IPC carries CP-split tensors. if self.pp_group.is_last_rank and dsa_use_prefill_cp(forward_batch): @@ -1253,6 +1538,22 @@ def determine_num_fused_shared_experts(self): if get_global_server_args().disable_shared_experts_fusion: return + # Waterfill needs shared-experts fusion so it can dispatch shared + # expert tokens to least-loaded EP ranks. + if get_global_server_args().enable_deepep_waterfill: + if self.config.n_shared_experts != 1: + raise ValueError( + "DeepEP Waterfill for DeepSeek V4 expects exactly one shared " + f"expert, but got n_shared_experts={self.config.n_shared_experts}." + ) + self.num_fused_shared_experts = self.config.n_shared_experts + log_info_on_rank0( + logger, + "DeepSeek V4: --enable-deepep-waterfill set; KEEP shared-experts " + "fusion enabled so waterfill can rebalance shared expert dispatch.", + ) + return + get_global_server_args().disable_shared_experts_fusion = True log_info_on_rank0( logger, @@ -1276,17 +1577,17 @@ def forward( self.cp_rank, self.cp_size, forward_batch.seq_lens_cpu.tolist(), + extend_lens=forward_batch.extend_seq_lens_cpu, ) if is_dsa_prefill_cp_round_robin_split(): - metadata = forward_batch.attn_backend.forward_metadata + attn_backend = get_attn_backend() + metadata = attn_backend.forward_metadata core_meta = metadata.core_attn_metadata core_meta.apply_cp_reindex() core_meta.init_flashmla_related() if metadata.indexer_metadata is not None: metadata.indexer_metadata = ( - forward_batch.attn_backend.init_forward_metadata_indexer( - core_meta - ) + attn_backend.init_forward_metadata_indexer(core_meta) ) with get_attn_tp_context().maybe_input_scattered(forward_batch): diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index f6d9a3f7d9e5..6b5c89e50999 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -37,6 +37,7 @@ VocabParallelEmbedding, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer, DeepseekV4ForCausalLM from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix @@ -248,17 +249,17 @@ def forward( self.cp_rank, self.cp_size, forward_batch.seq_lens_cpu.tolist(), + extend_lens=forward_batch.extend_seq_lens_cpu, ) if is_dsa_prefill_cp_round_robin_split(): - metadata = forward_batch.attn_backend.forward_metadata + attn_backend = get_attn_backend() + metadata = attn_backend.forward_metadata core_meta = metadata.core_attn_metadata core_meta.apply_cp_reindex() core_meta.init_flashmla_related() if metadata.indexer_metadata is not None: metadata.indexer_metadata = ( - forward_batch.attn_backend.init_forward_metadata_indexer( - core_meta - ) + attn_backend.init_forward_metadata_indexer(core_meta) ) hidden_states, pre_hc_head = self.model(input_ids, positions, forward_batch) diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 72f684c2bb9c..3be39824e77f 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -33,6 +33,7 @@ VocabParallelEmbedding, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_cuda, make_layers @@ -338,7 +339,7 @@ def forward( ) attention_hidden_states = attention_hidden_states * self.attn_out_multiplier - attn_backend = forward_batch.attn_backend + attn_backend = get_attn_backend() assert isinstance(attn_backend, HybridLinearAttnBackend) assert isinstance(attn_backend.linear_attn_backend, Mamba2AttnBackend) # Mamba block @@ -348,6 +349,7 @@ def forward( hidden_states * self.ssm_in_multiplier, mamba_hidden_states, layer_id=self.layer_id, + forward_batch=forward_batch, mup_vector=self.mup_vector, ) mamba_hidden_states = mamba_hidden_states * self.ssm_out_multiplier diff --git a/python/sglang/srt/models/gemma3_mm.py b/python/sglang/srt/models/gemma3_mm.py index 25745d3310b3..9b362dbba896 100644 --- a/python/sglang/srt/models/gemma3_mm.py +++ b/python/sglang/srt/models/gemma3_mm.py @@ -40,6 +40,7 @@ flatten_nested_list, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -220,7 +221,7 @@ def prepare_attn_masks( mask_dtype: torch.dtype, ): """Prepare attention masks for multimodal inputs.""" - if isinstance(forward_batch.attn_backend, TritonAttnBackend): + if isinstance(get_attn_backend(), TritonAttnBackend): assert forward_batch.forward_mode == ForwardMode.EXTEND bidirectional_attn_masks_list = [] bidirectional_attn_mask_indptr = torch.zeros( @@ -265,10 +266,10 @@ def prepare_attn_masks( bidirectional_attn_masks = torch.cat( bidirectional_attn_masks_list, dim=0 ) - forward_batch.attn_backend.forward_metadata.mask_indptr = ( + get_attn_backend().forward_metadata.mask_indptr = ( bidirectional_attn_mask_indptr ) - forward_batch.attn_backend.forward_metadata.custom_mask = ( + get_attn_backend().forward_metadata.custom_mask = ( bidirectional_attn_masks ) diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index 190452fcd124..c406f12a2b6c 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -1147,7 +1147,8 @@ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): ("experts.w13_weight", "experts.gate_up_proj", ("w1", "w3")), ("experts.w2_weight", "experts.down_proj", ("w2",)), ] - num_experts = self.config.num_experts + # Dense subclasses (e.g. the Gemma4 MTP assistant) reuse this. + num_experts = getattr(self.config, "num_experts", None) or 0 # Per-expert checkpoint format used by compressed-tensors / FP8 # (e.g. RedHatAI/*-FP8-Dynamic) and by ModelOpt NVFP4 @@ -1159,11 +1160,15 @@ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): # in a trailing dot, so the standard `name.replace(weight_name, # param_name)` collapses every suffix uniformly to the fused # FusedMoE params (experts.w13_*, experts.w2_*). - per_expert_params_mapping = FusedMoE.make_expert_params_mapping( - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=num_experts, + per_expert_params_mapping = ( + FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=num_experts, + ) + if num_experts + else [] ) k_eq_v_layers = self._get_k_eq_v_layers() diff --git a/python/sglang/srt/models/gemma4_mm.py b/python/sglang/srt/models/gemma4_mm.py index fb14dd17ab3b..cafc31f20ce8 100644 --- a/python/sglang/srt/models/gemma4_mm.py +++ b/python/sglang/srt/models/gemma4_mm.py @@ -52,6 +52,7 @@ ForwardMode, PPProxyTensors, ) +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -315,7 +316,7 @@ def prepare_attn_masks( TODO(kpham-sgl): Guard appropriately for gemma3_mm.py:prepare_attn_masks() """ - if not isinstance(forward_batch.attn_backend, TritonAttnBackend): + if not isinstance(get_attn_backend(), TritonAttnBackend): logger.warning_once( "Bidirectional attention for image tokens requires TritonAttnBackend. " "Falling back to causal attention, which may degrade image quality." @@ -389,12 +390,10 @@ def prepare_attn_masks( ) if bidirectional_attn_masks_list: bidirectional_attn_masks = torch.cat(bidirectional_attn_masks_list, dim=0) - forward_batch.attn_backend.forward_metadata.mask_indptr = ( + get_attn_backend().forward_metadata.mask_indptr = ( bidirectional_attn_mask_indptr ) - forward_batch.attn_backend.forward_metadata.custom_mask = ( - bidirectional_attn_masks - ) + get_attn_backend().forward_metadata.custom_mask = bidirectional_attn_masks def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: vt = self.vision_tower diff --git a/python/sglang/srt/models/gemma4_mtp.py b/python/sglang/srt/models/gemma4_mtp.py index 1cb87b7c2e99..ade10ce5b990 100644 --- a/python/sglang/srt/models/gemma4_mtp.py +++ b/python/sglang/srt/models/gemma4_mtp.py @@ -21,6 +21,7 @@ from torch import nn from transformers import PretrainedConfig, PreTrainedModel +from sglang.srt.distributed import get_pp_group from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import ( LogitsMetadata, @@ -72,6 +73,7 @@ def __init__( self.assistant_config = config self.config = text_config self.quant_config = quant_config + self.pp_group = get_pp_group() self.vocab_size = text_config.vocab_size self.hidden_size = text_config.hidden_size diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 4a58ab6972c9..6d1bb48a1588 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -1,4 +1,4 @@ -# Copyright 2025-2026 SGLang Team +# Copyright 2026-2027 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,11 +12,11 @@ # limitations under the License. # ============================================================================== -"""Inference-only GLM-4.7-Flash model compatible with HuggingFace weights""" +"""Inference-only GLM-4.7-Flash model compatible with HuggingFace weights.""" import logging import re -from typing import Iterable, Optional, Tuple +from typing import Iterable, List, Optional, Tuple, Union import torch import torch.nn.functional as F @@ -24,21 +24,29 @@ from transformers import PretrainedConfig from sglang.srt.batch_overlap.single_batch_overlap import SboFlags +from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo from sglang.srt.distributed import ( get_moe_expert_parallel_world_size, get_pp_group, get_tensor_model_parallel_world_size, + parallel_state, + tensor_model_parallel_all_reduce, ) +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.communicator import ( LayerCommunicator, LayerScatterModes, enable_moe_dense_fully_dp, + get_attn_tp_context, ) from sglang.srt.layers.dp_attention import ( - get_attention_tp_rank, - get_attention_tp_size, + is_allocation_symmetric, is_dp_attention_enabled, ) from sglang.srt.layers.layernorm import RMSNorm @@ -46,43 +54,39 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, + should_skip_post_experts_all_reduce, should_use_flashinfer_cutlass_moe_fp4_allgather, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.moe.kt_ep_wrapper import KTEPWrapperMethod from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat +from sglang.srt.layers.moe.utils import filter_moe_weight_param_global_expert from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import PPMissingLayer from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) +from sglang.srt.model_executor.cuda_graph_runner import get_is_capture_mode +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.models.deepseek_v2 import ( - DeepseekV2AttentionMLA, - DeepseekV2DecoderLayer, - DeepseekV2ForCausalLM, - DeepseekV2Model, - DeepseekV2MoE, +from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( + DeepseekV2WeightLoaderMixin, ) +from sglang.srt.models.deepseek_common.utils import _is_cuda, _use_aiter +from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( BumpAllocator, LazyValue, add_prefix, - get_device_sm, - is_cuda, + is_non_idle_and_non_empty, log_info_on_rank0, make_layers, ) from sglang.srt.utils.hf_transformers_utils import get_rope_config -_is_cuda = is_cuda() -_device_sm = get_device_sm() - -if _is_cuda: - from sgl_kernel import dsv3_router_gemm - logger = logging.getLogger(__name__) @@ -132,34 +136,14 @@ def forward( forward_batch=None, should_allreduce_fusion: bool = False, use_reduce_scatter: bool = False, - gemm_output_zero_allocator: BumpAllocator = None, ): - # Keep parity with DeepseekV2MLP.forward signature since DeepseekV2DecoderLayer - # invokes MLP modules with these extra arguments. if (self.tp_size == 1) and x.shape[0] == 0: return x - # Some quantization wrappers store the underlying parameter as `weight_packed`. - if not hasattr(self.gate_up_proj, "weight"): - self.gate_up_proj.weight = getattr(self.gate_up_proj, "weight_packed") - if not hasattr(self.down_proj, "weight"): - self.down_proj.weight = getattr(self.down_proj, "weight_packed") - - if ( - gemm_output_zero_allocator is not None - and x.shape[0] <= 256 - and self.gate_up_proj.weight.dtype == torch.uint8 - ): - y = gemm_output_zero_allocator.allocate( - x.shape[0] * self.gate_up_proj.output_size_per_partition - ).view(x.shape[0], self.gate_up_proj.output_size_per_partition) - x = (x, None, y) - gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) x, _ = self.down_proj( - x, - skip_all_reduce=should_allreduce_fusion or use_reduce_scatter, + x, skip_all_reduce=should_allreduce_fusion or use_reduce_scatter ) return x @@ -179,28 +163,18 @@ def __init__( self.e_score_correction_bias = nn.Parameter( torch.empty((config.n_routed_experts), dtype=torch.float32) ) - - def forward(self, hidden_states, gemm_output_zero_allocator: BumpAllocator = None): - # NOTE: For some unknown reason, router_gemm seems degrade accept length. - if ( - _is_cuda - and not self.is_nextn - and hidden_states.shape[0] < 4 - and hidden_states.shape[1] == 7168 - and self.weight.shape[0] == 256 - and _device_sm >= 90 - ): - - logits = dsv3_router_gemm(hidden_states, self.weight).to( - hidden_states.dtype - ) - else: - logits = F.linear(hidden_states, self.weight, None) - + # GLM requires FP32 gate projection; cache to avoid per-forward cast. + # FIXME: if gate weight is updated at runtime (e.g. expert rebalancing), _weight_fp32 must be invalidated. + self.register_buffer("_weight_fp32", None, persistent=False) + + def forward(self, hidden_states): + if self._weight_fp32 is None: + self._weight_fp32 = self.weight.data.to(torch.float32) + logits = F.linear(hidden_states.to(torch.float32), self._weight_fp32, None) return logits -class Glm4MoeLiteSparseMoeBlock(DeepseekV2MoE): +class Glm4MoeLiteSparseMoeBlock(nn.Module): def __init__( self, config: PretrainedConfig, @@ -210,7 +184,7 @@ def __init__( alt_stream: Optional[torch.cuda.Stream] = None, is_nextn: bool = False, ): - nn.Module.__init__(self) + super().__init__() self.tp_size = get_tensor_model_parallel_world_size() self.routed_scaling_factor = config.routed_scaling_factor self.n_shared_experts = config.n_shared_experts @@ -273,7 +247,8 @@ def __init__( self.shared_experts_is_int8 = False self.shared_experts_is_fp8 = False - # self.shared_experts_weight_block_size = None + self.shared_experts_weight_block_size = None + self._shared_expert_tp1 = False if config.n_shared_experts is not None and self.num_fused_shared_experts == 0: intermediate_size = config.moe_intermediate_size * config.n_shared_experts # disable tp for shared experts when enable deepep moe, or with fp4 allgather @@ -327,8 +302,241 @@ def __init__( ) self._fuse_shared_experts_inside_sbo = SboFlags.fuse_shared_experts_inside_sbo() + def get_moe_weights(self): + return [ + x.data + for name, x in self.experts.named_parameters() + if name not in ["correction_bias"] + and filter_moe_weight_param_global_expert( + name, x, self.experts.num_local_experts + ) + ] + + def forward( + self, + hidden_states: torch.Tensor, + forward_batch: Optional[ForwardBatch] = None, + should_allreduce_fusion: bool = False, + use_reduce_scatter: bool = False, + ) -> torch.Tensor: + if not self._enable_a2a_moe: + if ( + self.alt_stream is not None + and self.num_fused_shared_experts == 0 + and hidden_states.shape[0] > 0 + and get_is_capture_mode() + ): + return self.forward_normal_dual_stream( + hidden_states, should_allreduce_fusion, use_reduce_scatter + ) + else: + return self.forward_normal( + hidden_states, should_allreduce_fusion, use_reduce_scatter + ) + else: + return self.forward_deepep(hidden_states, forward_batch) + + def forward_normal_dual_stream( + self, + hidden_states: torch.Tensor, + should_allreduce_fusion: bool = False, + use_reduce_scatter: bool = False, + ) -> torch.Tensor: + current_stream = torch.cuda.current_stream() + self.alt_stream.wait_stream(current_stream) + shared_output = self._forward_shared_experts(hidden_states) + + with torch.cuda.stream(self.alt_stream): + # router_logits: (num_tokens, n_experts) + router_logits = self.gate(hidden_states) + topk_output = self.topk(hidden_states, router_logits) + final_hidden_states = self.experts(hidden_states, topk_output) + if not _is_cuda or isinstance(self.experts.quant_method, KTEPWrapperMethod): + final_hidden_states *= self.routed_scaling_factor + + current_stream.wait_stream(self.alt_stream) + final_hidden_states += shared_output + if self.tp_size > 1 and not should_skip_post_experts_all_reduce( + is_tp_path=True, + use_reduce_scatter=use_reduce_scatter, + should_allreduce_fusion=should_allreduce_fusion, + ): + final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + return final_hidden_states + + def forward_normal( + self, + hidden_states: torch.Tensor, + should_allreduce_fusion: bool = False, + use_reduce_scatter: bool = False, + ) -> torch.Tensor: + if hidden_states.shape[0] > 0: + shared_output = self._forward_shared_experts(hidden_states) + # router_logits: (num_tokens, n_experts) + router_logits = self.gate(hidden_states) + topk_output = self.topk(hidden_states, router_logits) + else: + shared_output = None + topk_output = self.topk.empty_topk_output(hidden_states.device) + + final_hidden_states = self.experts(hidden_states, topk_output) + if not _is_cuda and not _use_aiter: + final_hidden_states *= self.routed_scaling_factor + if shared_output is not None: + with use_symmetric_memory( + parallel_state.get_tp_group(), disabled=not is_allocation_symmetric() + ): + final_hidden_states_out = torch.empty_like(final_hidden_states) + torch.add(final_hidden_states, shared_output, out=final_hidden_states_out) + final_hidden_states = final_hidden_states_out + if self.tp_size > 1 and not should_skip_post_experts_all_reduce( + is_tp_path=True, + use_reduce_scatter=use_reduce_scatter, + should_allreduce_fusion=should_allreduce_fusion, + ): + final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + return final_hidden_states + + def forward_deepep( + self, hidden_states: torch.Tensor, forward_batch: ForwardBatch + ) -> torch.Tensor: + shared_output = None + if hidden_states.shape[0] > 0: + # router_logits: (num_tokens, n_experts) + router_logits = self.gate(hidden_states) + shared_output = self._forward_shared_experts(hidden_states) + topk_output = self.topk( + hidden_states, + router_logits, + num_token_non_padded=forward_batch.num_token_non_padded, + expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( + layer_id=self.layer_id, + ), + ) + else: + topk_output = self.topk.empty_topk_output(hidden_states.device) + + final_hidden_states = self.experts( + hidden_states=hidden_states, + topk_output=topk_output, + ) + + if shared_output is not None: + x = shared_output + if self.experts.should_fuse_routed_scaling_factor_in_topk: + x.add_(final_hidden_states) + else: + x.add_(final_hidden_states, alpha=self.routed_scaling_factor) + final_hidden_states = x + else: + if not self.experts.should_fuse_routed_scaling_factor_in_topk: + final_hidden_states *= self.routed_scaling_factor + + return final_hidden_states -class Glm4MoeLiteDecoderLayer(DeepseekV2DecoderLayer): + def _forward_shared_experts(self, hidden_states: torch.Tensor): + if (hidden_states.shape[0] > 0) and (self.num_fused_shared_experts == 0): + return self.shared_experts(hidden_states) + else: + return None + + def op_gate(self, state): + if is_non_idle_and_non_empty( + state.forward_batch.forward_mode, state.hidden_states_mlp_input + ): + # router_logits: (num_tokens, n_experts) + state.router_logits = self.gate(state.hidden_states_mlp_input) + else: + state.router_logits = None + + def op_shared_experts(self, state): + hidden_states_mlp_input = state.pop("hidden_states_mlp_input") + if (self.num_fused_shared_experts == 0) and is_non_idle_and_non_empty( + state.forward_batch.forward_mode, hidden_states_mlp_input + ): + state.shared_output = self.shared_experts(hidden_states_mlp_input) + else: + state.shared_output = None + + def op_select_experts(self, state): + router_logits = state.pop("router_logits") + hidden_states = state.hidden_states_mlp_input + + if router_logits is not None: + with get_global_expert_distribution_recorder().with_current_layer( + self.layer_id + ): + state.topk_output = self.topk( + hidden_states=hidden_states, + router_logits=router_logits, + num_token_non_padded=state.forward_batch.num_token_non_padded, + expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( + layer_id=self.layer_id, + ), + ) + else: + state.topk_output = self.topk.empty_topk_output(hidden_states.device) + + def op_dispatch_a(self, state): + if self.ep_size > 1: + self.experts.dispatcher.dispatch_a( + hidden_states=state.hidden_states_mlp_input, + topk_output=state.pop("topk_output"), + tbo_subbatch_index=state.get("tbo_subbatch_index"), + ) + + def op_dispatch_b(self, state): + if self.ep_size > 1: + with get_global_expert_distribution_recorder().with_current_layer( + self.layer_id + ): + state.dispatch_output = self.experts.dispatcher.dispatch_b( + tbo_subbatch_index=state.get("tbo_subbatch_index"), + ) + + def op_experts(self, state): + state.combine_input = self.experts.run_moe_core( + dispatch_output=state.dispatch_output, + ) + + def op_combine_a(self, state): + if self.ep_size > 1: + self.experts.dispatcher.combine_a( + combine_input=state.pop("combine_input"), + tbo_subbatch_index=state.get("tbo_subbatch_index"), + ) + state.pop("dispatch_output") + + def op_combine_b(self, state): + if self.ep_size > 1: + state.hidden_states_after_combine = self.experts.dispatcher.combine_b( + tbo_subbatch_index=state.get("tbo_subbatch_index"), + ) + + def op_output(self, state): + final_hidden_states = state.pop("hidden_states_after_combine") + + if get_moe_a2a_backend().is_mori(): + num_tokens = state.pop("num_tokens") + final_hidden_states = final_hidden_states[:num_tokens] + + if (shared_output := state.pop("shared_output")) is not None: + x = shared_output + if _use_aiter: + x.add_(final_hidden_states) + else: + x.add_(final_hidden_states, alpha=self.routed_scaling_factor) + final_hidden_states = x + elif _use_aiter: + # fused in aiter_biased_grouped_topk so we can skip here + pass + else: + final_hidden_states *= self.routed_scaling_factor + + state.hidden_states_mlp_output = final_hidden_states + + +class Glm4MoeLiteDecoderLayer(nn.Module): def __init__( self, config: PretrainedConfig, @@ -338,13 +546,14 @@ def __init__( prefix: str = "", alt_stream: Optional[torch.cuda.Stream] = None, ) -> None: - nn.Module.__init__(self) + + super().__init__() self.hidden_size = config.hidden_size self.config = config - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() rope_theta, rope_scaling = get_rope_config(config) max_position_embeddings = getattr(config, "max_position_embeddings", 202752) self.layer_id = layer_id + self.is_nextn = is_nextn self.self_attn = DeepseekV2AttentionMLA( config=config, @@ -418,26 +627,171 @@ def __init__( qkv_latent_func=self.self_attn.prepare_qkv_latent, ) + def _detect_gfx95_quant_format(self) -> str: + from sglang.srt.models.deepseek_common.utils import _is_gfx95_supported + + if not _is_gfx95_supported: + return "" + weight = getattr( + getattr(self.self_attn, "fused_qkv_a_proj_with_mqa", None), "weight", None + ) + if weight is None: + return "" + if weight.dtype == torch.uint8: + return "mxfp4" + if weight.dtype == getattr(torch, "float8_e4m3fn", None): + return "fp8" + return "" + + def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool: + return is_nextn or ( + self.config.n_routed_experts is not None + and layer_id >= self.config.first_k_dense_replace + and layer_id % self.config.moe_layer_freq == 0 + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + residual: Optional[torch.Tensor], + zero_allocator: BumpAllocator, + ) -> torch.Tensor: + hidden_states, residual = self.layer_communicator.prepare_attn( + hidden_states, + residual, + forward_batch, + getattr(self, "_gfx95_quant_format", ""), + ) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + zero_allocator=zero_allocator, + layer_scatter_modes=self.layer_scatter_modes, + ) + if isinstance(hidden_states, tuple): + hidden_states = hidden_states[0] + get_attn_tp_context().clear_attn_inputs() + + hidden_states, residual = self.layer_communicator.prepare_mlp( + hidden_states, residual, forward_batch + ) + + should_allreduce_fusion = ( + self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer( + forward_batch + ) + ) + + # For DP with padding, reduce scatter can be used instead of all-reduce. + use_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( + forward_batch + ) + + hidden_states = self.mlp( + hidden_states, forward_batch, should_allreduce_fusion, use_reduce_scatter + ) + + if should_allreduce_fusion: + hidden_states._sglang_needs_allreduce_fusion = True + else: + hidden_states, residual = self.layer_communicator.postprocess_layer( + hidden_states, residual, forward_batch + ) + + return hidden_states, residual + + def op_comm_prepare_attn( + self, + state, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + residual: Optional[torch.Tensor], + zero_allocator: BumpAllocator, + tbo_subbatch_index: Optional[int] = None, + ): + state.hidden_states_after_comm_pre_attn, state.residual_after_input_ln = ( + self.layer_communicator.prepare_attn(hidden_states, residual, forward_batch) + ) + if get_moe_a2a_backend().is_mori(): + state.num_tokens = hidden_states.shape[0] + state.update( + dict( + forward_batch=forward_batch, + positions=positions, + zero_allocator=zero_allocator, + tbo_subbatch_index=tbo_subbatch_index, + ) + ) + + def op_comm_prepare_mlp(self, state): + state.hidden_states_mlp_input, state.residual_after_comm_pre_mlp = ( + self.layer_communicator.prepare_mlp( + state.pop("hidden_states_after_attn"), + state.pop("residual_after_input_ln"), + state.forward_batch, + ) + ) + + def op_mlp(self, state): + hidden_states = state.pop("hidden_states_mlp_input") + if not ( + enable_moe_dense_fully_dp() + and (not self.is_layer_sparse) + and hidden_states.shape[0] == 0 + ): + state.hidden_states_mlp_output = self.mlp( + hidden_states, state.forward_batch + ) + else: + state.hidden_states_mlp_output = hidden_states + + def op_comm_postprocess_layer(self, state): + hidden_states, residual = self.layer_communicator.postprocess_layer( + state.pop("hidden_states_mlp_output"), + state.pop("residual_after_comm_pre_mlp"), + state.forward_batch, + ) + + output = dict( + positions=state.positions, + hidden_states=hidden_states, + residual=residual, + forward_batch=state.forward_batch, + zero_allocator=state.zero_allocator, + tbo_subbatch_index=state.tbo_subbatch_index, + ) + + state.clear( + expect_keys={ + "positions", + "forward_batch", + "zero_allocator", + "tbo_subbatch_index", + } + ) + return output + + +class Glm4MoeLiteModel(nn.Module): + fall_back_to_pt_during_load = False -class Glm4MoeLiteModel(DeepseekV2Model): def __init__( self, config: PretrainedConfig, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", ): - nn.Module.__init__(self) + super().__init__() self.padding_id = config.pad_token_id self.vocab_size = config.vocab_size self.first_k_dense_replace = config.first_k_dense_replace self.pp_group = get_pp_group() - # DeepseekV2Model.forward expects these attributes to exist. - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() - self.cp_size = get_attention_tp_size() if self.dsa_enable_prefill_cp else None - self.gemm_output_zero_allocator_size = 0 - self.llama_4_scaling_config = getattr(config, "llama_4_scaling", None) - if self.pp_group.is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, @@ -467,15 +821,103 @@ def __init__( self.norm = PPMissingLayer(return_tuple=True) self.layers_to_capture = [] + def get_input_embeddings(self) -> torch.Tensor: + return self.embed_tokens + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor = None, + pp_proxy_tensors: Optional[PPProxyTensors] = None, + ) -> Union[torch.Tensor, PPProxyTensors]: + total_num_layers = self.end_layer - self.start_layer + if self.pp_group.is_first_rank: + if input_embeds is None: + hidden_states = self.embed_tokens(input_ids) + else: + hidden_states = input_embeds + residual = None + else: + assert pp_proxy_tensors is not None + hidden_states = pp_proxy_tensors["hidden_states"] + residual = pp_proxy_tensors["residual"] + device = hidden_states.device + zero_allocator = BumpAllocator( + buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1), + dtype=torch.float32, + device=device, + ) + + normal_start_layer = self.start_layer + normal_end_layer = self.end_layer + if forward_batch.can_run_tbo: + if ( + self.first_k_dense_replace > normal_start_layer + and self.first_k_dense_replace < normal_end_layer + ): + normal_end_layer = self.first_k_dense_replace + elif self.first_k_dense_replace < normal_start_layer: + normal_end_layer = normal_start_layer = 0 + aux_hidden_states = [] + for i in range(normal_start_layer, normal_end_layer): + with get_global_expert_distribution_recorder().with_current_layer(i): + if i in self.layers_to_capture: + aux_hidden_states.append(hidden_states + residual) + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + forward_batch, + residual, + zero_allocator, + ) + + if normal_end_layer != self.end_layer: + hidden_states, residual = model_forward_maybe_tbo( + layers=self.layers[normal_end_layer : self.end_layer], + enable_tbo=True, + positions=positions, + forward_batch=forward_batch, + hidden_states=hidden_states, + residual=residual, + input_data_scatter_mode=self.layers[ + normal_end_layer - 1 + ].layer_scatter_modes.layer_output_mode, + zero_allocator=zero_allocator, + ) + + if not self.pp_group.is_last_rank: + return PPProxyTensors( + { + "hidden_states": hidden_states, + "residual": residual, + } + ) + else: + if not forward_batch.forward_mode.is_idle(): + if residual is None: + hidden_states = self.norm(hidden_states) + else: + hidden_states, _ = self.norm(hidden_states, residual) + + if len(aux_hidden_states) == 0: + return hidden_states + return hidden_states, aux_hidden_states + + +class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): + # for quark model load + packed_modules_mapping = {} -class Glm4MoeLiteForCausalLM(DeepseekV2ForCausalLM): def __init__( self, config: PretrainedConfig, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", ) -> None: - nn.Module.__init__(self) + super().__init__() config.moe_layer_freq = 1 self.config = config self.tp_size = get_tensor_model_parallel_world_size() @@ -503,12 +945,9 @@ def __init__( ) self.capture_aux_hidden_states = False - self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() - if self.dsa_enable_prefill_cp: - self.cp_rank = get_attention_tp_rank() - self.cp_size = get_attention_tp_size() - else: - self.cp_rank = self.cp_size = None + @property + def routed_experts_weights_of_layer(self): + return self._routed_experts_weights_of_layer.value def determine_num_fused_shared_experts( self, architecture: str = "Glm4MoeLiteForCausalLM" @@ -539,6 +978,89 @@ def determine_num_fused_shared_experts( self.num_fused_shared_experts = self.config.n_shared_experts + def get_input_embeddings(self) -> nn.Embedding: + return self.model.embed_tokens + + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor = None, + pp_proxy_tensors: Optional[PPProxyTensors] = None, + ) -> torch.Tensor: + with get_attn_tp_context().maybe_input_scattered(forward_batch): + hidden_states = self.model( + input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors + ) + aux_hidden_states = None + if self.capture_aux_hidden_states: + hidden_states, aux_hidden_states = hidden_states + + if self.pp_group.is_last_rank: + return self.logits_processor( + input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states + ) + else: + return hidden_states + + @property + def start_layer(self): + return self.model.start_layer + + @property + def end_layer(self): + return self.model.end_layer + + def get_embed_and_head(self): + return self.model.embed_tokens.weight, self.lm_head.weight + + def set_embed_and_head(self, embed, head): + del self.model.embed_tokens.weight + del self.lm_head.weight + self.model.embed_tokens.weight = embed + self.lm_head.weight = head + torch.cuda.empty_cache() + torch.cuda.synchronize() + + @classmethod + def get_model_config_for_expert_location(cls, config): + return ModelConfigForExpertLocation( + num_layers=config.num_hidden_layers, + num_logical_experts=config.n_routed_experts, + num_groups=config.n_group, + ) + + def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None): + if not self.pp_group.is_last_rank: + return + + if layer_ids is None: + self.capture_aux_hidden_states = True + num_layers = self.config.num_hidden_layers + self.model.layers_to_capture = [2, num_layers // 2, num_layers - 3] + else: + self.capture_aux_hidden_states = True + # TODO (Qiaolin-Yu): check if other draft models need similar layer id + # adjustment + if layer_ids and layer_ids[0] == 1: + self.model.layers_to_capture = [val + 1 for val in layer_ids] + else: + self.model.layers_to_capture = list(layer_ids) + + def set_dflash_layers_to_capture(self, layer_ids: List[int]): + if not self.pp_group.is_last_rank: + return + + if layer_ids is None: + raise ValueError( + "DFLASH requires explicit layer_ids for aux hidden capture." + ) + + self.capture_aux_hidden_states = True + self.model.layers_to_capture = [val + 1 for val in layer_ids] + def load_weights( self, weights: Iterable[Tuple[str, torch.Tensor]], diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py new file mode 100644 index 000000000000..103ce18ab7a4 --- /dev/null +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -0,0 +1,182 @@ +# Copyright 2026-2027 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Inference-only GLM-4.7-Flash Speculative Decoding (NextN) compatible with HuggingFace weights.""" + +import logging +from typing import Iterable, Optional, Tuple + +import torch +from torch import nn +from transformers import PretrainedConfig + +from sglang.srt.distributed import get_tensor_model_parallel_world_size +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.models.glm4_moe_lite import ( + Glm4MoeLiteDecoderLayer, + Glm4MoeLiteForCausalLM, +) +from sglang.srt.server_args import get_global_server_args +from sglang.srt.utils import BumpAllocator, add_prefix, is_npu + +logger = logging.getLogger(__name__) + + +class Glm4MoeLiteModelNextN(nn.Module): + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + if quant_config is not None and quant_config.get_name() == "modelopt_fp4": + logger.warning( + "Overriding Glm4MoeLiteForCausalLMNextN quant config for modelopt_fp4 " + "GLM-4.7-Flash model." + ) + quant_config = None + + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + use_attn_tp_group=is_dp_attention_enabled(), + prefix=add_prefix("embed_tokens", prefix), + ) + + self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.eh_proj = nn.Linear(2 * config.hidden_size, config.hidden_size, bias=False) + + self.decoder = Glm4MoeLiteDecoderLayer( + config, + 0, + quant_config=quant_config, + is_nextn=True, + prefix=add_prefix("decoder", prefix), + ) + + self.shared_head = nn.Module() + self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor = None, + ) -> torch.Tensor: + # Glm4MoeLiteDecoderLayer uses DeepseekV2AttentionMLA, which requires a + # zero_allocator (the GQA glm4_moe_nextn path does not pass one). + zero_allocator = BumpAllocator( + buffer_size=2, + dtype=torch.float32, + device=( + input_embeds.device if input_embeds is not None else input_ids.device + ), + ) + + if input_embeds is None: + hidden_states = self.embed_tokens(input_ids) + else: + hidden_states = input_embeds + + if hidden_states.shape[0] > 0: + hidden_states = self.eh_proj( + torch.cat( + ( + self.enorm(hidden_states), + self.hnorm(forward_batch.spec_info.hidden_states), + ), + dim=-1, + ) + ) + + residual = None + with get_global_expert_distribution_recorder().disable_this_region(): + hidden_states, residual = self.decoder( + positions, hidden_states, forward_batch, residual, zero_allocator + ) + + if not forward_batch.forward_mode.is_idle(): + if residual is not None: + hidden_states, _ = self.shared_head.norm(hidden_states, residual) + else: + hidden_states = self.shared_head.norm(hidden_states) + + return hidden_states + + +class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM): + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + nn.Module.__init__(self) + self.config = config + self.tp_size = get_tensor_model_parallel_world_size() + if ( + is_npu() + and get_global_server_args().speculative_draft_model_quantization is None + ): + quant_config = None + self.quant_config = quant_config + + self.model = Glm4MoeLiteModelNextN( + config, quant_config, prefix=add_prefix("model", prefix) + ) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=add_prefix("model.shared_head.head", prefix), + use_attn_tp_group=get_global_server_args().enable_dp_lm_head, + ) + self.logits_processor = LogitsProcessor(config) + + self.num_fused_shared_experts = ( + 0 if get_global_server_args().disable_shared_experts_fusion else 1 + ) + + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + hidden_states = self.model(input_ids, positions, forward_batch) + return self.logits_processor( + input_ids, hidden_states, self.lm_head, forward_batch + ) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + super().load_weights(weights, is_nextn=True) + + +EntryClass = [Glm4MoeLiteForCausalLMNextN] diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index f6f2e72df38e..84f8890ee454 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -219,7 +219,7 @@ def __init__( bias=True, quant_config=None, prefix=add_prefix("gate", prefix), - params_dtype=config.torch_dtype, + params_dtype=config.dtype, ) def forward( @@ -468,7 +468,7 @@ def __init__( prefix=add_prefix("self_attn", prefix), sliding_window_size=self.sliding_window_size, layer_type=config.layer_types[layer_id], - params_dtype=config.torch_dtype, + params_dtype=config.dtype, ) self.layer_id = layer_id diff --git a/python/sglang/srt/models/granitemoehybrid.py b/python/sglang/srt/models/granitemoehybrid.py index e18aeb466a9c..85385b4fd220 100644 --- a/python/sglang/srt/models/granitemoehybrid.py +++ b/python/sglang/srt/models/granitemoehybrid.py @@ -29,6 +29,7 @@ VocabParallelEmbedding, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.transformers import maybe_prefix from sglang.srt.utils import make_layers @@ -139,7 +140,7 @@ def forward( hidden_states = self.input_layernorm(hidden_states) output = torch.empty_like(hidden_states) - attn_backend = forward_batch.attn_backend + attn_backend = get_attn_backend() assert isinstance(attn_backend, HybridLinearAttnBackend) assert isinstance(attn_backend.linear_attn_backend, Mamba2AttnBackend) attn_backend.linear_attn_backend.forward( @@ -147,6 +148,7 @@ def forward( layer_id=self.layer_idx, hidden_states=hidden_states, output=output, + forward_batch=forward_batch, use_triton_causal_conv=True, ) diff --git a/python/sglang/srt/models/jet_nemotron.py b/python/sglang/srt/models/jet_nemotron.py index 1e6d2ec87e1c..fec8c1fb6e63 100644 --- a/python/sglang/srt/models/jet_nemotron.py +++ b/python/sglang/srt/models/jet_nemotron.py @@ -28,6 +28,7 @@ from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen2 import Qwen2MLP, Qwen2Model from sglang.srt.utils import add_prefix @@ -258,11 +259,9 @@ def forward( hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - assert isinstance(forward_batch.attn_backend, HybridLinearAttnBackend) - assert isinstance( - forward_batch.attn_backend.linear_attn_backend, MambaAttnBackendBase - ) - linear_attn_backend = forward_batch.attn_backend.linear_attn_backend + assert isinstance(get_attn_backend(), HybridLinearAttnBackend) + assert isinstance(get_attn_backend().linear_attn_backend, MambaAttnBackendBase) + linear_attn_backend = get_attn_backend().linear_attn_backend forward_metadata = linear_attn_backend.forward_metadata layer_cache = linear_attn_backend.req_to_token_pool.mamba2_layer_cache( self.layer_id diff --git a/python/sglang/srt/models/kimi_k25.py b/python/sglang/srt/models/kimi_k25.py index 832ee74dd00e..c1d134e8e388 100644 --- a/python/sglang/srt/models/kimi_k25.py +++ b/python/sglang/srt/models/kimi_k25.py @@ -680,9 +680,13 @@ def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: pixel_values = torch.cat([item.feature for item in items], dim=0).to( device=device, dtype=target_dtype ) - grid_thws = torch.concat([item.image_grid_thw for item in items], dim=0).to( - device - ) + image_grid_thws = [] + for item in items: + grid_thw = item.model_specific_data.get("image_grid_thw") + if grid_thw is None: + grid_thw = item.model_specific_data["grid_thws"] + image_grid_thws.append(grid_thw) + grid_thws = torch.concat(image_grid_thws, dim=0).to(device) if self.use_data_parallel: image_embeds = run_dp_sharded_mrope_vision_model( diff --git a/python/sglang/srt/models/lfm2.py b/python/sglang/srt/models/lfm2.py index 2750a0f81e47..1f4f7544e3c7 100644 --- a/python/sglang/srt/models/lfm2.py +++ b/python/sglang/srt/models/lfm2.py @@ -40,6 +40,7 @@ VocabParallelEmbedding, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import get_req_to_token_pool from sglang.srt.model_loader.weight_utils import ( default_weight_loader, sharded_weight_loader, @@ -263,12 +264,10 @@ def forward( if forward_batch.forward_mode.is_idle(): return hidden_states - layer_cache = forward_batch.req_to_token_pool.mamba2_layer_cache(self.layer_idx) + layer_cache = get_req_to_token_pool().mamba2_layer_cache(self.layer_idx) conv_state = layer_cache.conv[0] req_pool_indices = forward_batch.req_pool_indices - mamba_indices = forward_batch.req_to_token_pool.get_mamba_indices( - req_pool_indices - ) + mamba_indices = get_req_to_token_pool().get_mamba_indices(req_pool_indices) # Project and split into gates: B (pre-conv), C (post-conv), x (input) proj, _ = self.in_proj(hidden_states) diff --git a/python/sglang/srt/models/lfm2_moe.py b/python/sglang/srt/models/lfm2_moe.py index fcc396357c83..4846b0b9954c 100644 --- a/python/sglang/srt/models/lfm2_moe.py +++ b/python/sglang/srt/models/lfm2_moe.py @@ -42,6 +42,7 @@ VocabParallelEmbedding, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import get_req_to_token_pool from sglang.srt.model_loader.weight_utils import ( default_weight_loader, sharded_weight_loader, @@ -326,12 +327,10 @@ def forward( if forward_batch.forward_mode.is_idle(): return hidden_states - layer_cache = forward_batch.req_to_token_pool.mamba2_layer_cache(self.layer_idx) + layer_cache = get_req_to_token_pool().mamba2_layer_cache(self.layer_idx) conv_state = layer_cache.conv[0] req_pool_indices = forward_batch.req_pool_indices - mamba_indices = forward_batch.req_to_token_pool.get_mamba_indices( - req_pool_indices - ) + mamba_indices = get_req_to_token_pool().get_mamba_indices(req_pool_indices) proj, _ = self.in_proj(hidden_states) B_gate, C_gate, x = proj.chunk(3, dim=-1) diff --git a/python/sglang/srt/models/llava.py b/python/sglang/srt/models/llava.py index 712c1f4f82d6..1f07f8a416b9 100644 --- a/python/sglang/srt/models/llava.py +++ b/python/sglang/srt/models/llava.py @@ -13,8 +13,11 @@ # ============================================================================== """Inference-only LLaVa model compatible with HuggingFace weights.""" +from __future__ import annotations + import math import re +from array import array from functools import lru_cache from typing import Dict, Iterable, List, Optional, Tuple, Type, Union @@ -73,7 +76,9 @@ def _infer_image_aspect_ratio(mm_items): return "pad" return "anyres" - def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs): + def pad_input_ids( + self, input_ids: array[int], image_inputs: MultimodalInputs + ) -> array[int]: image_sizes = flatten_nested_list( [item.image_sizes for item in image_inputs.mm_items] ) @@ -125,9 +130,10 @@ def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs): except ValueError: offset = 0 # old_len + pad_len - 1, because we need to remove image_token_id + pad_token = pad_values[image_idx % len(pad_values)] input_ids = ( input_ids[:offset] - + [pad_values[image_idx % len(pad_values)]] * new_image_feature_len + + array("q", [pad_token]) * new_image_feature_len + input_ids[offset + 1 :] ) offset_list.append(offset) diff --git a/python/sglang/srt/models/llavavid.py b/python/sglang/srt/models/llavavid.py index dc4df698ebd9..f21c744855ec 100644 --- a/python/sglang/srt/models/llavavid.py +++ b/python/sglang/srt/models/llavavid.py @@ -13,7 +13,10 @@ # ============================================================================== """Inference-only LLaVa video model compatible with HuggingFace weights.""" -from typing import Iterable, List, Optional, Tuple +from __future__ import annotations + +from array import array +from typing import Iterable, Optional, Tuple import numpy as np import torch @@ -57,8 +60,10 @@ def __init__( torch.empty(config.text_config.hidden_size, dtype=torch.float16) ) - def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs): - pad_values = [item.pad_value for item in image_inputs.mm_items] + def pad_input_ids( + self, input_ids: array[int], image_inputs: MultimodalInputs + ) -> array[int]: + pad_values = array("q", (item.pad_value for item in image_inputs.mm_items)) new_image_feature_len = self.image_feature_len pad_ids = pad_values * ( diff --git a/python/sglang/srt/models/mindspore.py b/python/sglang/srt/models/mindspore.py index da95ab139f17..b91197286019 100644 --- a/python/sglang/srt/models/mindspore.py +++ b/python/sglang/srt/models/mindspore.py @@ -14,6 +14,10 @@ from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import ( + get_req_to_token_pool, + get_token_to_kv_pool, +) from sglang.srt.models.registry import import_model_classes from sglang.srt.utils import is_npu @@ -221,9 +225,9 @@ def get_kvcache(self, forward_batch: ForwardBatch): def prepare_cache(cache_list, is_key_cache): for i in range(self.config.num_hidden_layers): if is_key_cache: - cache = forward_batch.token_to_kv_pool.get_key_buffer(i) + cache = get_token_to_kv_pool().get_key_buffer(i) else: - cache = forward_batch.token_to_kv_pool.get_value_buffer(i) + cache = get_token_to_kv_pool().get_value_buffer(i) cache_ms = tensor_torch2ms(cache) if self.use_mla and cache_ms.ndim == 3: cache_ms = mint.unsqueeze(cache_ms, 2) @@ -275,10 +279,10 @@ def prepare_inputs(self, input_ids, positions, forward_batch): if forward_batch.forward_mode.is_target_verify(): q_seq_lens = q_seq_lens * forward_batch.spec_info.num_tokens_per_req - page_size = forward_batch.token_to_kv_pool.page_size + page_size = get_token_to_kv_pool().page_size block_tables = tensor_torch2ms( ( - forward_batch.req_to_token_pool.req_to_token[ + get_req_to_token_pool().req_to_token[ forward_batch.req_pool_indices, : batch_valid_length.max() ][:, ::page_size] // page_size diff --git a/python/sglang/srt/models/mistral_large_3_eagle.py b/python/sglang/srt/models/mistral_large_3_eagle.py index 65ffbd820213..ae487c52c86e 100644 --- a/python/sglang/srt/models/mistral_large_3_eagle.py +++ b/python/sglang/srt/models/mistral_large_3_eagle.py @@ -7,11 +7,13 @@ from torch import nn from transformers import PretrainedConfig +from sglang.srt.configs.model_config import is_deepseek_dsa from sglang.srt.distributed import get_pp_group from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import RowParallelLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.utils.cp_utils import is_prefill_context_parallel_enabled from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.models.deepseek_v2 import DeepseekV2DecoderLayer, DeepseekV2Model @@ -36,6 +38,9 @@ def __init__( assert get_pp_group().world_size == 1 self.pp_group = get_pp_group() self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp() + self.mla_enable_prefill_cp = ( + is_prefill_context_parallel_enabled() and not is_deepseek_dsa(config) + ) self.embed_tokens = VocabParallelEmbedding( config.vocab_size, @@ -50,6 +55,8 @@ def __init__( prefix=add_prefix(prefix, f"layers.{i}"), quant_config=quant_config, layer_id=i, + dsa_enable_prefill_cp=self.dsa_enable_prefill_cp, + mla_enable_prefill_cp=self.mla_enable_prefill_cp, ) for i in range(self.config.num_hidden_layers) ] diff --git a/python/sglang/srt/models/mllama.py b/python/sglang/srt/models/mllama.py index 8f05d9432620..ba50cf8ebf16 100644 --- a/python/sglang/srt/models/mllama.py +++ b/python/sglang/srt/models/mllama.py @@ -4,7 +4,10 @@ # https://github.com/vllm-project/vllm/blob/7193774b1ff8603ad5bf4598e5efba0d9a39b436/vllm/model_executor/models/mllama.py """PyTorch Mllama model.""" +from __future__ import annotations + import math +from array import array from typing import Iterable, List, Optional, Tuple, Union import torch @@ -823,9 +826,11 @@ def __init__( ) self.logits_processor = LogitsProcessor(config.text_config) - def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): + def pad_input_ids( + self, input_ids: array[int], mm_inputs: MultimodalInputs + ) -> array[int]: pixel_values = torch.cat([item.feature for item in mm_inputs.mm_items], dim=0) - pad_values = [item.pad_value for item in mm_inputs.mm_items] + pad_values = array("q", (item.pad_value for item in mm_inputs.mm_items)) num_concurrent_media, num_tiles = pixel_values.shape[1:3] num_patches = self.vision_model.num_patches diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index f3e09e7bd346..3a47b58c4df5 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +from array import array from functools import partial from typing import Iterable, List, Optional, Tuple @@ -1122,15 +1123,17 @@ def _get_encoder_len(self, mm_inputs: MultimodalInputs) -> int: return total_len - def _build_encoder_prefix_pad_ids(self, mm_inputs: MultimodalInputs) -> List[int]: + def _build_encoder_prefix_pad_ids(self, mm_inputs: MultimodalInputs) -> array[int]: encoder_len = self._get_encoder_len(mm_inputs) if encoder_len == 0 or not mm_inputs.mm_items: - return [] + return array("q") pad_value = mm_inputs.mm_items[0].pad_value - return [pad_value] * encoder_len + return array("q", [pad_value]) * encoder_len - def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): + def pad_input_ids( + self, input_ids: array[int], mm_inputs: MultimodalInputs + ) -> array[int]: encoder_len = self._get_encoder_len(mm_inputs) mm_inputs.num_image_tokens = encoder_len if encoder_len == 0: diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 1e879455f198..1840b0727fb1 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -69,6 +69,7 @@ is_in_breakable_cuda_graph, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -414,7 +415,7 @@ def _forward_mamba( ) -> torch.Tensor: """Core Mamba forward logic, called directly or via split op.""" output = torch.empty_like(hidden_states) - attn_backend = forward_batch.attn_backend + attn_backend = get_attn_backend() assert isinstance(attn_backend, HybridLinearAttnBackend) assert isinstance(attn_backend.linear_attn_backend, Mamba2AttnBackend) attn_backend.linear_attn_backend.forward( @@ -422,6 +423,7 @@ def _forward_mamba( layer_id=self.layer_id, hidden_states=hidden_states, output=output, + forward_batch=forward_batch, use_triton_causal_conv=True, ) return output @@ -1020,7 +1022,7 @@ def nemotron_mamba2_with_output( # In piecewise CUDA graph mode, hidden_states may be padded to the # captured graph size. Slice to actual token count for Mamba forward. - attn_backend = forward_batch.attn_backend + attn_backend = get_attn_backend() metadata = attn_backend.linear_attn_backend.forward_metadata num_actual_tokens = metadata.num_prefill_tokens + ( metadata.num_decodes * metadata.draft_token_num diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 21d262b71806..2a18e2adb191 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -23,6 +23,7 @@ from sglang.srt.layers.utils import PPMissingLayer, get_layer_id from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, @@ -214,7 +215,7 @@ def forward_prepare_aiter_fused_mrope( qkv_3d = qkv.view(num_tokens, -1, self.head_dim) - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = get_token_to_kv_pool() k_cache, v_cache = token_to_kv_pool.get_kv_buffer(self.attn.layer_id) slot_mapping = forward_batch.out_cache_loc diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 0887bfc3f6ad..1e31930fbe9b 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -1005,6 +1005,7 @@ def forward( self.attn_cp_rank, self.attn_cp_size, forward_batch.seq_lens_cpu.tolist(), + extend_lens=forward_batch.extend_seq_lens_cpu, ) hidden_states = self.model( diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 1b6c185bcbda..44dddf1bfb5d 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -756,7 +756,7 @@ def forward( return self.forward_with_npu_graph(x, grid_thw) return self.forward_with_cuda_graph(x, grid_thw) - x = x.to(device=self.device, dtype=self.dtype) + x = x.to(device=self.device, dtype=self.dtype, non_blocking=True) x = self.patch_embed(x) if isinstance(grid_thw, list): @@ -938,7 +938,7 @@ def _prepare_graph_inputs(self, x: torch.Tensor, grid_thw: torch.Tensor) -> tupl torch.Tensor, ]: # patchify - x = x.to(device=self.device, dtype=self.dtype) + x = x.to(device=self.device, dtype=self.dtype, non_blocking=True) x = self.patch_embed(x) if isinstance(grid_thw, list): diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index bca26936d99d..83683933cbb0 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -54,6 +54,10 @@ ) from sglang.srt.model_executor.cuda_graph_runner import get_is_capture_mode from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.model_executor.forward_context import ( + get_attn_backend, + get_token_to_kv_pool, +) from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.bailing_moe import BailingMoEForCausalLM from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import ( @@ -75,8 +79,9 @@ if _is_cuda: try: - from sgl_kernel import bmm_fp8, concat_mla_k, merge_state_v2 + from sgl_kernel import bmm_fp8, merge_state_v2 + from sglang.jit_kernel.concat_mla import concat_mla_k from sglang.srt.layers.quantization.fp8_kernel import per_tensor_quant_mla_fp8 _has_fp8_support = True @@ -605,7 +610,7 @@ def _concat_and_cast_mha_k( self.current_attention_backend == "fa3" and self.kv_cache_dtype != "auto" ): - attn_dtype = forward_batch.token_to_kv_pool.dtype + attn_dtype = get_token_to_kv_pool().dtype else: attn_dtype = k_nope.dtype k = k_nope.new_empty(*k_shape, dtype=attn_dtype) @@ -671,7 +676,7 @@ def _run_mha_prefill( q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) q[..., self.qk_nope_head_dim :] = q_pe - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + get_token_to_kv_pool().set_mla_kv_buffer( self.attn_mha, forward_batch.out_cache_loc, k_nope, @@ -701,8 +706,8 @@ def _run_mha_prefill( forward_batch.prepare_chunked_prefix_cache_info(q.device) else: forward_batch.num_prefix_chunks = 0 - if hasattr(forward_batch.attn_backend, "init_mha_chunk_metadata"): - forward_batch.attn_backend.init_mha_chunk_metadata(forward_batch) + if hasattr(get_attn_backend(), "init_mha_chunk_metadata"): + get_attn_backend().init_mha_chunk_metadata(forward_batch) forward_batch.set_attn_attend_prefix_cache(False) forward_batch.mha_return_lse = do_prefix_merge diff --git a/python/sglang/srt/models/utils.py b/python/sglang/srt/models/utils.py index 92588e1775e6..341f7b458541 100644 --- a/python/sglang/srt/models/utils.py +++ b/python/sglang/srt/models/utils.py @@ -31,6 +31,7 @@ from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.cuda_graph_runner import get_is_capture_mode from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import get_current_device_stream_fast, is_cuda, is_hip @@ -275,11 +276,11 @@ def load_weights( def enable_fused_set_kv_buffer(forward_batch: ForwardBatch): """Enable fused set_kv_buffer only on CUDA with bfloat16 KV cache.""" + pool = get_token_to_kv_pool() return ( _is_cuda - and hasattr(forward_batch.token_to_kv_pool, "dtype") - and forward_batch.token_to_kv_pool.dtype == torch.bfloat16 - and not isinstance(forward_batch.token_to_kv_pool, SWAKVPool) + and pool.dtype == torch.bfloat16 + and not isinstance(pool, SWAKVPool) and not is_prefill_context_parallel_enabled() ) or (_is_hip and not is_prefill_context_parallel_enabled()) @@ -292,7 +293,7 @@ def create_fused_set_kv_buffer_arg( from sglang.jit_kernel.rope import FusedSetKVBufferArg layer_id = layer.layer_id - token_to_kv_pool = forward_batch.token_to_kv_pool + token_to_kv_pool = get_token_to_kv_pool() k_buffer = token_to_kv_pool.get_key_buffer(layer_id) v_buffer = token_to_kv_pool.get_value_buffer(layer_id) diff --git a/python/sglang/srt/models/whisper.py b/python/sglang/srt/models/whisper.py index 091b4cde4d8b..2c8f7aa4306b 100644 --- a/python/sglang/srt/models/whisper.py +++ b/python/sglang/srt/models/whisper.py @@ -1,4 +1,7 @@ -from typing import Any, Iterable, List, Optional, Tuple +from __future__ import annotations + +from array import array +from typing import Any, Iterable, Optional, Tuple import torch from transformers import WhisperConfig @@ -418,14 +421,15 @@ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, loaded_weight) - def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): + def pad_input_ids( + self, input_ids: array[int], mm_inputs: MultimodalInputs + ) -> array[int]: # Prepend dummy encoder tokens so that prepare_encoder_info_extend # correctly allocates encoder KV cache locations in the KV pool. # These dummy tokens are stripped before the model forward receives input_ids. encoder_len = self.config.max_source_positions mm_inputs.num_image_tokens = encoder_len - pad_ids = [0] * encoder_len - return pad_ids + input_ids + return array("q", [0]) * encoder_len + input_ids def forward( self, diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index cf64b06feb7c..d3bb63f5ff5a 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -1,3 +1,4 @@ +import asyncio import concurrent import concurrent.futures import dataclasses @@ -49,6 +50,10 @@ class BaseMultiModalProcessorOutput: # input_text with all multimodality placeholder token expanded input_text: str + # original pre-tokenized ids, useful for processor_output/precomputed inputs, + # when they already carry the input ids + input_ids: Optional[Union[List[int], torch.Tensor]] = None + # frames loaded from image, in given order images: Optional[list[Union[Image.Image, dict]]] = dataclasses.field( default_factory=list @@ -517,15 +522,8 @@ def _load_single_item( Class method that can be pickled for multiprocessing """ - if isinstance(data, dict): - data_format = data.get("format") - if data_format in ( - MultimodalInputFormat.PROCESSOR_OUTPUT.name, - MultimodalInputFormat.PRECOMPUTED_EMBEDDING.name, - "processor_output", - "precomputed_embedding", - ): - return data + if cls._is_preprocessed_input(data): + return data try: if modality == Modality.IMAGE: img, _ = load_image(data, cls.gpu_image_decode) @@ -545,6 +543,49 @@ def _load_single_item( except Exception as e: raise RuntimeError(f"Error while loading data {data}: {e}") + @staticmethod + def _get_preprocessed_input_format(data): + """returns the detailed format if the provided data is already preprocessed. + returns none if the provided data is not preprocessed + """ + if not isinstance(data, dict): + return None + data_format = data.get("format") + if isinstance(data_format, MultimodalInputFormat): + return data_format + if data_format in ( + MultimodalInputFormat.PROCESSOR_OUTPUT.name, + "processor_output", + ): + return MultimodalInputFormat.PROCESSOR_OUTPUT + if data_format in ( + MultimodalInputFormat.PRECOMPUTED_EMBEDDING.name, + "precomputed_embedding", + ): + return MultimodalInputFormat.PRECOMPUTED_EMBEDDING + return None + + @classmethod + def _is_preprocessed_input(cls, data): + """returns if the data is already preprocessed (by the vlm processor)""" + return cls._get_preprocessed_input_format(data) is not None + + @classmethod + def _all_mm_data_is_preprocessed(cls, *data_lists): + has_mm_data = False + for data_list in data_lists: + if not data_list: + continue + if not isinstance(data_list, list): + data_list = [data_list] + for item in data_list: + if item is None: + continue + has_mm_data = True + if not cls._is_preprocessed_input(item): + return False + return has_mm_data + def _submit_mm_data_loading_tasks_simple( self, data_list: Optional[list], @@ -669,10 +710,8 @@ def _validate_one_modality(modality: Modality, data_list: Optional[list]): formatted_indices = [] for idx, item in enumerate(data_list): - if isinstance(item, dict): - fmt = item.get("format") - if fmt in {"processor_output", "precomputed_embedding"}: - formatted_indices.append(idx) + if BaseMultimodalProcessor._is_preprocessed_input(item): + formatted_indices.append(idx) if formatted_indices: if len(data_list) != 1: @@ -707,12 +746,7 @@ def validate_mm_data( def _process_loaded_mm_data(self, modality, raw_data, result): images, videos, audios = [], [], [] - is_precomputed = isinstance(raw_data, dict) and raw_data.get("format") in [ - MultimodalInputFormat.PROCESSOR_OUTPUT.name, - MultimodalInputFormat.PRECOMPUTED_EMBEDDING.name, - "processor_output", - "precomputed_embedding", - ] + is_precomputed = self._is_preprocessed_input(raw_data) if modality == Modality.IMAGE: if is_precomputed: @@ -729,7 +763,7 @@ def _process_loaded_mm_data(self, modality, raw_data, result): return is_precomputed, images, videos, audios - def load_mm_data( + async def load_mm_data( self, prompt: str, multimodal_tokens: MultimodalSpecialTokens, @@ -743,6 +777,19 @@ def load_mm_data( BaseMultimodalProcessor.validate_mm_data(image_data, video_data, audio_data) + input_ids = prompt if isinstance(prompt, list) else None + if input_ids is not None and self._all_mm_data_is_preprocessed( + image_data, video_data, audio_data + ): + # fast path for preprocessed data: early return + return BaseMultiModalProcessorOutput( + input_text="", + input_ids=input_ids, + images=list(image_data or []), + videos=list(video_data or []), + audios=list(audio_data or []), + ) + multimodal_tokens_pattern = multimodal_tokens.get_combined_regex() if isinstance(prompt, list) and return_text: assert len(prompt) and isinstance(prompt[0], int) @@ -772,7 +819,7 @@ def load_mm_data( or cnt[Modality.AUDIO] != n_audio or getattr(self, "support_dynamic_frame_expansion", False) ): - return self.legacy_load_mm_data( + return await self.legacy_load_mm_data( prompt=prompt, multimodal_tokens=multimodal_tokens, image_data=image_data, @@ -781,10 +828,11 @@ def load_mm_data( return_text=return_text, discard_alpha_channel=discard_alpha_channel, audio_sample_rate=audio_sample_rate, + input_ids=input_ids, ) # For models other than MiniCPMO and MiniCPMV, # totally align multimodal_tokens, fast path - return self.fast_load_mm_data( + return await self.fast_load_mm_data( prompt=prompt, multimodal_tokens=multimodal_tokens, image_data=image_data, @@ -793,9 +841,10 @@ def load_mm_data( return_text=return_text, discard_alpha_channel=discard_alpha_channel, audio_sample_rate=audio_sample_rate, + input_ids=input_ids, ) - def fast_load_mm_data( + async def fast_load_mm_data( self, prompt: str, multimodal_tokens: MultimodalSpecialTokens, @@ -805,6 +854,7 @@ def fast_load_mm_data( return_text: Optional[bool] = True, discard_alpha_channel: bool = True, audio_sample_rate: Optional[int] = None, + input_ids: Optional[Union[List[int], torch.Tensor]] = None, ) -> BaseMultiModalProcessorOutput: """ A fast version of `load_mm_data` that loads multimodal data directly. @@ -847,7 +897,7 @@ def fast_load_mm_data( for modality, idx, future in futures: try: - result = future.result() + result = await asyncio.wrap_future(future) except Exception as e: logger.exception( "[load_mm_data(simple)] error loading %s data at index=%d", @@ -877,9 +927,10 @@ def fast_load_mm_data( audios=audios, videos=videos, input_text=prompt_str, + input_ids=input_ids, ) - def legacy_load_mm_data( + async def legacy_load_mm_data( self, prompt: str, multimodal_tokens: MultimodalSpecialTokens, @@ -889,6 +940,7 @@ def legacy_load_mm_data( return_text: Optional[bool] = True, discard_alpha_channel: bool = True, audio_sample_rate: Optional[int] = None, + input_ids: Optional[Union[List[int], torch.Tensor]] = None, ) -> BaseMultiModalProcessorOutput: """ Each frame of video/image will be replaced by a single image token @@ -939,7 +991,7 @@ def legacy_load_mm_data( try: if multimodal_tokens_pattern.match(text_part): modality, raw_data, frame_limit = next(task_info_iter) - result = next(futures_iter).result() + result = await asyncio.wrap_future(next(futures_iter)) is_precomputed, new_imgs, new_vids, new_auds = ( self._process_loaded_mm_data(modality, raw_data, result) @@ -996,6 +1048,7 @@ def legacy_load_mm_data( audios=audios, videos=videos, input_text="".join(new_text_parts), + input_ids=input_ids, ) @staticmethod @@ -1027,28 +1080,53 @@ def collect_mm_items_from_processor_output( self, data_dict: dict, modality: Modality = None ) -> List[MultimodalDataItem]: """ - Create mm_items from processor output. Initially creates one item per modality; - these are later split into per-image/video items by get_new_expanded_mm_items. + Create mm_items from processor output. + + Initially creates one item per modality; these are later split into per-image/video items by get_new_expanded_mm_items. - Note that the data_dict can be passed via offline engine api + Note that the data_dict can be hf processor output, or passed via offline engine api + + Args: + modality: if provided, force the data into a single MultimodalDataItem of that modality """ + # universal getter for data_dict + get_data_value = ( + data_dict.get + if hasattr(data_dict, "get") + else lambda name, default=None: getattr(data_dict, name, default) + ) + + # decide explicitly-set modality + explicit_modality = modality + modality_value = get_data_value("modality") + if explicit_modality is None and modality_value is not None: + explicit_modality = ( + modality_value + if isinstance(modality_value, Modality) + else Modality.from_str(str(modality_value)) + ) + items: dict[Modality, MultimodalDataItem] = {} for attr_name, value in data_dict.items(): - if attr_name == "input_ids": + if attr_name in ( + "input_ids", + "format", + "modality", + "hash", + "pad_value", + "offsets", + ): + # metadata fields need explicit handling, skip generic item.set continue # Get modality for this attribute - current_modality = modality or self.ATTR_NAME_TO_MODALITY.get(attr_name) + current_modality = explicit_modality or self.ATTR_NAME_TO_MODALITY.get( + attr_name + ) if attr_name == "precomputed_embeddings": - modality_str = data_dict.get("modality") - current_modality = Modality.IMAGE - if modality_str: - try: - current_modality = Modality.from_str(modality_str) - except ValueError: - pass + current_modality = current_modality or Modality.IMAGE if current_modality: # Create item if needed @@ -1062,6 +1140,30 @@ def collect_mm_items_from_processor_output( items[current_modality].set(attr_name, value) + # deal with metadata fields when data_dict is preprocessed input: convert from tensor to expected python types + # the attribution of the metadata fields is only clear when number of MultimodalDataItem is 1 + if len(items) == 1: + item = next(iter(items.values())) + + # adjust offset + offsets = get_data_value("offsets") + if offsets is not None: + if isinstance(offsets, torch.Tensor): + offsets = offsets.detach().cpu().tolist() + item.offsets = [(int(start), int(end)) for start, end in offsets] + + # adjust hash_value + hash_value = get_data_value("hash") + if hash_value is not None: + if isinstance(hash_value, torch.Tensor): + hash_value = hash_value.item() + item.hash = int(hash_value) + pad_value = get_data_value("pad_value") + if pad_value is not None: + if isinstance(pad_value, torch.Tensor): + pad_value = pad_value.item() + item.pad_value = int(pad_value) + return list(items.values()) def _process_and_collect_mm_items( @@ -1082,6 +1184,41 @@ def _process_and_collect_mm_items( return collected_items, input_ids, ret + @staticmethod + def _ensure_input_ids_is_tensor(input_ids) -> Optional[torch.Tensor]: + """make sure the input_ids is a flattened tensor""" + if input_ids is None: + return None + if isinstance(input_ids, torch.Tensor): + return input_ids.flatten().to(dtype=torch.long) + return torch.tensor(input_ids, dtype=torch.long).flatten() + + def _wrap_tensor_for_cuda_ipc(self, tensor: torch.Tensor): + """helper function to turn a tensor into a cuda-ipc tensor""" + if not tensor.is_cuda: + return tensor + + sync_flag, available_slice, byte_offset = ( + self.cudaipc_mmfeature_pool.return_a_slice_tensor_with_flag(tensor) + ) + if isinstance(available_slice, torch.Tensor): + available_slice.copy_(tensor.view(torch.int8).view(-1), non_blocking=True) + return CudaIpcTensorTransportProxy( + data=available_slice, + info_data=tensor, + sync_buffer_meta=sync_flag, + pool_ipc_handle=( + self.cudaipc_mmfeature_pool._pool_ipc_handle + if _IPC_POOL_HANDLE_CACHE + else None + ), + pool_byte_offset=byte_offset, + pool_device_index=self.cudaipc_mmfeature_pool._pool_device_index, + ) + if self.server_args.keep_mm_feature_on_device: + return tensor + return tensor.cpu() + def process_and_combine_mm_data( self, base_output: BaseMultiModalProcessorOutput, @@ -1135,16 +1272,19 @@ def process_and_combine_mm_data( ret = None # Handle dict items (processed or precomputed) + dict_ret = None for modality, dict_item in dict_items: - input_format = dict_item.get("format", None) - if input_format == "processor_output": + input_format = self._get_preprocessed_input_format(dict_item) + if input_format is not None and dict_ret is None: + dict_ret = dict_item + if input_format == MultimodalInputFormat.PROCESSOR_OUTPUT: items = self.collect_mm_items_from_processor_output(dict_item) for item in items: item.format = MultimodalInputFormat.PROCESSOR_OUTPUT all_collected_items.extend(items) - elif input_format == "precomputed_embedding": - feature = dict_item["feature"] - del dict_item["feature"] + elif input_format == MultimodalInputFormat.PRECOMPUTED_EMBEDDING: + dict_item = dict(dict_item) + feature = dict_item.pop("feature") all_collected_items.append( MultimodalDataItem( modality=modality, @@ -1154,6 +1294,18 @@ def process_and_combine_mm_data( ) ) # Fallback tokenization if no raw items were processed + if ret is None and dict_ret is not None: + ret = dict_ret + + if input_ids is None: + input_ids = self._ensure_input_ids_is_tensor(base_output.input_ids) + + if input_ids is None: + for _, dict_item in dict_items: + input_ids = self._ensure_input_ids_is_tensor(dict_item.get("input_ids")) + if input_ids is not None: + break + if input_ids is None: input_ids = self._tokenizer( base_output.input_text, @@ -1163,6 +1315,8 @@ def process_and_combine_mm_data( # Add offsets to all items for mm_item in all_collected_items: + if mm_item.offsets is not None: + continue mm_token_id = mm_tokens.get_token_id_by_modality(mm_item.modality) if mm_token_id is None: raise ValueError(f"No token id found for modality: {mm_item.modality}") @@ -1176,6 +1330,13 @@ def process_and_combine_mm_data( all_collected_items = get_new_expanded_mm_items(all_collected_items) + for item in all_collected_items: + if item.format in ( + MultimodalInputFormat.PROCESSOR_OUTPUT, + MultimodalInputFormat.PRECOMPUTED_EMBEDDING, + ): + item.set_pad_value() + """ solution for cuda-ipc memory-leak: 1. memory-pool: each time get a slice from memory-pool and use it as transport-data (with async lock guard) @@ -1185,60 +1346,13 @@ def process_and_combine_mm_data( """ if SGL_USE_CUDA_IPC: - # post-process + # post-process, prepare for cuda-ipc transfer for item in all_collected_items: - if isinstance(item.feature, torch.Tensor) and item.feature.is_cuda: - sync_flag, available_slice, byte_offset = ( - self.cudaipc_mmfeature_pool.return_a_slice_tensor_with_flag( - item.feature - ) + if isinstance(item.feature, torch.Tensor): + item.feature = self._wrap_tensor_for_cuda_ipc(item.feature) + if isinstance(item.precomputed_embeddings, torch.Tensor): + item.precomputed_embeddings = self._wrap_tensor_for_cuda_ipc( + item.precomputed_embeddings ) - if isinstance(available_slice, torch.Tensor): - available_slice.copy_( - item.feature.view(torch.int8).view(-1), non_blocking=True - ) - item.feature = CudaIpcTensorTransportProxy( - data=available_slice, - info_data=item.feature, - sync_buffer_meta=sync_flag, - pool_ipc_handle=( - self.cudaipc_mmfeature_pool._pool_ipc_handle - if _IPC_POOL_HANDLE_CACHE - else None - ), - pool_byte_offset=byte_offset, - pool_device_index=self.cudaipc_mmfeature_pool._pool_device_index, - ) - elif not self.server_args.keep_mm_feature_on_device: - item.feature = item.feature.cpu() - elif ( - isinstance(item.precomputed_embeddings, torch.Tensor) - and item.precomputed_embeddings.is_cuda - ): - - sync_flag, available_slice, byte_offset = ( - self.cudaipc_mmfeature_pool.return_a_slice_tensor_with_flag( - item.precomputed_embeddings - ) - ) - if isinstance(available_slice, torch.Tensor): - available_slice.copy_( - item.precomputed_embeddings.view(torch.int8).view(-1), - non_blocking=True, - ) - item.precomputed_embeddings = CudaIpcTensorTransportProxy( - data=available_slice, - info_data=item.precomputed_embeddings, - sync_buffer_meta=sync_flag, - pool_ipc_handle=( - self.cudaipc_mmfeature_pool._pool_ipc_handle - if _IPC_POOL_HANDLE_CACHE - else None - ), - pool_byte_offset=byte_offset, - pool_device_index=self.cudaipc_mmfeature_pool._pool_device_index, - ) - elif not self.server_args.keep_mm_feature_on_device: - item.precomputed_embeddings = item.precomputed_embeddings.cpu() return all_collected_items, input_ids, ret diff --git a/python/sglang/srt/multimodal/processors/clip.py b/python/sglang/srt/multimodal/processors/clip.py index 06f785b85ce8..3265b6ab4269 100644 --- a/python/sglang/srt/multimodal/processors/clip.py +++ b/python/sglang/srt/multimodal/processors/clip.py @@ -20,7 +20,7 @@ def __init__(self, hf_config, server_args, _processor, *args, **kwargs): async def process_mm_data_async( self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/deepseek_ocr.py b/python/sglang/srt/multimodal/processors/deepseek_ocr.py index becb0b2b32d0..bbf64cbd5451 100644 --- a/python/sglang/srt/multimodal/processors/deepseek_ocr.py +++ b/python/sglang/srt/multimodal/processors/deepseek_ocr.py @@ -29,7 +29,7 @@ def __init__(self, hf_config, server_args, _processor, *args, **kwargs): async def process_mm_data_async( self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py b/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py index 3a08a3c6170a..56f325175c30 100644 --- a/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py +++ b/python/sglang/srt/multimodal/processors/deepseek_vl_v2.py @@ -44,7 +44,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/dots_vlm.py b/python/sglang/srt/multimodal/processors/dots_vlm.py index c8e76562ad57..bc269cdaef1e 100644 --- a/python/sglang/srt/multimodal/processors/dots_vlm.py +++ b/python/sglang/srt/multimodal/processors/dots_vlm.py @@ -61,7 +61,7 @@ async def process_mm_data_async( ): image_data = sum(image_data, []) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/ernie45_vl.py b/python/sglang/srt/multimodal/processors/ernie45_vl.py index 8bb3475be871..1bd690555390 100644 --- a/python/sglang/srt/multimodal/processors/ernie45_vl.py +++ b/python/sglang/srt/multimodal/processors/ernie45_vl.py @@ -388,7 +388,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/gemma3.py b/python/sglang/srt/multimodal/processors/gemma3.py index c6b35e843f8c..7390f14ea131 100644 --- a/python/sglang/srt/multimodal/processors/gemma3.py +++ b/python/sglang/srt/multimodal/processors/gemma3.py @@ -37,7 +37,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/gemma3n.py b/python/sglang/srt/multimodal/processors/gemma3n.py index 5cb6d796289f..6c6c62064f7d 100644 --- a/python/sglang/srt/multimodal/processors/gemma3n.py +++ b/python/sglang/srt/multimodal/processors/gemma3n.py @@ -52,7 +52,7 @@ async def process_mm_data_async( **kwargs, ): """Process multimodal data including images and audio.""" - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, audio_data=audio_data, diff --git a/python/sglang/srt/multimodal/processors/gemma4.py b/python/sglang/srt/multimodal/processors/gemma4.py index 80bb37061358..d8fd6bd0a323 100644 --- a/python/sglang/srt/multimodal/processors/gemma4.py +++ b/python/sglang/srt/multimodal/processors/gemma4.py @@ -124,7 +124,7 @@ async def process_mm_data_async( **kwargs, ): """Process multimodal data including images, video, and audio.""" - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data if request_obj else None, diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py index a44f14b6ca28..db684259d2f6 100644 --- a/python/sglang/srt/multimodal/processors/glm4v.py +++ b/python/sglang/srt/multimodal/processors/glm4v.py @@ -90,7 +90,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/glmasr.py b/python/sglang/srt/multimodal/processors/glmasr.py index 1fcaf490a3b5..e55656fe1cb2 100644 --- a/python/sglang/srt/multimodal/processors/glmasr.py +++ b/python/sglang/srt/multimodal/processors/glmasr.py @@ -35,7 +35,7 @@ async def process_mm_data_async( input_text, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/interns1pro.py b/python/sglang/srt/multimodal/processors/interns1pro.py index 0f4a909ad67d..21c6ff16ff6f 100644 --- a/python/sglang/srt/multimodal/processors/interns1pro.py +++ b/python/sglang/srt/multimodal/processors/interns1pro.py @@ -49,7 +49,7 @@ async def process_mm_data_async( **kwargs, ): entry_time = time.perf_counter() - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/internvl.py b/python/sglang/srt/multimodal/processors/internvl.py index 800f6811066e..899c4acce495 100644 --- a/python/sglang/srt/multimodal/processors/internvl.py +++ b/python/sglang/srt/multimodal/processors/internvl.py @@ -310,7 +310,7 @@ async def _process_special_format( videos=videos, ) else: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=image_data, video_data=video_data, @@ -423,7 +423,7 @@ async def process_qwen_mm_data_async( prompt.count(self.VIDEO_PLACEHOLDER_TOKEN), ) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=image_data, video_data=video_data, @@ -644,7 +644,7 @@ async def process_internlm2_mm_data_async( prompt.count(self.IMG_CONTEXT), ) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=image_data, multimodal_tokens=self.mm_tokens_internlm2, # expects diff --git a/python/sglang/srt/multimodal/processors/janus_pro.py b/python/sglang/srt/multimodal/processors/janus_pro.py index f6711058d870..4c8a755c23a3 100644 --- a/python/sglang/srt/multimodal/processors/janus_pro.py +++ b/python/sglang/srt/multimodal/processors/janus_pro.py @@ -26,7 +26,7 @@ async def process_mm_data_async( request_obj, **kwargs, ): - base_out = self.load_mm_data( + base_out = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/kimi_k25.py b/python/sglang/srt/multimodal/processors/kimi_k25.py index 9838ca510e4b..ff2af7b92a75 100644 --- a/python/sglang/srt/multimodal/processors/kimi_k25.py +++ b/python/sglang/srt/multimodal/processors/kimi_k25.py @@ -373,7 +373,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/kimi_vl.py b/python/sglang/srt/multimodal/processors/kimi_vl.py index 6c0e16a1c43d..7bcf2885cfae 100644 --- a/python/sglang/srt/multimodal/processors/kimi_vl.py +++ b/python/sglang/srt/multimodal/processors/kimi_vl.py @@ -34,7 +34,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/lfm2_vl.py b/python/sglang/srt/multimodal/processors/lfm2_vl.py index c80720651c56..28d98dc59467 100644 --- a/python/sglang/srt/multimodal/processors/lfm2_vl.py +++ b/python/sglang/srt/multimodal/processors/lfm2_vl.py @@ -68,7 +68,7 @@ async def process_mm_data_async( "im_token_id": self.IMAGE_TOKEN_ID, } - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/midashenglm.py b/python/sglang/srt/multimodal/processors/midashenglm.py index 526cdc979b5b..985be22fd5d8 100644 --- a/python/sglang/srt/multimodal/processors/midashenglm.py +++ b/python/sglang/srt/multimodal/processors/midashenglm.py @@ -103,7 +103,7 @@ async def process_mm_data_async( input_text = f"{self.AUDIO_TOKEN}{input_text}" logger.info("Auto-prepended audio token") - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/mimo_v2.py b/python/sglang/srt/multimodal/processors/mimo_v2.py index 89c2619f36f8..0c6c82d712cf 100644 --- a/python/sglang/srt/multimodal/processors/mimo_v2.py +++ b/python/sglang/srt/multimodal/processors/mimo_v2.py @@ -28,6 +28,7 @@ Qwen2_5_VLVisionConfig, ) +from sglang.srt.environ import envs from sglang.srt.managers.schedule_batch import ( Modality, MultimodalDataItem, @@ -1815,9 +1816,7 @@ def __init__(self, hf_config, server_args, _processor, *args, **kwargs): self.video_end_token_id = self._require_config_value( processor_config, "video_end_token_id" ) - self.use_image_processor_gpu = ( - int(os.getenv("SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU", "0")) == 1 - ) + self.use_image_processor_gpu = envs.SGLANG_ENCODER_IMAGE_PROCESSOR_USE_GPU.get() device = server_args.device if self.use_image_processor_gpu else None self.mimo_processor = MiMoProcessor( @@ -2107,7 +2106,7 @@ async def process_mm_data_async( input_text = f"{self.mm_tokens.audio_token}{input_text}" video_data = getattr(request_obj, "video_data", []) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=video_data, diff --git a/python/sglang/srt/multimodal/processors/minicpm.py b/python/sglang/srt/multimodal/processors/minicpm.py index d4c407c13703..9df74cbde904 100644 --- a/python/sglang/srt/multimodal/processors/minicpm.py +++ b/python/sglang/srt/multimodal/processors/minicpm.py @@ -118,7 +118,7 @@ async def _process_special_format( audios=audios, ) else: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, image_data=normalized_images, audio_data=audio_data, @@ -190,7 +190,7 @@ async def process_mm_data_async( **kwargs, ) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/minicpmv4_6.py b/python/sglang/srt/multimodal/processors/minicpmv4_6.py index 25529b9b86e1..c2a0f67b1bee 100644 --- a/python/sglang/srt/multimodal/processors/minicpmv4_6.py +++ b/python/sglang/srt/multimodal/processors/minicpmv4_6.py @@ -419,7 +419,7 @@ async def process_mm_data_async( video_data = getattr(request_obj, "video_data", None) or kwargs.get( "video_data" ) - base = self.load_mm_data( + base = await self.load_mm_data( prompt=input_text, audio_data=audio_data, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/mlama.py b/python/sglang/srt/multimodal/processors/mlama.py index 52129765c75a..a12c9e2c4f96 100644 --- a/python/sglang/srt/multimodal/processors/mlama.py +++ b/python/sglang/srt/multimodal/processors/mlama.py @@ -21,7 +21,7 @@ def __init__(self, hf_config, server_args, _processor, *args, **kwargs): async def process_mm_data_async( self, image_data: List[Union[str, bytes]], input_text, *args, **kwargs ): - base_out = self.load_mm_data( + base_out = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/mllama4.py b/python/sglang/srt/multimodal/processors/mllama4.py index 3983df2755af..470e6de588a9 100644 --- a/python/sglang/srt/multimodal/processors/mllama4.py +++ b/python/sglang/srt/multimodal/processors/mllama4.py @@ -30,7 +30,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/moss_vl.py b/python/sglang/srt/multimodal/processors/moss_vl.py index a4b77a739357..409d9eb23de8 100644 --- a/python/sglang/srt/multimodal/processors/moss_vl.py +++ b/python/sglang/srt/multimodal/processors/moss_vl.py @@ -511,7 +511,7 @@ async def process_mm_data_async( ) try: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.image_only_mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py b/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py index 04f9b5f3f338..32bab7ae4470 100644 --- a/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py +++ b/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py @@ -213,7 +213,7 @@ def render_audio(self, *, num_tokens: int): async def process_mm_data_async( self, image_data, audio_data, input_text, request_obj, **kwargs ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/nvila.py b/python/sglang/srt/multimodal/processors/nvila.py index 5fe64d10d155..63a706218a15 100644 --- a/python/sglang/srt/multimodal/processors/nvila.py +++ b/python/sglang/srt/multimodal/processors/nvila.py @@ -55,7 +55,7 @@ async def process_mm_data_async( request_obj: GenerateReqInput, **kwargs, ) -> dict[str, Any] | None: - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=request_obj.image_data, # type: ignore diff --git a/python/sglang/srt/multimodal/processors/phi4mm.py b/python/sglang/srt/multimodal/processors/phi4mm.py index 6ae194eacfdd..0cce5b296592 100644 --- a/python/sglang/srt/multimodal/processors/phi4mm.py +++ b/python/sglang/srt/multimodal/processors/phi4mm.py @@ -74,7 +74,7 @@ async def process_mm_data_async( request_obj, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/pixtral.py b/python/sglang/srt/multimodal/processors/pixtral.py index 963bd68205c1..a9ee65e81158 100644 --- a/python/sglang/srt/multimodal/processors/pixtral.py +++ b/python/sglang/srt/multimodal/processors/pixtral.py @@ -71,7 +71,7 @@ async def process_mm_data_async( *args, **kwargs, ): - mm_data = self.load_mm_data( + mm_data = await self.load_mm_data( prompt=input_text, multimodal_tokens=self.mm_tokens, image_data=image_data, diff --git a/python/sglang/srt/multimodal/processors/points_v15_chat.py b/python/sglang/srt/multimodal/processors/points_v15_chat.py index 7fac7e909159..9bf7490fc1a4 100644 --- a/python/sglang/srt/multimodal/processors/points_v15_chat.py +++ b/python/sglang/srt/multimodal/processors/points_v15_chat.py @@ -26,7 +26,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/qwen3_asr.py b/python/sglang/srt/multimodal/processors/qwen3_asr.py index 31368077f256..8b82334bc73b 100644 --- a/python/sglang/srt/multimodal/processors/qwen3_asr.py +++ b/python/sglang/srt/multimodal/processors/qwen3_asr.py @@ -71,7 +71,7 @@ async def process_mm_data_async( prompt = self._build_transcription_prompt(input_text) - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/qwen_audio.py b/python/sglang/srt/multimodal/processors/qwen_audio.py index 5ca7c957c50e..88f93190361e 100644 --- a/python/sglang/srt/multimodal/processors/qwen_audio.py +++ b/python/sglang/srt/multimodal/processors/qwen_audio.py @@ -87,7 +87,7 @@ async def process_mm_data_async( input_text, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index fb9fd856be0a..99a4b12e69ec 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py +++ b/python/sglang/srt/multimodal/processors/qwen_vl.py @@ -2,7 +2,7 @@ import os import re import time -from typing import List, Union +from typing import List, Optional, Union import numpy as np import torch @@ -24,6 +24,7 @@ Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration, ) +from sglang.srt.models.qwen3_5_mtp import Qwen3_5ForCausalLMMTP from sglang.srt.models.qwen3_omni_moe import Qwen3OmniMoeForConditionalGeneration from sglang.srt.models.qwen3_vl import Qwen3VLForConditionalGeneration from sglang.srt.models.qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration @@ -247,6 +248,7 @@ class QwenVLImageProcessor(SGLangBaseProcessor): Qwen3VLMoeForConditionalGeneration, Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration, + Qwen3_5ForCausalLMMTP, InternS2PreviewForConditionalGeneration, Qwen3OmniMoeForConditionalGeneration, ] @@ -291,7 +293,6 @@ def build_input_ids_with_timestamps( img_token_id = getattr(self, "IM_TOKEN_ID", None) video_token_id = getattr(self, "VIDEO_TOKEN_ID", None) - audio_token_id = getattr(self, "audio_token_id", None) spatial_merge_size = getattr(self, "spatial_merge_size", 1) vision_start_token_id = getattr(self, "vision_start_token_id", None) vision_end_token_id = getattr(self, "vision_end_token_id", None) @@ -310,7 +311,6 @@ def build_input_ids_with_timestamps( img_idx = 0 video_idx = 0 - model_type = getattr(self, "model_type", None) for mm_start_idx, modality in vision_start_indices: modality_list.append(modality) video_tokens = None @@ -374,13 +374,12 @@ def build_input_ids_with_timestamps( return input_ids, offsets, modality_list def compute_mrope_positions(self, input_ids, mm_items): - image_grid_thw = None - video_grid_thw = None - for item in mm_items: - if "image_grid_thw" in item.model_specific_data: - image_grid_thw = item.model_specific_data["image_grid_thw"] - if "video_grid_thw" in item.model_specific_data: - video_grid_thw = item.model_specific_data["video_grid_thw"] + image_grid_thw = self._concat_mm_item_grid( + mm_items, "image_grid_thw", Modality.IMAGE + ) + video_grid_thw = self._concat_mm_item_grid( + mm_items, "video_grid_thw", Modality.VIDEO + ) input_ids_tensor = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0) mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index( @@ -398,6 +397,163 @@ def compute_mrope_positions(self, input_ids, mm_items): ) return mrope_positions.squeeze(1), mrope_position_delta + @staticmethod + def _get_processor_output_value(ret, key): + if ret is None: + return None + return ret.get(key) if hasattr(ret, "get") else getattr(ret, key, None) + + def _get_precomputed_mrope_from_output(self, ret): + mrope_positions = self._get_processor_output_value(ret, "mrope_positions") + mrope_position_delta = self._get_processor_output_value( + ret, "mrope_position_delta" + ) + if mrope_positions is None or mrope_position_delta is None: + return None + + mrope_positions = torch.as_tensor(mrope_positions) + if mrope_positions.ndim == 3: + if mrope_positions.shape[1] != 1: + return None + mrope_positions = mrope_positions.squeeze(1) + if mrope_positions.ndim != 2 or mrope_positions.shape[0] != 3: + return None + + mrope_position_delta = torch.as_tensor(mrope_position_delta) + if mrope_position_delta.ndim <= 1: + mrope_position_delta = mrope_position_delta.reshape(-1, 1) + return mrope_positions, mrope_position_delta + + @staticmethod + def _as_grid_batch(value): + if value is None: + return None + if isinstance(value, torch.Tensor): + return value.unsqueeze(0) if value.ndim == 1 else value + tensor = torch.as_tensor(value, dtype=torch.long) + return tensor.unsqueeze(0) if tensor.ndim == 1 else tensor + + def _compute_image_only_mrope_positions_from_offsets( + self, + input_len: int, + mm_items: List[MultimodalDataItem], + dtype: torch.dtype, + device: torch.device, + ) -> Optional[tuple[torch.Tensor, torch.Tensor]]: + """instead of calling get_rope_index, build mrope position from mm_items.offsets and image_grid_thw of each image + basically a simplified version of get_rope_index for image-only reqs + """ + if self.model_type not in ( + "qwen3_vl", + "qwen3_vl_moe", + "qwen3_5", + "qwen3_5_moe", + "intern_s2_preview", + ): + return None + + image_items = [item for item in mm_items if item.is_image()] + if not image_items or len(image_items) != len(mm_items): + return None + + spatial_merge_size = self.hf_config.vision_config.spatial_merge_size + sorted_items = sorted(image_items, key=lambda item: item.offsets[0][0]) + position_segments = [] + st = 0 + next_pos = 0 + + for item in sorted_items: + if item.offsets is None or len(item.offsets) != 1: + return None + + start, end = item.offsets[0] + if start < st or end >= input_len: + return None + + text_len = start - st + if text_len > 0: + position_segments.append( + torch.arange(text_len, dtype=dtype, device=device) + .view(1, -1) + .expand(3, -1) + + next_pos + ) + next_pos += text_len + + grid = self._as_grid_batch(item.model_specific_data.get("image_grid_thw")) + if grid is None or grid.shape[0] != 1: + return None + t, h, w = [int(x) for x in grid[0].tolist()] + llm_grid_t = t + llm_grid_h = h // spatial_merge_size + llm_grid_w = w // spatial_merge_size + num_image_tokens = llm_grid_t * llm_grid_h * llm_grid_w + if num_image_tokens != end - start + 1: + return None + + t_index = ( + torch.arange(llm_grid_t, dtype=dtype, device=device) + .view(-1, 1) + .expand(llm_grid_t, llm_grid_h * llm_grid_w) + .reshape(-1) + ) + h_index = ( + torch.arange(llm_grid_h, dtype=dtype, device=device) + .view(1, -1, 1) + .expand(llm_grid_t, llm_grid_h, llm_grid_w) + .reshape(-1) + ) + w_index = ( + torch.arange(llm_grid_w, dtype=dtype, device=device) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, llm_grid_w) + .reshape(-1) + ) + position_segments.append( + torch.stack([t_index, h_index, w_index]) + next_pos + ) + next_pos += max(llm_grid_t, llm_grid_h, llm_grid_w) + st = end + 1 + + if st < input_len: + text_len = input_len - st + position_segments.append( + torch.arange(text_len, dtype=dtype, device=device) + .view(1, -1) + .expand(3, -1) + + next_pos + ) + + mrope_positions = torch.cat(position_segments, dim=1).unsqueeze(1) + mrope_position_delta = (mrope_positions.max() + 1 - input_len).reshape(1, 1) + return mrope_positions, mrope_position_delta + + @classmethod + def _concat_mm_item_grid(cls, mm_items: list[MultimodalDataItem], key, modality): + grids = [] + for item in mm_items: + if not item.is_modality(modality): + continue + grid = cls._as_grid_batch(item.model_specific_data.get(key)) + if grid is not None: + grids.append(grid) + if not grids: + return None + if len(grids) == 1: + return grids[0] + return torch.cat(grids, dim=0) + + @classmethod + def _get_grid_from_output_or_items( + cls, ret, mm_items, key, modality, input_data=None + ): + grid = cls._get_processor_output_value(ret, key) + if grid is None: + grid = cls._concat_mm_item_grid(mm_items, key, modality) + if grid is None and input_data and isinstance(input_data[0], dict): + grid = input_data[0].get(key) + return grid + def get_mm_data(self, prompt, embeddings, **kwargs): img_grid_thw = kwargs.get("img_grid_thw", None) video_grid_thw = kwargs.get("video_grid_thw", None) @@ -475,7 +631,6 @@ def get_mm_data(self, prompt, embeddings, **kwargs): embedding_start : embedding_start + num_tokens ] consumed_per_modality[modality] = embedding_start + num_tokens - logger.info(f"Get embedding slice for {modality}, num_tokens={num_tokens}") mm_items.append( MultimodalDataItem( modality=modality, @@ -505,7 +660,7 @@ async def process_mm_data_async( **kwargs, ): entry_time = time.perf_counter() - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, @@ -516,7 +671,7 @@ async def process_mm_data_async( rid = getattr(request_obj, "rid", "anonymous_rid") video_metadata = None - if base_output.videos: + if base_output.videos and not isinstance(base_output.videos[0], dict): videos_processed = [ await preprocess_video(video, video_config=self.video_config) for video in base_output.videos @@ -553,53 +708,88 @@ async def process_mm_data_async( audio_item.feature_attention_mask, dim=1 ) - second_per_grid_ts = getattr(ret, "second_per_grid_ts", None) + second_per_grid_ts = self._get_processor_output_value(ret, "second_per_grid_ts") if second_per_grid_ts is None: - second_per_grid_ts = getattr(ret, "video_second_per_grid", None) + second_per_grid_ts = self._get_processor_output_value( + ret, "video_second_per_grid" + ) process_time = time.perf_counter() input_ids = input_ids.flatten() + base_input_ids = getattr(base_output, "input_ids", None) + if ( + isinstance(base_input_ids, list) + and len(base_input_ids) == input_ids.numel() + ): + # reuse preprocess input if it already carries list of input_ids + input_ids_list = base_input_ids + else: + input_ids_list = input_ids.tolist() - image_grid_thw = None - if hasattr(ret, "image_grid_thw"): - image_grid_thw = ret.image_grid_thw - - if image_grid_thw is None and image_data and isinstance(image_data[0], dict): - image_grid_thw = image_data[0].get("image_grid_thw") + # look for if padded_input_ids already exists before computing + padded_input_ids = self._get_processor_output_value(ret, "padded_input_ids") + if padded_input_ids is None: + padded_input_ids = MultimodalProcessorOutput.build_padded_input_ids( + input_ids_list, mm_items + ) + elif isinstance(padded_input_ids, torch.Tensor): + # reuse existing padded_input_ids + padded_input_ids = padded_input_ids.flatten().tolist() + else: + padded_input_ids = list(padded_input_ids) - video_grid_thw = None - if hasattr(ret, "video_grid_thw"): - video_grid_thw = ret.video_grid_thw + image_grid_thw = self._get_grid_from_output_or_items( + ret, mm_items, "image_grid_thw", Modality.IMAGE, image_data + ) + video_grid_thw = self._get_grid_from_output_or_items( + ret, + mm_items, + "video_grid_thw", + Modality.VIDEO, + request_obj.video_data, + ) - if video_grid_thw is None and request_obj.video_data: - first_video = request_obj.video_data[0] - if isinstance(first_video, dict): - video_grid_thw = first_video.get("video_grid_thw") + mrope_result = self._get_precomputed_mrope_from_output(ret) + if mrope_result is None: + if ( + video_grid_thw is None + and second_per_grid_ts is None + and audio_feature_lengths is None + ): + mrope_result = self._compute_image_only_mrope_positions_from_offsets( + input_len=input_ids.numel(), + mm_items=mm_items, + dtype=input_ids.dtype, + device=input_ids.device, + ) + if mrope_result is None: + mrope_result = MRotaryEmbedding.get_rope_index( + spatial_merge_size=self.hf_config.vision_config.spatial_merge_size, + image_token_id=self.mm_tokens.image_token_id, + video_token_id=self.mm_tokens.video_token_id, + vision_start_token_id=self.vision_start_token_id, + model_type=self.model_type, + tokens_per_second=getattr( + self.hf_config.vision_config, "tokens_per_second", None + ), + # use the expanded token ids + input_ids=input_ids.unsqueeze(0), + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + second_per_grid_ts=second_per_grid_ts, + use_audio_in_video=False, + audio_seqlens=audio_feature_lengths, + audio_token_id=getattr(self.hf_config, "audio_token_id", None), + audio_start_token_id=self.audio_start_token_id, + position_id_per_seconds=getattr( + self.hf_config, "position_id_per_seconds", None + ), + ) - mrope_positions, mrope_position_delta = MRotaryEmbedding.get_rope_index( - spatial_merge_size=self.hf_config.vision_config.spatial_merge_size, - image_token_id=self.mm_tokens.image_token_id, - video_token_id=self.mm_tokens.video_token_id, - vision_start_token_id=self.vision_start_token_id, - model_type=self.model_type, - tokens_per_second=getattr( - self.hf_config.vision_config, "tokens_per_second", None - ), - # use the expanded token ids - input_ids=input_ids.unsqueeze(0), - image_grid_thw=getattr(ret, "image_grid_thw", None), - video_grid_thw=getattr(ret, "video_grid_thw", None), - second_per_grid_ts=second_per_grid_ts, - use_audio_in_video=False, - audio_seqlens=audio_feature_lengths, - audio_token_id=getattr(self.hf_config, "audio_token_id", None), - audio_start_token_id=self.audio_start_token_id, - position_id_per_seconds=getattr( - self.hf_config, "position_id_per_seconds", None - ), - ) - mrope_positions = mrope_positions.squeeze(1) + mrope_positions, mrope_position_delta = mrope_result + if mrope_positions.ndim == 3: + mrope_positions = mrope_positions.squeeze(1) get_rope_index_time = time.perf_counter() logger.debug( f"[QwenVLProcessor Perf] {rid=}, " @@ -611,7 +801,8 @@ async def process_mm_data_async( ) return MultimodalProcessorOutput( - input_ids=input_ids.tolist(), + input_ids=input_ids_list, + padded_input_ids=padded_input_ids, mm_items=mm_items, im_start_id=self.vision_start_token_id, im_end_id=self.vision_end_token_id, diff --git a/python/sglang/srt/multimodal/processors/sarashina2_vision.py b/python/sglang/srt/multimodal/processors/sarashina2_vision.py index c56f969c644d..761c067d87ac 100644 --- a/python/sglang/srt/multimodal/processors/sarashina2_vision.py +++ b/python/sglang/srt/multimodal/processors/sarashina2_vision.py @@ -62,7 +62,7 @@ async def process_mm_data_async( **kwargs, ): """Process image data for Sarashina2Vision model using standard SGLang pattern.""" - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/multimodal/processors/step3_vl.py b/python/sglang/srt/multimodal/processors/step3_vl.py index e31985192ccd..e1e14d16f7a7 100644 --- a/python/sglang/srt/multimodal/processors/step3_vl.py +++ b/python/sglang/srt/multimodal/processors/step3_vl.py @@ -8,6 +8,7 @@ from PIL import Image from torchvision import transforms from torchvision.transforms import InterpolationMode +from torchvision.transforms import functional as F from transformers import BatchFeature, ProcessorMixin, TensorType from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput @@ -20,14 +21,37 @@ MultimodalSpecialTokens, ) -ImageWithPatches = tuple[Image.Image, list[Image.Image], list[int] | None] +Step3Image = Union[Image.Image, torch.Tensor] +ImageWithPatches = tuple[Step3Image, list[Step3Image], list[int] | None] class GPUToTensor(torch.nn.Module): - def forward(self, raw_image: Union[np.ndarray, Image.Image]) -> torch.Tensor: + def forward( + self, raw_image: Union[np.ndarray, Image.Image, torch.Tensor] + ) -> torch.Tensor: + if isinstance(raw_image, torch.Tensor): + image_tensor = raw_image + if image_tensor.ndim != 3: + raise TypeError( + f"Expected CHW image tensor, got shape {tuple(image_tensor.shape)}" + ) + if image_tensor.shape[0] == 1: + image_tensor = image_tensor.repeat(3, 1, 1) + elif image_tensor.shape[0] != 3: + raise TypeError( + f"Expected CHW image tensor with 1 or 3 channels, got shape {tuple(image_tensor.shape)}" + ) + if image_tensor.dtype == torch.uint8: + image_tensor = image_tensor.to(torch.float32).div(255) + elif not image_tensor.is_floating_point(): + image_tensor = image_tensor.to(torch.float32) + return image_tensor.contiguous() if isinstance(raw_image, Image.Image): - return transforms.ToTensor()(raw_image) + image_tensor = transforms.ToTensor()(raw_image) + if torch.cuda.is_available(): + image_tensor = image_tensor.to(torch.device("cuda")) + return image_tensor if raw_image.ndim == 2: raw_image = raw_image[:, :, None].repeat(3, -1) if torch.cuda.is_available(): @@ -91,6 +115,16 @@ def __call__(self, image, is_patch=False): class ImagePatcher: + def get_image_size(self, img: Step3Image) -> tuple[int, int]: + if isinstance(img, Image.Image): + return img.size + if isinstance(img, torch.Tensor): + if img.ndim != 3: + raise TypeError( + f"Expected CHW image tensor, got shape {tuple(img.shape)}" + ) + return int(img.shape[-1]), int(img.shape[-2]) + raise TypeError(f"Unsupported image type: {type(img)}") def determine_window_size(self, long: int, short: int) -> int: if long <= 728: @@ -132,14 +166,16 @@ def slide_window( for box in windows ], (x_num, y_num) - def square_pad(self, img: Image.Image) -> Image.Image: - w, h = img.size + def square_pad(self, img: Step3Image) -> Step3Image: + w, h = self.get_image_size(img) if w == h: return img size = max(w, h) - padded = Image.new(img.mode, (size, size), 0) - padded.paste(img, (0, 0)) - return padded + if isinstance(img, Image.Image): + padded = Image.new(img.mode, (size, size), 0) + padded.paste(img, (0, 0)) + return padded + return torch.nn.functional.pad(img, (0, size - w, 0, size - h), value=0) def get_image_size_for_padding( self, img_width: int, img_height: int @@ -182,9 +218,22 @@ def get_image_size_for_crop( height_new = window_size * h_ratio return int(width_new), int(height_new) - def patch_crop(self, img: Image.Image, i: int, j: int, th: int, tw: int): - target = img.crop((j, i, j + tw, i + th)) - return target + def resize(self, img: Step3Image, size: tuple[int, int]) -> Step3Image: + if isinstance(img, Image.Image): + return img.resize(size, Image.Resampling.BILINEAR) + return F.resize( + img, + [size[1], size[0]], + interpolation=InterpolationMode.BILINEAR, + antialias=True, + ).contiguous() + + def patch_crop( + self, img: Step3Image, i: int, j: int, th: int, tw: int + ) -> Step3Image: + if isinstance(img, Image.Image): + return img.crop((j, i, j + tw, i + th)) + return img[:, i : i + th, j : j + tw].contiguous() def get_num_patches(self, img_width: int, img_height: int) -> tuple[int, int]: img_width, img_height = self.get_image_size_for_padding(img_width, img_height) @@ -212,20 +261,20 @@ def get_num_patches(self, img_width: int, img_height: int) -> tuple[int, int]: return len(center_list), full_rows def __call__( - self, img: Image.Image - ) -> tuple[Image.Image, list[Image.Image], list[bool] | None]: - img_width, img_height = img.size + self, img: Step3Image + ) -> tuple[Step3Image, list[Step3Image], list[bool] | None]: + img_width, img_height = self.get_image_size(img) new_img_width, new_img_height = self.get_image_size_for_padding( img_width, img_height ) if new_img_width != img_width or new_img_height != img_height: img = self.square_pad(img) - img_width, img_height = img.size + img_width, img_height = self.get_image_size(img) new_img_width, new_img_height = self.get_image_size_for_preprocess( img_width, img_height ) - img = img.resize((new_img_width, new_img_height), Image.Resampling.BILINEAR) + img = self.resize(img, (new_img_width, new_img_height)) window_size = self.determine_window_size( max(new_img_height, new_img_width), min(new_img_height, new_img_width) ) @@ -236,9 +285,7 @@ def __call__( new_img_width, new_img_height, window_size ) if (new_img_width, new_img_height) != (img_width, img_height): - img_for_crop = img.resize( - (new_img_width, new_img_height), Image.Resampling.BILINEAR - ) + img_for_crop = self.resize(img, (new_img_width, new_img_height)) else: img_for_crop = img @@ -320,7 +367,7 @@ def _split_images(self, images: list[Image.Image]) -> list[ImageWithPatches]: def _convert_images_to_pixel_values( self, - images: list[Image.Image], + images: list[Step3Image], is_patch: bool = False, ) -> list[torch.Tensor]: return [ @@ -504,7 +551,7 @@ async def process_mm_data_async( *args, **kwargs, ): - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=input_text, image_data=image_data, video_data=request_obj.video_data, diff --git a/python/sglang/srt/multimodal/processors/voxtral.py b/python/sglang/srt/multimodal/processors/voxtral.py index e6dc15321999..7ed9b544c7d2 100644 --- a/python/sglang/srt/multimodal/processors/voxtral.py +++ b/python/sglang/srt/multimodal/processors/voxtral.py @@ -80,7 +80,7 @@ async def process_mm_data_async( # load_mm_data handles async loading, format detection, resampling. # process_and_combine_mm_data cannot be used: HF VoxtralProcessor.__call__ # does not support audio (only apply_chat_template does). - base_output = self.load_mm_data( + base_output = await self.load_mm_data( prompt=prompt_with_placeholders, audio_data=audio_data, multimodal_tokens=self.mm_tokens, diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index d82b7eeb9484..41c6496d3cd4 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -183,6 +183,44 @@ def to_labels(self): return dataclasses.asdict(self) +# Role keys used by ServerArgs.stat_loggers to look up collector overrides. +# Embedded-use callers (e.g. Ray Serve LLM) pass {"scheduler": MyClass, ...} on +# ServerArgs and the five collector instantiation sites pick the right class. +STAT_LOGGER_ROLE_SCHEDULER = "scheduler" +STAT_LOGGER_ROLE_TOKENIZER = "tokenizer" +STAT_LOGGER_ROLE_STORAGE = "storage" +STAT_LOGGER_ROLE_RADIX_CACHE = "radix_cache" +STAT_LOGGER_ROLE_EXPERT_DISPATCH = "expert_dispatch" + + +def resolve_collector_class( + server_args: Optional["ServerArgs"], role: str, default_cls: type +) -> type: + """Return the subclass registered for `role` on `server_args.stat_loggers`, + or `default_cls` if none is registered. Tolerates `server_args=None` and + `stat_loggers=None`.""" + if server_args is None: + return default_cls + stat_loggers = getattr(server_args, "stat_loggers", None) + if not stat_loggers: + return default_cls + return stat_loggers.get(role, default_cls) + + +class _StatLoggerDIMixin: + """Shared DI override hooks for all *MetricsCollector classes. + + Subclasses (e.g. a Ray-backed wrapper) replace these class attributes with + classes that mirror the prometheus_client API but emit through a different + backend. ``None`` keeps the prometheus_client default. + """ + + _counter_cls = None + _gauge_cls = None + _histogram_cls = None + _summary_cls = None + + @dataclass(kw_only=True, frozen=True, slots=True) class SchedulerMetricsCollectorContext: enable_metrics: bool @@ -192,7 +230,7 @@ class SchedulerMetricsCollectorContext: collector: Optional["SchedulerMetricsCollector"] -class SchedulerMetricsCollector: +class SchedulerMetricsCollector(_StatLoggerDIMixin): def __init__( self, @@ -203,7 +241,15 @@ def __init__( server_args: Optional["ServerArgs"] = None, ) -> None: # We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR` - from prometheus_client import Counter, Gauge, Histogram, Summary + from prometheus_client import Counter as _PromCounter + from prometheus_client import Gauge as _PromGauge + from prometheus_client import Histogram as _PromHistogram + from prometheus_client import Summary as _PromSummary + + Counter = self._counter_cls or _PromCounter + Gauge = self._gauge_cls or _PromGauge + Histogram = self._histogram_cls or _PromHistogram + Summary = self._summary_cls or _PromSummary self.labels = labels self.enable_lora = enable_lora @@ -989,7 +1035,10 @@ def init_new( labels["dp_rank"] = dp_rank if server_args.extra_metric_labels: labels.update(server_args.extra_metric_labels) - collector = cls( + scheduler_collector_cls = resolve_collector_class( + server_args, STAT_LOGGER_ROLE_SCHEDULER, cls + ) + collector = scheduler_collector_cls( labels=labels, enable_lora=enable_lora, enable_hierarchical_cache=enable_hierarchical_cache, @@ -1318,7 +1367,7 @@ def emit_constants( ) -class TokenizerMetricsCollector: +class TokenizerMetricsCollector(_StatLoggerDIMixin): def __init__( self, server_args: Optional[ServerArgs] = None, @@ -1328,7 +1377,11 @@ def __init__( bucket_e2e_request_latency: Optional[List[float]] = None, ) -> None: # We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR` - from prometheus_client import Counter, Histogram + from prometheus_client import Counter as _PromCounter + from prometheus_client import Histogram as _PromHistogram + + Counter = self._counter_cls or _PromCounter + Histogram = self._histogram_cls or _PromHistogram self.labels = labels or {} @@ -1634,12 +1687,16 @@ class StorageMetrics: backup_bandwidth: List[float] = field(default_factory=list) -class StorageMetricsCollector: +class StorageMetricsCollector(_StatLoggerDIMixin): def __init__( self, labels: Dict[str, str], ): - from prometheus_client import Counter, Histogram + from prometheus_client import Counter as _PromCounter + from prometheus_client import Histogram as _PromHistogram + + Counter = self._counter_cls or _PromCounter + Histogram = self._histogram_cls or _PromHistogram self.labels = labels @@ -1728,9 +1785,11 @@ def log_storage_metrics(self, storage_metrics: Optional[StorageMetrics] = None): self._log_histogram(self.histogram_backup_bandwidth, v) -class ExpertDispatchCollector: +class ExpertDispatchCollector(_StatLoggerDIMixin): def __init__(self, ep_size: int) -> None: - from prometheus_client import Histogram + from prometheus_client import Histogram as _PromHistogram + + Histogram = self._histogram_cls or _PromHistogram ep_size_buckets = [i for i in range(ep_size)] self.eplb_gpu_physical_count = Histogram( @@ -1741,13 +1800,17 @@ def __init__(self, ep_size: int) -> None: ) -class RadixCacheMetricsCollector: +class RadixCacheMetricsCollector(_StatLoggerDIMixin): def __init__( self, labels: Dict[str, str], ) -> None: # We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR` - from prometheus_client import Counter, Histogram + from prometheus_client import Counter as _PromCounter + from prometheus_client import Histogram as _PromHistogram + + Counter = self._counter_cls or _PromCounter + Histogram = self._histogram_cls or _PromHistogram self.labels = labels diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index f3ea254c0683..da847a4a3b8d 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -272,6 +272,8 @@ ] NSA_CHOICES = DSA_CHOICES # deprecated alias +DSA_TOPK_BACKEND_CHOICES = ["sgl-kernel", "torch", "flashinfer"] + MAMBA_SCHEDULER_STRATEGY_CHOICES = ["auto", "no_buffer", "extra_buffer"] MAMBA_BACKEND_CHOICES = ["triton", "flashinfer"] @@ -485,6 +487,14 @@ class ServerArgs: export_metrics_to_file: bool = False export_metrics_to_file_dir: Optional[str] = None + # Class-level DI for the five *MetricsCollector classes. Maps collector role + # (one of: "scheduler", "tokenizer", "storage", "radix_cache", "expert_dispatch") + # to a subclass of the matching base collector. The five instantiation sites + # read from this map and fall back to the base class. Class-object only (no + # CLI surface) since this exists for embedded use cases that pass a Python + # class directly. Default None preserves existing behavior. + stat_loggers: Optional[Dict[str, type]] = None + # API related api_key: Optional[str] = None admin_api_key: Optional[str] = None @@ -555,6 +565,7 @@ class ServerArgs: dsa_decode_backend: Optional[str] = ( None # auto-detect based on hardware/kv_cache_dtype ) + dsa_topk_backend: str = "sgl-kernel" disable_flashinfer_autotune: bool = False mamba_backend: str = "triton" @@ -758,7 +769,6 @@ class ServerArgs: piecewise_cuda_graph_tokens: Optional[List[int]] = None piecewise_cuda_graph_compiler: str = "eager" torchao_config: str = "" - enable_nan_detection: bool = False enable_p2p_check: bool = False triton_attention_reduce_in_fp32: bool = False triton_attention_num_kv_splits: int = 8 @@ -784,6 +794,7 @@ class ServerArgs: enable_deterministic_inference: bool = False rl_on_policy_target: Optional[str] = None enable_attn_tp_input_scattered: bool = False + disable_attn_tp_gather: bool = False gc_threshold: Optional[List[int]] = None # Context parallelism used in the long sequence prefill phase of DeepSeek v3.2 enable_dsa_prefill_context_parallel: bool = False @@ -1139,14 +1150,6 @@ def _handle_deprecated_args(self): ) self.tool_call_parser = deprecated_tool_call_parsers[self.tool_call_parser] - if self.enable_nan_detection: - logger.warning( - "--enable-nan-detection is deprecated. " - "Use SGLANG_SPEC_NAN_DETECTION=1 and SGLANG_SPEC_OOB_DETECTION=1 instead." - ) - envs.SGLANG_SPEC_NAN_DETECTION.set(True) - envs.SGLANG_SPEC_OOB_DETECTION.set(True) - # Deprecated attention-backend alias: "compressed" -> "dsv4". for attr in ( "attention_backend", @@ -1410,6 +1413,10 @@ def _handle_piecewise_cuda_graph(self): # 18. CUDA Graph debug mode if self.debug_cuda_graph: self.disable_piecewise_cuda_graph = True + # 19. DSA prefill context parallelism (attn_cp_size is set later in + # _handle_model_specific_adjustments, so check the flag directly here) + if self.enable_dsa_prefill_context_parallel: + self.disable_piecewise_cuda_graph = True def _handle_multi_item_scoring(self): """Setup and validate multi-item scoring constraints. @@ -1884,10 +1891,20 @@ def _handle_model_specific_adjustments(self): assert ( self.tp_size <= 8 ), "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues." + # Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP. + # DSACPLayerCommunicator does not all-reduce attention-TP + # partial o_proj outputs before replicated dense FFNs. self.attn_cp_size = self.tp_size // self.dp_size - + self.disable_piecewise_cuda_graph = True logger.warning( - f"Enable Context Parallel opt for deeeseekv3.2-DSA, Setting dp_size == {self.dp_size} and moe_dense_tp_size == {self.moe_dense_tp_size}, ep_size == {self.ep_size}, tp_size == {self.tp_size}, kv_cache_dtype == {self.kv_cache_dtype}, moe_a2a_backend {self.moe_a2a_backend} " + f"Enable DSA Context Parallel opt, " + f"Setting dp_size == {self.dp_size} and " + f"moe_dense_tp_size == {self.moe_dense_tp_size}, " + f"ep_size == {self.ep_size}, " + f"tp_size == {self.tp_size}, " + f"kv_cache_dtype == {self.kv_cache_dtype}, " + f"moe_a2a_backend {self.moe_a2a_backend}, " + f"disable_piecewise_cuda_graph=True" ) else: # Pure TP and partial DP Attention mode is active for DSA, logging a warning @@ -1929,7 +1946,7 @@ def _handle_model_specific_adjustments(self): ), "CP is only supported for prefill when PD disaggregation, please remove --enable-dsa-prefill-context-parallel." else: - # DeepSeek V3/R1/V3.1 + # DeepSeek V3/R1/V3.1 and Kimi K2.5 if not self.disable_piecewise_cuda_graph: logger.info("Piecewise CUDA graph is enabled, use MLA for prefill.") @@ -1944,6 +1961,37 @@ def _handle_model_specific_adjustments(self): "Use trtllm_mla as attention backend on sm100 for DeepseekV3ForCausalLM" ) + # MLA prefill CP auto-config. Mirrors the NSA CP block above + # (minus the in-seq/round-robin mode split, which MLA CP does not support) + if self.enable_prefill_context_parallel and self.use_mla_backend(): + logger.warning( + "MLA prefill context parallel is still experimental. " + "Verified on Hopper with the fa3 backend." + ) + self.enable_dp_attention = True + # TODO(kpham-sgl) Supports moe_dense_tp_size != 1. + self.moe_dense_tp_size = 1 + self.moe_a2a_backend = "deepep" + self.ep_size = self.tp_size + logger.warning( + "For MLA CP, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, batch_size == 1" + ) + # FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP. + # DSACPLayerCommunicator does not all-reduce attention-TP + # partial o_proj outputs before replicated dense FFNs. + self.attn_cp_size = self.tp_size // self.dp_size + self.disable_piecewise_cuda_graph = True + logger.warning( + f"Enable Context Parallel opt for MLA, " + f"Setting dp_size == {self.dp_size} and " + f"attn_cp_size == {self.attn_cp_size}, " + f"moe_dense_tp_size == {self.moe_dense_tp_size}, " + f"ep_size == {self.ep_size}, " + f"tp_size == {self.tp_size}, " + f"moe_a2a_backend {self.moe_a2a_backend}, " + f"disable_piecewise_cuda_graph=True" + ) + # Set moe backend for DeepSeek if is_sm100_supported(): quant_method = get_quantization_config(hf_config) @@ -2295,11 +2343,13 @@ def _handle_model_specific_adjustments(self): ) if is_sm100_supported() and self.moe_runner_backend == "auto": - - self.moe_runner_backend = "flashinfer_trtllm" - logger.info( - "Use flashinfer_trtllm as MoE runner backend on SM100 for Gemma-4 NVFP4" - ) + if self.get_model_config().quantization == "modelopt_fp4": + self.quantization = "modelopt_fp4" + self.moe_runner_backend = "flashinfer_trtllm" + logger.info( + "Use flashinfer_trtllm as MoE runner backend on " + "SM100 for Gemma-4 (modelopt_fp4)" + ) elif model_arch == "MossVLForConditionalGeneration": if self.is_attention_backend_not_set(): self.prefill_attention_backend = "flashinfer" @@ -2460,8 +2510,6 @@ def _handle_model_specific_adjustments(self): ]: self._handle_mamba_radix_cache( model_arch=model_arch, - support_mamba_cache=True, - support_mamba_cache_extra_buffer=False, sm100_default_attention_backend="triton", ) @@ -2474,8 +2522,7 @@ def _handle_model_specific_adjustments(self): if has_mamba: self._handle_mamba_radix_cache( model_arch=model_arch, - support_mamba_cache_extra_buffer=False, - sm100_default_attention_backend="triton", + sm100_default_attention_backend="flashinfer", ) elif model_arch in ["Lfm2ForCausalLM"]: @@ -2560,6 +2607,7 @@ def _handle_mamba_radix_cache( support_mamba_cache: bool = True, support_mamba_cache_extra_buffer: bool = True, sm100_default_attention_backend: str = None, + fallback_attention_backend: str = "triton", ): if ( is_sm100_supported() @@ -2586,7 +2634,7 @@ def _handle_mamba_radix_cache( if self.enable_mamba_extra_buffer(): # extra_buffer if self.disable_radix_cache: raise ValueError( - "mamba extra_buffer is not compatible with --disable-radix-cache " + "mamba extra_buffer is not compatible with --disable-radix-cache. " "Overlap scheduling is already supported with no_buffer + disable_radix_cache. " "Please use --mamba-scheduler-strategy no_buffer instead." ) @@ -2603,11 +2651,7 @@ def _handle_mamba_radix_cache( assert ( self.mamba_track_interval % self.page_size == 0 ), f"mamba_track_interval {self.mamba_track_interval} must be divisible by page_size {self.page_size}" - assert ( - max(FLA_CHUNK_SIZE, self.page_size) - % min(FLA_CHUNK_SIZE, self.page_size) - == 0 - ), f"For SSM models with extra buffer, either FLA_CHUNK_SIZE or page_size must be divisible by the other, got {FLA_CHUNK_SIZE=}, {self.page_size=}" + assert self.mamba_cache_chunk_size is not None elif not self.disable_radix_cache: # no_buffer if self.page_size is not None and self.page_size != 1: logger.warning( @@ -2625,7 +2669,7 @@ def _handle_mamba_radix_cache( if self.attention_backend == "trtllm_mha": logger.warning( "Disabling radix cache since trtllm_mha does not support page_size = 1, which is required by MambaRadixCache. " - "Try to use --attention-backend triton if radix cache is necessary." + f"Try to use --attention-backend {fallback_attention_backend} if radix cache is necessary." ) self.disable_radix_cache = True self.disable_overlap_schedule = False @@ -3098,6 +3142,19 @@ def _handle_linear_attn_backend(self): ) def _handle_context_parallelism(self): + if ( + self.enable_prefill_context_parallel + and self.enable_dsa_prefill_context_parallel + ): + raise ValueError( + "--enable-prefill-context-parallel and " + "--enable-nsa-prefill-context-parallel are mutually " + "exclusive. Use --enable-nsa-prefill-context-parallel for " + "DeepSeek V3.2 (NSA) models and " + "--enable-prefill-context-parallel for MLA-based models " + "(DeepSeek V3/R1, Kimi K2.5) or MHA/GQA-based models." + ) + if self.attn_cp_size > 1: # The tp_size is the world size, not the real tensor parallel size assert ( @@ -3246,22 +3303,6 @@ def _handle_moe_kernel_config(self): self.ep_size == 1 ), "FP8/MXFP8 Cutlass MoE is only supported with ep_size == 1" - # TODO(yuwei): Fix piecewise cuda graph support for bypassed topk MoE backends. - # Exception: GptOssForCausalLM wraps the entire MoE block in its own - # custom op (moe_impl), so bypassed topk is handled inside the op body. - if ( - not self.enforce_piecewise_cuda_graph - and self.moe_runner_backend in ("flashinfer_trtllm", "flashinfer_mxfp4") - and self.get_model_config().hf_config.architectures[0] - != "GptOssForCausalLM" - ): - self.disable_piecewise_cuda_graph = True - logger.info( - f"Piecewise cuda graph is disabled for MoE runner backend " - f"'{self.moe_runner_backend}' (bypassed topk is incompatible " - f"with torch.compile)." - ) - def _handle_a2a_moe(self): if self.enable_deepep_waterfill and self.moe_a2a_backend != "deepep": logger.warning( @@ -5504,6 +5545,15 @@ def add_cli_args(parser: argparse.ArgumentParser): choices=DSA_CHOICES, help="[Deprecated] Use --dsa-decode-backend instead.", ) + parser.add_argument( + "--dsa-topk-backend", + dest="dsa_topk_backend", + default=ServerArgs.dsa_topk_backend, + type=str, + choices=DSA_TOPK_BACKEND_CHOICES, + help="DSA indexer top-k backend. Options: 'sgl-kernel', 'torch', 'flashinfer'. " + "The 'torch' backend currently requires SGLANG_DSA_FUSE_TOPK=false.", + ) parser.add_argument( "--fp8-gemm-backend", type=str, @@ -6566,11 +6616,6 @@ def _nonneg_int(value): default=ServerArgs.torchao_config, help="Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-, fp8wo, fp8dq-per_tensor, fp8dq-per_row", ) - parser.add_argument( - "--enable-nan-detection", - action="store_true", - help="[Deprecated] Use SGLANG_SPEC_NAN_DETECTION=1 and SGLANG_SPEC_OOB_DETECTION=1 instead.", - ) parser.add_argument( "--enable-p2p-check", action="store_true", @@ -6712,6 +6757,18 @@ def _nonneg_int(value): action="store_true", help="Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.", ) + parser.add_argument( + "--disable-attn-tp-gather", + action="store_true", + help="Disable scheduler-side attn_tp_gather (the upstream SP path " + "that pads num_tokens to attn_tp_size and pre-allocates a gathered " + "buffer). Use for models that manage SP scatter/gather at the " + "model level (e.g., perform their own all_gather/reduce_scatter " + "inside attention) and do not consume the upstream gathered_buffer. " + "Without this, the cuda graph runner pads num_tokens to attn_tp_size, " + "which can cause kernel autotuners to select wrong-sized variants " + "at small batches.", + ) parser.add_argument( "--enable-dsa-prefill-context-parallel", dest="enable_dsa_prefill_context_parallel", @@ -7099,7 +7156,12 @@ def from_cli_args(cls, args: argparse.Namespace): args.dp_size = args.data_parallel_size args.ep_size = args.expert_parallel_size - attrs = [attr.name for attr in dataclasses.fields(cls)] + # Some dataclass fields (e.g. stat_loggers) intentionally have no CLI + # surface and won't appear on the argparse Namespace. Skip them so the + # dataclass default applies. + attrs = [ + attr.name for attr in dataclasses.fields(cls) if hasattr(args, attr.name) + ] return cls(**{attr: getattr(args, attr) for attr in attrs}) def url(self, port: Optional[int] = None): @@ -7206,9 +7268,17 @@ def effective_max_speculative_num_draft_tokens(self) -> Optional[int]: @property def mamba_cache_chunk_size(self) -> int: - # For mamba cache with extra buffer, the chunk size is the max of FLA_CHUNK_SIZE and page_size. + # For mamba cache with extra buffer, the chunk size is the max of FLA_CHUNK_SIZE + # (or mamba_chunk_size if it is defined in the model's config) and page_size. # It is used to determine the caching point in a sequence during prefill. - return max(FLA_CHUNK_SIZE, self.page_size) + if not hasattr(self, "_mamba_cache_chunk_size"): + hf_config = self.get_model_config().hf_config + chunk_size = getattr(hf_config, "mamba_chunk_size", FLA_CHUNK_SIZE) + assert ( + max(chunk_size, self.page_size) % min(chunk_size, self.page_size) == 0 + ), f"For SSM models, either chunk_size or page_size must be divisible by the other, got {chunk_size=}, {self.page_size=}" + self._mamba_cache_chunk_size = max(chunk_size, self.page_size) + return self._mamba_cache_chunk_size def check_server_args(self): # Check parallel size constraints @@ -7671,6 +7741,92 @@ def remote_instance_weight_loader_use_transfer_engine(self): else: return False + def describe_kv_events_publisher(self) -> Optional[dict]: + """Return a structured description of this server's KV-event + publisher, or `None` if publishing is disabled / misconfigured. + + This is the wire contract surfaced under the `kv_events` key on + `/server_info` so KV-aware routers (e.g. the SGLang model + gateway) can subscribe per-worker without operator-supplied port + coordination. The router constructs the per-DP-rank SUB endpoint + as ``tcp://:`` for + every rank reported in ``dp_size``. + + Returned descriptor shape: + + { + "publisher": "zmq", + "endpoint_host": "*", # may be a ZMQ wildcard + # ("*", "0.0.0.0", "::"); + # subscribers MUST substitute + # the worker URL's host when + # dialing + "endpoint_port_base": 5557, # base TCP port; per-rank + # port = base + dp_rank + "topic": "", # ZMQ topic prefix on the + # SUB filter (empty = + # subscribe-all) + "block_size": , # subscribers MUST hash + # prompts at this size + "dp_size": , # number of SUB sockets + # to open + } + + Returns ``None`` (i.e. "no publisher to describe") when any of: + + * ``--kv-events-config`` is unset / empty / malformed JSON, + * the configured publisher is ``"null"``, + * ``page_size`` is missing or non-positive (a placeholder + ``block_size`` would cause silent KV-cache misses by hashing + prompts at the wrong granularity on the router side), + * the endpoint is not a routable TCP address (``inproc://`` / + ``ipc://``, missing port, non-integer port, or port outside + ``1..65535``). + + Reuses ``KVEventsConfig.from_cli`` for JSON parsing; the inline + ``rfind(":")`` endpoint split mirrors + ``ZmqEventPublisher.offset_endpoint_port`` rather than adding a + new module-level helper. + """ + # Lazy import so loading ``server_args`` doesn't pull in + # disaggregation / msgspec / zmq at module top level. + from sglang.srt.disaggregation.kv_events import KVEventsConfig + + raw = self.kv_events_config + page_size = self.page_size + if not raw or page_size is None or page_size <= 0: + return None + try: + cfg = KVEventsConfig.from_cli(raw) + except Exception: + # Malformed JSON / schema mismatch. The publisher would + # have failed at server startup; ``/server_info`` must + # keep working, so just report "no publisher" to consumers. + return None + if cfg.publisher == "null" or not cfg.endpoint: + return None + if not cfg.endpoint.startswith("tcp://"): + return None + body = cfg.endpoint[len("tcp://") :] + last_colon = body.rfind(":") + if last_colon < 0: + return None + host = body[:last_colon] + try: + port = int(body[last_colon + 1 :]) + except ValueError: + return None + if not host or not (0 < port < 65536): + return None + return { + "publisher": cfg.publisher, + "endpoint_host": host, + "endpoint_port_base": port, + "topic": cfg.topic, + "block_size": page_size, + "dp_size": self.dp_size, + } + # NOTE: This is a global variable to hold the server args for scheduler. _global_server_args: Optional[ServerArgs] = None diff --git a/python/sglang/srt/speculative/dflash_worker.py b/python/sglang/srt/speculative/dflash_worker.py index 87ddcfe233e5..86cd76bf7ff0 100644 --- a/python/sglang/srt/speculative/dflash_worker.py +++ b/python/sglang/srt/speculative/dflash_worker.py @@ -646,9 +646,6 @@ def _prepare_for_speculative_decoding( seq_lens_sum=seq_lens_sum, seq_lens_cpu=seq_lens_cpu, positions=positions, - req_to_token_pool=self.draft_model_runner.req_to_token_pool, - token_to_kv_pool=self.draft_model_runner.token_to_kv_pool, - attn_backend=self.draft_model_runner.attn_backend, input_embeds=input_embeds, spec_algorithm=SpeculativeAlgorithm.DFLASH, spec_info=draft_spec_info, diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index dc379878b90c..79927e989496 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -24,18 +24,16 @@ ForwardBatch, ForwardMode, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.input_buffers import ForwardInputBuffers from sglang.srt.speculative.eagle_info import EagleDraftInput -from sglang.srt.speculative.spec_utils import ( - maybe_detect_nan, - maybe_detect_oob, -) from sglang.srt.utils import ( require_attn_tp_gather, require_gathered_buffer, require_mlp_sync, require_mlp_tp_gather, ) +from sglang.srt.utils.async_probe import maybe_detect_nan, maybe_detect_oob if TYPE_CHECKING: from sglang.srt.speculative.eagle_worker import EAGLEWorker @@ -332,8 +330,6 @@ def capture_one_batch_size( seq_lens_cpu=seq_lens_cpu, extend_seq_lens=extend_seq_lens, extend_seq_lens_cpu=extend_seq_lens_cpu, - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, out_cache_loc=out_cache_loc, seq_lens_sum=seq_lens.sum().item(), return_logprob=False, @@ -350,15 +346,10 @@ def capture_one_batch_size( ), ) - # Attention backend - self.draft_attn_backend.init_forward_metadata_capture_cuda_graph(forward_batch) - - # Run and capture def run_once(): if self.model_runner.is_hybrid_swa: self.model_runner.token_to_kv_pool.invalidate_loc_cache() - # Clean intermediate result cache for DP attention forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = None set_dp_buffer_len( global_dp_buffer_len, @@ -367,7 +358,6 @@ def run_once(): ) set_is_extend_in_batch(False) - # Backup fields that are modified in-place in `draft_forward`. output_cache_loc_backup = forward_batch.out_cache_loc hidden_states_backup = forward_batch.spec_info.hidden_states @@ -378,13 +368,15 @@ def run_once(): forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1) return ret - self.deepep_adapter.capture(is_extend_in_batch=False) - - self._capture_init(run_once) - - out = self._capture_graph( - graph, get_global_graph_memory_pool(), stream, run_once - ) + with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)): + self.draft_attn_backend.init_forward_metadata_capture_cuda_graph( + forward_batch + ) + self.deepep_adapter.capture(is_extend_in_batch=False) + self._capture_init(run_once) + out = self._capture_graph( + graph, get_global_graph_memory_pool(), stream, run_once + ) set_global_graph_memory_pool(graph.pool()) return graph, out diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index 23e79648f7c2..ad17631bcc88 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -25,6 +25,7 @@ ForwardBatch, ForwardMode, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.input_buffers import ForwardInputBuffers from sglang.srt.speculative.eagle_info import EagleDraftExtendInput from sglang.srt.speculative.spec_utils import fast_topk @@ -352,8 +353,6 @@ def capture_one_batch_size(self, bs: int, forward: Callable, stream_idx: int = 0 num_accept_tokens=num_accept_tokens, ) - self.deepep_adapter.capture(is_extend_in_batch=True) - # Forward batch forward_batch = ForwardBatch( forward_mode=self.forward_mode, @@ -365,8 +364,6 @@ def capture_one_batch_size(self, bs: int, forward: Callable, stream_idx: int = 0 next_token_logits_buffer=next_token_logits_buffer, extend_seq_lens=extend_seq_lens, extend_seq_lens_cpu=extend_seq_lens_cpu, - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, out_cache_loc=out_cache_loc, seq_lens_sum=seq_lens.sum().item(), return_logprob=False, @@ -379,23 +376,10 @@ def capture_one_batch_size(self, bs: int, forward: Callable, stream_idx: int = 0 spec_algorithm=self.model_runner.spec_algorithm, spec_info=spec_info, capture_hidden_mode=CaptureHiddenMode.LAST, - attn_backend=self.draft_extend_attn_backend, padded_static_len=self.padded_static_len, ) - self.draft_extend_attn_backend.init_forward_metadata_capture_cuda_graph( - bs=bs, - num_tokens=num_tokens, - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - encoder_lens=None, - forward_mode=self.forward_mode, - spec_info=spec_info, - ) - - # Run and capture def run_once(): - # model.forward() bypasses _forward_raw(), so invalidate manually. if self.model_runner.is_hybrid_swa: self.model_runner.token_to_kv_pool.invalidate_loc_cache() @@ -424,11 +408,23 @@ def run_once(): forward_batch.spec_info.hidden_states = hidden_states_backup return ret - self._capture_init(run_once) - - out = self._capture_graph( - graph, get_global_graph_memory_pool(), stream, run_once - ) + with forward_context( + ForwardContext(attn_backend=self.draft_extend_attn_backend) + ): + self.draft_extend_attn_backend.init_forward_metadata_capture_cuda_graph( + bs=bs, + num_tokens=num_tokens, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + encoder_lens=None, + forward_mode=self.forward_mode, + spec_info=spec_info, + ) + self.deepep_adapter.capture(is_extend_in_batch=True) + self._capture_init(run_once) + out = self._capture_graph( + graph, get_global_graph_memory_pool(), stream, run_once + ) set_global_graph_memory_pool(graph.pool()) return graph, out @@ -458,6 +454,9 @@ def replay(self, forward_batch: ForwardBatch): buffers.seq_lens.fill_(self.seq_len_fill_value) buffers.out_cache_loc.zero_() buffers.positions.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init). + buffers.req_pool_indices.zero_() buffers.num_correct_drafts.fill_(self.num_tokens_per_bs) buffers.num_accept_tokens.fill_(self.num_tokens_per_bs) buffers.extend_seq_lens.fill_(self.num_tokens_per_bs) diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index bcdeaf0693c1..106423b0a266 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -16,7 +16,6 @@ ) from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.sampler import apply_custom_logit_processor -from sglang.srt.managers.overlap_utils import FutureIndices from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.common import ( @@ -45,6 +44,7 @@ get_target_cache_loc, ) from sglang.srt.utils import is_cuda, is_musa, next_power_of_2 +from sglang.srt.utils.async_probe import maybe_detect_nan, maybe_detect_oob if is_cuda() or is_musa(): from sgl_kernel import ( @@ -126,6 +126,12 @@ def prepare_for_verify(self, batch: ScheduleBatch, page_size: int): return batch.input_ids = self.draft_token + maybe_detect_oob( + batch.input_ids, + 0, + batch.model_config.vocab_size, + "eagle prepare_for_verify input_ids", + ) if page_size == 1: batch.out_cache_loc = alloc_token_slots( @@ -350,12 +356,14 @@ def verify( target_probs = F.softmax( logits_output.next_token_logits / expanded_temperature, dim=-1 ) # (bs * draft_token_num, vocab_size) + maybe_detect_nan(target_probs, "verify: target_probs after softmax") target_probs = top_k_renorm_prob( target_probs, torch.repeat_interleave( sampling_info.top_ks, self.draft_token_num, dim=0 ), ) # (bs * draft_token_num, vocab_size) + maybe_detect_nan(target_probs, "verify: target_probs after top_k_renorm") if sampling_info.need_top_p_sampling: target_probs = top_p_renorm_prob( target_probs, @@ -363,6 +371,9 @@ def verify( sampling_info.top_ps, self.draft_token_num, dim=0 ), ) + maybe_detect_nan( + target_probs, "verify: target_probs after top_p_renorm" + ) target_probs = target_probs.reshape(bs, self.draft_token_num, -1) draft_probs = torch.zeros( @@ -419,6 +430,21 @@ def verify( spec_steps=self.spec_steps, ) + # accept_index values index batch.out_cache_loc (size = bs * draft_token_num); + # -1 is the reject sentinel. + maybe_detect_oob( + accept_index, + -1, + bs * self.draft_token_num, + "eagle verify accept_index post-sampling", + ) + maybe_detect_oob( + num_correct_drafts, + 0, + self.draft_token_num + 1, + "eagle verify num_correct_drafts post-sampling", + ) + unfinished_index = [] unfinished_accept_index = [] accept_index_cpu = accept_index.tolist() @@ -476,6 +502,12 @@ def verify( # TODO: fuse them accept_index = accept_index[accept_index != -1] accept_tokens = predict[accept_index] + maybe_detect_oob( + accept_tokens, + 0, + batch.model_config.vocab_size, + "eagle verify accept_tokens", + ) evict_mask = torch.full_like(self.draft_token, True, dtype=torch.bool) evict_mask[accept_index] = False num_correct_drafts_cpu = num_correct_drafts.cpu() @@ -693,9 +725,8 @@ class EagleDraftInput(SpecInput, EagleDraftInputV2Mixin): num_tokens_per_req: int = -1 num_tokens_for_logprob_per_req: int = -1 - # V2 overlap worker only - future_indices: Optional[FutureIndices] = None - new_seq_lens: Optional[torch.Tensor] = None + # V2 overlap worker only: req_pool_indices used as buf slot keys. + future_indices: Optional[torch.Tensor] = None # V2 reuses `EagleDraftInput` across phases (V1 has a separate # `EagleDraftExtendInput` for these). Set during V2's draft-extend. num_correct_drafts: Optional[torch.Tensor] = None @@ -742,12 +773,11 @@ def create_idle_input( topk_p=torch.empty((0, topk), device=device, dtype=torch.float32), topk_index=torch.empty((0, topk), device=device, dtype=torch.int64), capture_hidden_mode=capture_hidden_mode, - new_seq_lens=torch.empty((0,), device=device, dtype=torch.int32), ) def filter_batch(self, new_indices: torch.Tensor, has_been_filtered: bool = True): if self.future_indices is not None: - self.future_indices.indices = self.future_indices.indices[new_indices] + self.future_indices = self.future_indices[new_indices] return strict_check = envs.SGLANG_SPEC_ENABLE_STRICT_FILTER_CHECK.get() @@ -777,10 +807,8 @@ def filter_batch(self, new_indices: torch.Tensor, has_been_filtered: bool = True def merge_batch(self, spec_info: "EagleDraftInput"): if self.future_indices is not None: assert spec_info.future_indices is not None - self.future_indices = FutureIndices( - indices=torch.cat( - [self.future_indices.indices, spec_info.future_indices.indices] - ) + self.future_indices = torch.cat( + [self.future_indices, spec_info.future_indices] ) return diff --git a/python/sglang/srt/speculative/eagle_info_v2.py b/python/sglang/srt/speculative/eagle_info_v2.py index 52d4ad4aabe5..390e2e78d928 100644 --- a/python/sglang/srt/speculative/eagle_info_v2.py +++ b/python/sglang/srt/speculative/eagle_info_v2.py @@ -38,6 +38,7 @@ SIMULATE_ACC_LEN, generate_simulated_accept_index, ) +from sglang.srt.utils.async_probe import maybe_detect_nan, maybe_detect_oob from sglang.srt.utils.common import is_cuda, is_hip, is_musa, is_npu, next_power_of_2 _is_cuda = is_cuda() @@ -226,9 +227,12 @@ def prepare_for_extend_to_fill_draft_kvcache( batch.spec_info = self batch.input_ids = predict - batch.seq_lens = batch.seq_lens + num_draft_tokens - batch.seq_lens_cpu = batch.seq_lens_cpu + num_draft_tokens - batch.seq_lens_sum = int(batch.seq_lens_cpu.sum()) + maybe_detect_oob( + batch.input_ids, + 0, + batch.model_config.vocab_size, + "v2 prepare_for_extend_to_fill_draft_kvcache input_ids", + ) batch.extend_lens = [num_draft_tokens for _ in range(len(batch.seq_lens))] batch.prefix_lens = seq_lens_cpu_.tolist() batch.extend_num_tokens = extend_num_tokens @@ -244,6 +248,11 @@ def prepare_for_extend_to_fill_draft_kvcache( ) batch.capture_hidden_mode = capture_mode forward_batch = ForwardBatch.init_new(batch, draft_model_runner) + # Forward sees post-write length (draft extend writes num_draft_tokens + # slots); mutation stays on forward_batch to preserve SB.seq_lens. + forward_batch.seq_lens = forward_batch.seq_lens + num_draft_tokens + forward_batch.seq_lens_cpu = forward_batch.seq_lens_cpu + num_draft_tokens + forward_batch.seq_lens_sum = int(forward_batch.seq_lens_cpu.sum()) can_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run(forward_batch) if not batch.forward_mode.is_idle() and not can_cuda_graph: draft_model_runner.attn_backend.init_forward_metadata(forward_batch) @@ -262,6 +271,12 @@ def prepare_for_v2_verify( # Assign cache locations bs = len(batch.req_pool_indices) batch.input_ids = self.draft_token + maybe_detect_oob( + batch.input_ids, + 0, + batch.model_config.vocab_size, + "v2 prepare_for_verify input_ids", + ) device = batch.input_ids.device batch.out_cache_loc = assign_extend_cache_locs_func( req_pool_indices=batch.req_pool_indices, @@ -304,11 +319,9 @@ def prepare_for_v2_verify( ) if can_run_cuda_graph: target_worker.model_runner.graph_runner.replay_prepare(verify_forward_batch) - else: - if not batch.forward_mode.is_idle(): - target_worker.model_runner.attn_backend.init_forward_metadata( - verify_forward_batch - ) + # Non-cuda-graph: defer init to forward_extend, which runs after + # `_forward_raw -> prepare_mlp_sync_batch` pads the batch. Initing + # here would use pre-pad shapes and trip DSv4 indexer shape match. return verify_forward_batch, can_run_cuda_graph @@ -398,18 +411,21 @@ def sample( target_probs = F.softmax( next_token_logits / expanded_temperature, dim=-1 ) # (bs * num_draft_tokens, vocab_size) + maybe_detect_nan(target_probs, "v2 verify: target_probs after softmax") target_probs = top_k_renorm_prob( target_probs, torch.repeat_interleave( sampling_info.top_ks, self.draft_token_num, dim=0 ), ) # (bs * num_draft_tokens, vocab_size) + maybe_detect_nan(target_probs, "v2 verify: target_probs after top_k_renorm") target_probs = top_p_renorm_prob( target_probs, torch.repeat_interleave( sampling_info.top_ps, self.draft_token_num, dim=0 ), ) + maybe_detect_nan(target_probs, "v2 verify: target_probs after top_p_renorm") target_probs = target_probs.reshape(bs, self.draft_token_num, -1) draft_probs = torch.zeros_like(target_probs) diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index 14f8fe34049b..a350bb2c4f1f 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -23,6 +23,30 @@ ) +def per_step_draft_out_cache_loc( + out_cache_loc: torch.Tensor, + batch_size: int, + topk: int, + num_steps: int, +) -> torch.Tensor: + """Per-step slice of the multi-step EAGLE draft out_cache_loc buffer. + + Single source of truth for the layout shared by EagleWorkerV2.draft_forward + (per-step write target) and DeepseekV4AttnBackend (per-step compression + write target baked into metadata). + """ + expected = batch_size * topk * num_steps + assert out_cache_loc.shape[0] == expected, ( + f"out_cache_loc.shape[0]={out_cache_loc.shape[0]} != " + f"batch_size * topk * num_steps = {batch_size}*{topk}*{num_steps}={expected}" + ) + return ( + out_cache_loc.view(batch_size, topk, num_steps) + .permute(2, 0, 1) + .reshape(num_steps, -1) + ) + + def apply_eagle_prefill_input_rotation( batch: ScheduleBatch, next_token_ids: torch.Tensor ) -> None: diff --git a/python/sglang/srt/speculative/eagle_worker.py b/python/sglang/srt/speculative/eagle_worker.py index 88488bf883cd..1bf696791641 100644 --- a/python/sglang/srt/speculative/eagle_worker.py +++ b/python/sglang/srt/speculative/eagle_worker.py @@ -1,3 +1,4 @@ +import contextlib import logging import time from contextlib import contextmanager @@ -30,6 +31,7 @@ ForwardBatch, ForwardMode, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.observability.req_time_stats import set_time_batch from sglang.srt.observability.trace import get_global_tracing_enabled from sglang.srt.server_args import ServerArgs @@ -63,8 +65,6 @@ generate_token_bitmask, get_last_loc_large_page_size_large_top_k, load_token_map, - maybe_detect_nan, - maybe_detect_oob, select_top_k_tokens, ) from sglang.srt.utils import ( @@ -77,6 +77,11 @@ log_info_on_rank0, next_power_of_2, ) +from sglang.srt.utils.async_probe import ( + maybe_detect_inf, + maybe_detect_nan, + maybe_detect_oob, +) from sglang.srt.utils.patch_torch import monkey_patch_torch_reductions _is_npu = is_npu() @@ -881,14 +886,20 @@ def draft_forward(self, forward_batch: ForwardBatch): ): out_cache_loc = out_cache_loc.contiguous() forward_batch.out_cache_loc = out_cache_loc[i] - forward_batch.attn_backend = self.draft_attn_backend.attn_backends[i] spec_info.hidden_states = hidden_states - # Run forward - logits_output = self.draft_model_runner.forward( - forward_batch, skip_attn_backend_init=True - ).logits_output + # Run forward under a per-step ForwardContext so the model layer + # reads attn_backends[i] for the i-th draft step. ``_forward_raw`` + # is no-op for the attn_backend half when a context is already + # active, so this outer wrap is what reaches RadixAttention. + with forward_context( + ForwardContext(attn_backend=self.draft_attn_backend.attn_backends[i]) + ): + logits_output = self.draft_model_runner.forward( + forward_batch, skip_attn_backend_init=True + ).logits_output maybe_detect_nan(logits_output.next_token_logits, f"draft_forward step {i}") + maybe_detect_inf(logits_output.next_token_logits, f"draft_forward step {i}") probs = torch.softmax(logits_output.next_token_logits, dim=-1) topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) maybe_detect_oob( @@ -900,6 +911,8 @@ def draft_forward(self, forward_batch: ForwardBatch): if self.hot_token_id is not None: topk_index = self.hot_token_id[topk_index] hidden_states = logits_output.hidden_states + maybe_detect_nan(hidden_states, f"draft_forward step {i}: hidden_states") + maybe_detect_inf(hidden_states, f"draft_forward step {i}: hidden_states") forward_batch.positions.add_(1) parent_list, top_scores_index, draft_tokens = organize_draft_results( @@ -962,6 +975,7 @@ def verify(self, batch: ScheduleBatch): batch.sampling_info.vocab_mask = None maybe_detect_nan(logits_output.next_token_logits, "verify: target model logits") + maybe_detect_inf(logits_output.next_token_logits, "verify: target model logits") spec_info.hidden_states = logits_output.hidden_states res: EagleVerifyOutput = spec_info.verify( @@ -1197,16 +1211,23 @@ def forward_draft_extend_after_decode( hidden_states = logits_output.hidden_states else: forward_batch.can_run_dp_cuda_graph = False + attn_backend = None if not forward_batch.forward_mode.is_idle(): attn_backend = ( self.draft_extend_attn_backend or self.draft_model_runner.attn_backend ) attn_backend.init_forward_metadata(forward_batch) - forward_batch.attn_backend = attn_backend - logits_output = self.draft_model_runner.forward( - forward_batch, skip_attn_backend_init=True - ).logits_output + # Publish the chosen backend via ForwardContext so model code + # picks it up for this forward (no runner-attr mutation). + if attn_backend is not None: + ctx_mgr = forward_context(ForwardContext(attn_backend=attn_backend)) + else: + ctx_mgr = contextlib.nullcontext() + with ctx_mgr: + logits_output = self.draft_model_runner.forward( + forward_batch, skip_attn_backend_init=True + ).logits_output # Non-cuda-graph path: compute topk_p / topk_index inline. probs = torch.softmax(logits_output.next_token_logits, dim=-1) topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 6e81c5c25ee4..bb1b51686163 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -33,6 +33,7 @@ from sglang.srt.managers.tp_worker import TpModelWorker from sglang.srt.model_executor.cuda_graph_runner import CudaGraphRunner from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardBatch +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.adaptive_runtime_state import ( AdaptiveController, @@ -52,18 +53,25 @@ fill_accepted_out_cache_loc, fill_bonus_tokens, ) -from sglang.srt.speculative.eagle_utils import TreeMaskMode, build_tree_kernel_efficient +from sglang.srt.speculative.eagle_utils import ( + TreeMaskMode, + build_tree_kernel_efficient, + per_step_draft_out_cache_loc, +) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_utils import ( draft_tp_context, generate_token_bitmask, load_token_map, - maybe_detect_nan, - maybe_detect_oob, record_stream_each, record_stream_for_v2_verify, select_top_k_tokens, ) +from sglang.srt.utils.async_probe import ( + maybe_detect_inf, + maybe_detect_nan, + maybe_detect_oob, +) from sglang.srt.utils.common import ( MultiprocessingSerializer, empty_context, @@ -437,11 +445,11 @@ def draft_forward(self, forward_batch: ForwardBatch): if self.hot_token_id is not None: topk_index = self.hot_token_id[topk_index] - out_cache_loc = out_cache_loc.reshape( - forward_batch.batch_size, self.topk, self.speculative_num_steps - ) - out_cache_loc = out_cache_loc.permute((2, 0, 1)).reshape( - self.speculative_num_steps, -1 + out_cache_loc = per_step_draft_out_cache_loc( + out_cache_loc, + forward_batch.batch_size, + self.topk, + self.speculative_num_steps, ) # Return values @@ -466,14 +474,19 @@ def draft_forward(self, forward_batch: ForwardBatch): # Set inputs forward_batch.input_ids = input_ids forward_batch.out_cache_loc = out_cache_loc[i] - forward_batch.attn_backend = self.draft_attn_backend.attn_backends[i] spec_info.hidden_states = hidden_states - # Run forward - logits_output = self.draft_runner.forward( - forward_batch, skip_attn_backend_init=True - ).logits_output + # Run forward under a per-step ForwardContext so the model layer + # reads attn_backends[i] for the i-th draft step. ``_forward_raw`` + # honors the outer context and does not override. + with forward_context( + ForwardContext(attn_backend=self.draft_attn_backend.attn_backends[i]) + ): + logits_output = self.draft_runner.forward( + forward_batch, skip_attn_backend_init=True + ).logits_output maybe_detect_nan(logits_output.next_token_logits, f"draft_forward step {i}") + maybe_detect_inf(logits_output.next_token_logits, f"draft_forward step {i}") probs = torch.softmax(logits_output.next_token_logits, dim=-1) topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) maybe_detect_oob( @@ -547,7 +560,6 @@ def _draft_extend_for_prefill( next_draft_input = EagleDraftInput( hidden_states=target_hidden_states, bonus_tokens=next_token_ids, - new_seq_lens=batch.seq_lens, # draft mode is same with decode mode, only 1 token per req num_tokens_per_req=1, num_tokens_for_logprob_per_req=1, @@ -570,6 +582,7 @@ def _draft_extend_for_prefill( forward_batch.mm_input_embeds = mm_input_embeds logits_output = self.draft_runner.forward(forward_batch).logits_output maybe_detect_nan(logits_output.next_token_logits, "draft_extend_for_prefill") + maybe_detect_inf(logits_output.next_token_logits, "draft_extend_for_prefill") # Update spec_info for the next draft step probs = torch.softmax(logits_output.next_token_logits, dim=-1) @@ -634,6 +647,10 @@ def _draft_extend_for_decode( draft_logits_output.next_token_logits, f"draft_extend_for_decode (cuda_graph={can_cuda_graph})", ) + maybe_detect_inf( + draft_logits_output.next_token_logits, + f"draft_extend_for_decode (cuda_graph={can_cuda_graph})", + ) # Reorganize the spec info for the next batch draft_logits_output.next_token_logits = draft_logits_output.next_token_logits[ @@ -756,7 +773,7 @@ def clear_cache_pool(self): # allocator and kv cache pool are shared with target worker, which are cleared in scheduler pass - def forward_batch_generation(self, batch: ScheduleBatch, on_verify_complete=None): + def forward_batch_generation(self, batch: ScheduleBatch, on_publish=None): if batch.forward_mode.is_extend() or batch.is_extend_in_batch: # Target prefill target_capture_mode = ( @@ -767,9 +784,12 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_verify_complete=None batch.capture_hidden_mode = target_capture_mode batch_output = self.target_worker.forward_batch_generation(batch) + # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. + # Extend processed L prompt tokens; next verify iter expects same L. + batch_output.new_seq_lens = batch.seq_lens # Publish before draft_extend so the fence is at target-end. - if on_verify_complete is not None: - on_verify_complete(batch.seq_lens) + if on_publish is not None: + on_publish(batch_output.new_seq_lens) # Draft prefill with ( @@ -814,8 +834,8 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_verify_complete=None batch.spec_info = verify_input batch_output = self.verify(batch) # Publish before draft_extend so the fence is at verify-end. - if on_verify_complete is not None: - on_verify_complete(batch_output.next_draft_input.new_seq_lens) + if on_publish is not None: + on_publish(batch_output.new_seq_lens) with ( self.draft_worker.draft_tp_context( self.draft_worker.draft_runner.tp_group @@ -1028,12 +1048,14 @@ def verify(self, batch: ScheduleBatch): verify_input.retrieve_next_token.shape ).cpu() - # Run target verify batch in the main compute stream (GPU compute) + # Run target verify batch in the main compute stream (GPU compute). + # Only skip metadata init when cuda-graph already ran replay_prepare; + # the non-cuda-graph path needs forward_extend's init (post-pad). forward_batch_output = self.target_worker.forward_batch_generation( batch=None, forward_batch=verify_forward_batch, is_verify=True, - skip_attn_backend_init=True, + skip_attn_backend_init=can_run_cuda_graph, ) logits_output = forward_batch_output.logits_output @@ -1059,6 +1081,7 @@ def verify(self, batch: ScheduleBatch): # Sample maybe_detect_nan(logits_output.next_token_logits, "verify: target model logits") + maybe_detect_inf(logits_output.next_token_logits, "verify: target model logits") ( predict, accept_lens, @@ -1092,9 +1115,7 @@ def verify(self, batch: ScheduleBatch): batch, logits_output, predict, accept_index, self.speculative_num_steps ) - next_draft_input = EagleDraftInput( - bonus_tokens=bonus_tokens, new_seq_lens=new_seq_lens - ) + next_draft_input = EagleDraftInput(bonus_tokens=bonus_tokens) # verify_forward_batch transitively holds verify-time GPU tensors # (draft_token / out_cache_loc / ...) that must outlive the imminent @@ -1107,6 +1128,7 @@ def verify(self, batch: ScheduleBatch): speculative_num_draft_tokens=self.speculative_num_draft_tokens, next_draft_input=next_draft_input, accept_lens=accept_lens, + new_seq_lens=new_seq_lens, routed_experts_output=forward_batch_output.routed_experts_output, indexer_topk_output=forward_batch_output.indexer_topk_output, extra_keep_alive_refs=[verify_forward_batch], @@ -1192,6 +1214,14 @@ def move_accepted_tokens_to_target_kvcache( bs = len(batch.seq_lens) size = bs * self.speculative_num_draft_tokens + # fill_accepted_out_cache_loc reads out_cache_loc[accept_index]; -1 sentinel ok. + maybe_detect_oob( + accept_index, + -1, + batch.out_cache_loc.size(0), + "eagle v2 move_accepted_tokens accept_index", + ) + tgt_cache_loc = torch.zeros( size, dtype=torch.int64, diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py index da69b3cbd333..fd331dfde691 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py @@ -23,6 +23,7 @@ ForwardBatch, ForwardMode, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.input_buffers import ForwardInputBuffers from sglang.srt.speculative.frozen_kv_mtp_info import FrozenKVMTPDraftInput from sglang.srt.utils import ( @@ -266,9 +267,6 @@ def capture_one_batch_size( req_pool_indices=req_pool_indices, seq_lens=seq_lens, seq_lens_cpu=seq_lens_cpu, - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.frozen_kv_mtp_worker.kv_context.target_token_to_kv_pool, - attn_backend=self.draft_attn_backend, out_cache_loc=None, seq_lens_sum=seq_lens.sum().item(), return_logprob=False, @@ -283,10 +281,6 @@ def capture_one_batch_size( capture_hidden_mode=CaptureHiddenMode.LAST, ) - self.frozen_kv_mtp_worker._init_frozen_kv_metadata_capture_cuda_graph( - forward_batch - ) - def run_once(): if self.model_runner.is_hybrid_swa: self.model_runner.token_to_kv_pool.invalidate_loc_cache() @@ -306,11 +300,25 @@ def run_once(): forward_batch.spec_info.hidden_states = hidden_states_backup return ret - self.deepep_adapter.capture(is_extend_in_batch=False) - self._capture_init(run_once) - out = self._capture_graph( - graph, get_global_graph_memory_pool(), stream, run_once - ) + # Swap the draft backend's token_to_kv_pool to the frozen target pool + # for the capture; the single backend-attr swap is seen by both + # ``get_token_to_kv_pool()`` (via ``get_attn_backend()``) and the + # backend's own reads. + target_pool = self.frozen_kv_mtp_worker.kv_context.target_token_to_kv_pool + saved_backend_pool = self.draft_attn_backend.token_to_kv_pool + self.draft_attn_backend.token_to_kv_pool = target_pool + try: + with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)): + self.frozen_kv_mtp_worker._init_frozen_kv_metadata_capture_cuda_graph( + forward_batch + ) + self.deepep_adapter.capture(is_extend_in_batch=False) + self._capture_init(run_once) + out = self._capture_graph( + graph, get_global_graph_memory_pool(), stream, run_once + ) + finally: + self.draft_attn_backend.token_to_kv_pool = saved_backend_pool set_global_graph_memory_pool(graph.pool()) return graph, out @@ -344,6 +352,9 @@ def replay(self, forward_batch: ForwardBatch): if bs != raw_bs: buffers.seq_lens.fill_(self.seq_len_fill_value) buffers.positions.zero_() + # Pair with seq_lens fill: padded rows must point at reserved + # req_pool slot 0 (req_to_token[0, :] is all zeros from init). + buffers.req_pool_indices.zero_() num_tokens = expanded_bs buffers.seq_lens[:raw_expanded_bs].copy_(forward_batch.seq_lens) diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_utils.py b/python/sglang/srt/speculative/frozen_kv_mtp_utils.py index 043d8b63f5a9..dbd63c2e444c 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_utils.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_utils.py @@ -14,7 +14,7 @@ from __future__ import annotations from contextlib import contextmanager -from typing import Tuple +from typing import TYPE_CHECKING, Tuple import torch @@ -28,39 +28,66 @@ ) from sglang.srt.speculative.spec_utils import fast_topk +if TYPE_CHECKING: + from sglang.srt.layers.attention.base_attn_backend import AttentionBackend + @contextmanager -def frozen_kv_target_view(forward_batch: ForwardBatch, kv_context: FrozenKVMTPContext): - """Build attention metadata against committed target-prefix geometry.""" +def frozen_kv_target_view( + forward_batch: ForwardBatch, + kv_context: FrozenKVMTPContext, + draft_attn_backend: "AttentionBackend", +): + """Build attention metadata against committed target-prefix geometry. + + Swaps ``draft_attn_backend.token_to_kv_pool`` to the frozen target pool + so any helper that reads ``get_token_to_kv_pool()`` during metadata init + sees the frozen target pool. Pool refs are derived from + ``get_attn_backend().token_to_kv_pool`` — the single backend-attribute + swap is seen by both readers (``get_token_to_kv_pool()`` and the + backend's own ``self.token_to_kv_pool``). + """ if kv_context is None: raise RuntimeError( "Frozen-KV MTP target view called before the model was bound; " "bind the frozen KV context first." ) saved_spec_info = forward_batch.spec_info - saved_kv_pool = forward_batch.token_to_kv_pool forward_batch.spec_info = None - forward_batch.token_to_kv_pool = kv_context.target_token_to_kv_pool + saved_backend_pool = draft_attn_backend.token_to_kv_pool + draft_attn_backend.token_to_kv_pool = kv_context.target_token_to_kv_pool try: yield finally: forward_batch.spec_info = saved_spec_info - forward_batch.token_to_kv_pool = saved_kv_pool + draft_attn_backend.token_to_kv_pool = saved_backend_pool @contextmanager -def target_kv_pool_view(forward_batch: ForwardBatch, kv_context: FrozenKVMTPContext): +def target_kv_pool_view( + forward_batch: ForwardBatch, + kv_context: FrozenKVMTPContext, + draft_attn_backend: "AttentionBackend", +): + """Run the draft model's forward with the target's frozen KV pool. + + Swaps ``draft_attn_backend.token_to_kv_pool`` to the frozen target pool. + The single backend-attribute swap is seen by both readers — + ``get_token_to_kv_pool()`` (because it resolves through + ``get_attn_backend()``) and the backend's own ``self.token_to_kv_pool`` + reads (because ``self is draft_attn_backend``). + """ if kv_context is None: raise RuntimeError( "Frozen-KV MTP target KV pool view called before the model was bound; " "bind the frozen KV context first." ) - saved_kv_pool = forward_batch.token_to_kv_pool - forward_batch.token_to_kv_pool = kv_context.target_token_to_kv_pool + saved_backend_pool = draft_attn_backend.token_to_kv_pool + draft_attn_backend.token_to_kv_pool = kv_context.target_token_to_kv_pool try: yield finally: - forward_batch.token_to_kv_pool = saved_kv_pool + draft_attn_backend.token_to_kv_pool = saved_backend_pool def set_frozen_kv_positions(forward_batch: ForwardBatch, topk: int) -> None: diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_worker.py b/python/sglang/srt/speculative/frozen_kv_mtp_worker.py index 6e4ecdf03e0b..ea8bcec7292f 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_worker.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_worker.py @@ -39,6 +39,7 @@ ForwardBatch, ForwardMode, ) +from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig from sglang.srt.observability.req_time_stats import set_time_batch from sglang.srt.observability.trace import get_global_tracing_enabled @@ -69,11 +70,14 @@ draft_tp_context, fast_topk, generate_token_bitmask, - maybe_detect_nan, - maybe_detect_oob, select_top_k_tokens, ) from sglang.srt.utils import empty_context +from sglang.srt.utils.async_probe import ( + maybe_detect_inf, + maybe_detect_nan, + maybe_detect_oob, +) logger = logging.getLogger(__name__) @@ -248,10 +252,14 @@ def _bind_kv_context(self) -> None: self.kv_context = ctx def _frozen_kv_target_view(self, forward_batch: ForwardBatch): - return frozen_kv_target_view(forward_batch, self.kv_context) + return frozen_kv_target_view( + forward_batch, self.kv_context, self.draft_attn_backend + ) def _target_kv_pool_view(self, forward_batch: ForwardBatch): - return target_kv_pool_view(forward_batch, self.kv_context) + return target_kv_pool_view( + forward_batch, self.kv_context, self.draft_attn_backend + ) def _set_positions(self, forward_batch: ForwardBatch) -> None: set_frozen_kv_positions(forward_batch, self.topk) @@ -275,7 +283,6 @@ def _init_frozen_kv_metadata(self, forward_batch: ForwardBatch) -> None: forward_batch.seq_lens_sum = torch.sum(forward_batch.seq_lens).item() with self._frozen_kv_target_view(forward_batch): self.draft_attn_backend.init_forward_metadata(forward_batch) - forward_batch.attn_backend = self.draft_attn_backend def _init_frozen_kv_metadata_capture_cuda_graph( self, forward_batch: ForwardBatch @@ -290,7 +297,6 @@ def _init_frozen_kv_metadata_capture_cuda_graph( forward_mode=ForwardMode.DECODE, spec_info=None, ) - forward_batch.attn_backend = self.draft_attn_backend def _init_frozen_kv_metadata_replay_cuda_graph( self, forward_batch: ForwardBatch, bs: int, seq_lens_sum: int @@ -310,7 +316,6 @@ def _init_frozen_kv_metadata_replay_cuda_graph( else None ), ) - forward_batch.attn_backend = self.draft_attn_backend def init_cuda_graphs(self) -> None: if self.server_args.disable_cuda_graph or self.speculative_num_steps <= 1: @@ -396,11 +401,15 @@ def _run_assistant_seed_step( forward_batch.mm_input_embeds = mm_input_embeds self._set_positions(forward_batch) self._init_frozen_kv_metadata(forward_batch) - with self._target_kv_pool_view(forward_batch): + with ( + self._target_kv_pool_view(forward_batch), + forward_context(ForwardContext(attn_backend=self.draft_attn_backend)), + ): logits_output = self.draft_model_runner.forward( forward_batch, skip_attn_backend_init=True ).logits_output maybe_detect_nan(logits_output.next_token_logits, "frozen_kv_mtp_seed") + maybe_detect_inf(logits_output.next_token_logits, "frozen_kv_mtp_seed") self._capture_for_decode(logits_output, draft_input) finally: batch.forward_mode = forward_mode_backup @@ -678,7 +687,10 @@ def draft_forward( forward_batch.spec_info.hidden_states = hidden_states self._set_positions(forward_batch) - with self._target_kv_pool_view(forward_batch): + with ( + self._target_kv_pool_view(forward_batch), + forward_context(ForwardContext(attn_backend=self.draft_attn_backend)), + ): logits_output = self.draft_model_runner.forward( forward_batch, skip_attn_backend_init=True ).logits_output @@ -686,6 +698,9 @@ def draft_forward( maybe_detect_nan( logits_output.next_token_logits, f"frozen_kv_mtp_draft step {i}" ) + maybe_detect_inf( + logits_output.next_token_logits, f"frozen_kv_mtp_draft step {i}" + ) probs = torch.softmax(logits_output.next_token_logits, dim=-1) topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) maybe_detect_oob( @@ -744,6 +759,7 @@ def verify(self, batch: ScheduleBatch): batch.sampling_info.vocab_mask = None maybe_detect_nan(logits_output.next_token_logits, "frozen_kv_mtp_verify") + maybe_detect_inf(logits_output.next_token_logits, "frozen_kv_mtp_verify") spec_info.hidden_states = logits_output.hidden_states res: FrozenKVMTPVerifyOutput = spec_info.verify( diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index c84f8c1e857a..30beb43b3605 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -40,6 +40,11 @@ ForwardBatch, ForwardMode, ) +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + forward_context, + get_req_to_token_pool, +) from sglang.srt.model_executor.input_buffers import ForwardInputBuffers from sglang.srt.speculative.eagle_info import EagleDraftExtendInput from sglang.srt.speculative.multi_layer_eagle_utils import assign_new_state_triton @@ -369,8 +374,6 @@ def get_forward_batch(self, bs: int) -> ForwardBatch: seq_lens=seq_lens, seq_lens_cpu=seq_lens_cpu, next_token_logits_buffer=next_token_logits_buffer, - req_to_token_pool=self.model_runner.req_to_token_pool, - token_to_kv_pool=self.model_runner.token_to_kv_pool, out_cache_loc=out_cache_loc, seq_lens_sum=seq_lens.sum().item(), return_logprob=False, @@ -383,7 +386,6 @@ def get_forward_batch(self, bs: int) -> ForwardBatch: spec_algorithm=self.model_runner.spec_algorithm, spec_info=spec_info, capture_hidden_mode=capture_mode, - attn_backend=self.eagle_worker.draft_extend_attn_backend_list[self.step], extend_seq_lens=extend_seq_lens, extend_seq_lens_cpu=extend_seq_lens_cpu, padded_static_len=self.padded_static_len, @@ -400,26 +402,11 @@ def capture_one_batch_size(self, bs: int, forward: Callable, stream_idx: int = 0 graph = self._create_graph() stream = self.stream - self.deepep_adapter.capture(is_extend_in_batch=True) - num_tokens = bs * self.num_tokens_per_bs forward_batch = self.get_forward_batch(bs) + attn_backend = self.eagle_worker.draft_extend_attn_backend_list[self.step] - self.eagle_worker.draft_extend_attn_backend_list[ - self.step - ].init_forward_metadata_capture_cuda_graph( - bs=bs, - num_tokens=num_tokens, - req_pool_indices=forward_batch.req_pool_indices, - seq_lens=forward_batch.seq_lens, - encoder_lens=None, - forward_mode=self.forward_mode, - spec_info=forward_batch.spec_info, - ) - - # Run and capture def run_once(): - # model.forward() bypasses _forward_raw(), so invalidate manually. if self.model_runner.is_hybrid_swa: self.model_runner.token_to_kv_pool.invalidate_loc_cache() @@ -490,18 +477,28 @@ def run_once(): forward_batch.batch_size, self.step, forward_batch.req_pool_indices, - forward_batch.req_to_token_pool.req_to_token, + get_req_to_token_pool().req_to_token, self.eagle_worker.req_to_hidden_states_pool, ) forward_batch.out_cache_loc = output_cache_loc_backup forward_batch.spec_info.hidden_states = hidden_states_backup return ret - self._capture_init(run_once) - - out = self._capture_graph( - graph, get_global_graph_memory_pool(), stream, run_once - ) + with forward_context(ForwardContext(attn_backend=attn_backend)): + attn_backend.init_forward_metadata_capture_cuda_graph( + bs=bs, + num_tokens=num_tokens, + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=forward_batch.seq_lens, + encoder_lens=None, + forward_mode=self.forward_mode, + spec_info=forward_batch.spec_info, + ) + self.deepep_adapter.capture(is_extend_in_batch=True) + self._capture_init(run_once) + out = self._capture_graph( + graph, get_global_graph_memory_pool(), stream, run_once + ) set_global_graph_memory_pool(graph.pool()) return graph, out diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker.py b/python/sglang/srt/speculative/multi_layer_eagle_worker.py index 974d2a669c8e..4c055535fb90 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker.py @@ -54,10 +54,10 @@ fast_topk, generate_token_bitmask, load_token_map, - maybe_detect_nan, select_top_k_tokens, ) from sglang.srt.utils import empty_context, get_available_gpu_memory, is_cuda, is_npu +from sglang.srt.utils.async_probe import maybe_detect_nan if TYPE_CHECKING: from sglang.srt.model_executor.model_runner import ModelRunner diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index 35f189c05c5a..3e18e81b4896 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -48,12 +48,15 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_utils import ( draft_tp_context, - maybe_detect_nan, - maybe_detect_oob, record_stream_each, record_stream_for_v2_verify, select_top_k_tokens, ) +from sglang.srt.utils.async_probe import ( + maybe_detect_inf, + maybe_detect_nan, + maybe_detect_oob, +) from sglang.srt.utils.common import empty_context, fast_topk if TYPE_CHECKING: @@ -384,7 +387,6 @@ def _draft_extend_for_prefill( next_draft_input = EagleDraftInput( hidden_states=target_hidden_states, bonus_tokens=next_token_ids, - new_seq_lens=batch.seq_lens, # draft mode is same with decode mode, only 1 token per req num_tokens_per_req=1, num_tokens_for_logprob_per_req=1, @@ -419,9 +421,6 @@ def _draft_extend_for_prefill( topk_p_list = [] topk_index_list = [] for step in range(self.speculative_num_steps): - forward_batch.req_to_token_pool = self.draft_runner_list[ - step - ].req_to_token_pool output: ModelRunnerOutput = self.draft_runner_list[step].forward( forward_batch ) @@ -429,6 +428,10 @@ def _draft_extend_for_prefill( output.logits_output.next_token_logits, f"draft_extend_for_prefill step {step}", ) + maybe_detect_inf( + output.logits_output.next_token_logits, + f"draft_extend_for_prefill step {step}", + ) probs = torch.softmax(output.logits_output.next_token_logits, dim=-1) topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) topk_p_list.append(topk_p) @@ -526,9 +529,6 @@ def _draft_extend_for_decode( draft_logits_output.topk_index, ) else: - forward_batch.req_to_token_pool = self.draft_runner_list[ - step - ].req_to_token_pool draft_logits_output = self.draft_runner_list[step].forward( forward_batch, skip_attn_backend_init=True ) @@ -669,7 +669,7 @@ def clear_cache_pool(self): # allocator and kv cache pool are shared with target worker, which are cleared in scheduler pass - def forward_batch_generation(self, batch: ScheduleBatch, on_verify_complete=None): + def forward_batch_generation(self, batch: ScheduleBatch, on_publish=None): if batch.forward_mode.is_extend() or batch.is_extend_in_batch: # Target prefill target_capture_mode = ( @@ -680,9 +680,12 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_verify_complete=None batch.capture_hidden_mode = target_capture_mode batch_output = self.target_worker.forward_batch_generation(batch) + # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. + # Extend processed L prompt tokens; next verify iter expects same L. + batch_output.new_seq_lens = batch.seq_lens # Publish before draft_extend so the fence is at target-end. - if on_verify_complete is not None: - on_verify_complete(batch.seq_lens) + if on_publish is not None: + on_publish(batch_output.new_seq_lens) # Chain-style MTP needs FULL to get all-token hidden states; # non-chain only needs LAST (the target model's hidden states). @@ -711,8 +714,8 @@ def forward_batch_generation(self, batch: ScheduleBatch, on_verify_complete=None batch.spec_info = verify_input batch_output = self.verify(batch) # Publish before draft_extend so the fence is at verify-end. - if on_verify_complete is not None: - on_verify_complete(batch_output.next_draft_input.new_seq_lens) + if on_publish is not None: + on_publish(batch_output.new_seq_lens) self.draft_worker._draft_extend_for_decode(batch, batch_output) return batch_output @@ -768,6 +771,7 @@ def verify( # Sample maybe_detect_nan(logits_output.next_token_logits, "verify: target model logits") + maybe_detect_inf(logits_output.next_token_logits, "verify: target model logits") ( predict, accept_lens, @@ -792,10 +796,7 @@ def verify( batch, logits_output, predict, accept_index, self.speculative_num_steps ) - next_draft_input = EagleDraftInput( - bonus_tokens=bonus_tokens, - new_seq_lens=new_seq_lens, - ) + next_draft_input = EagleDraftInput(bonus_tokens=bonus_tokens) # verify_forward_batch transitively holds verify-time GPU tensors that # must outlive the imminent batch.input_ids rebind; scheduler pins it # in batch_record_buf via extra_keep_alive_refs. See EAGLEWorkerV2.verify. @@ -806,6 +807,7 @@ def verify( speculative_num_draft_tokens=self.speculative_num_draft_tokens, next_draft_input=next_draft_input, accept_lens=accept_lens, + new_seq_lens=new_seq_lens, routed_experts_output=forward_batch_output.routed_experts_output, indexer_topk_output=forward_batch_output.indexer_topk_output, extra_keep_alive_refs=[verify_forward_batch], diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index ca2be56661df..75b9af39f230 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -82,6 +82,9 @@ def _factory(server_args): spec_class=spec_class, ) + def is_some(self) -> bool: + return self != SpeculativeAlgorithm.NONE + def is_none(self) -> bool: return self == SpeculativeAlgorithm.NONE diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 3a39dcd4984b..e1b0f9fe8ee3 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -464,7 +464,9 @@ def get_src_tgt_cache_loc( page_size: int, ): src_cache_loc = out_cache_loc[accept_index] - tgt_cache_loc = torch.empty_like(src_cache_loc) + # zeros_like, not empty_like: any uncovered tail stays at slot 0 (padding) + # instead of caching-allocator garbage. + tgt_cache_loc = torch.zeros_like(src_cache_loc) extended_len = seq_lens + draft_token_num keep_len = torch.minimum( (seq_lens + num_correct_drafts + 1 + page_size - 1) // page_size * page_size, @@ -803,25 +805,6 @@ def draft_tp_context(tp_group: GroupCoordinator): yield -def maybe_detect_nan(tensor: torch.Tensor, msg: str = ""): - """Async NaN check — no GPU-CPU sync, error surfaces at next sync point.""" - if not envs.SGLANG_SPEC_NAN_DETECTION.get(): - return - torch._assert_async(~torch.any(torch.isnan(tensor)), f"NaN detected! {msg}") - - -def maybe_detect_oob(indices: torch.Tensor, low: int, high: int, msg: str): - """Async OOB check — no GPU-CPU sync, error surfaces at next sync point.""" - if not envs.SGLANG_SPEC_OOB_DETECTION.get(): - return - if indices.numel() == 0: - return - torch._assert_async( - (indices.min() >= low) & (indices.max() < high), - f"OOB indices not in [{low}, {high}): {msg}", - ) - - # Disable torch.compile for this function because it will be # even slower. # @torch.compile(dynamic=True) diff --git a/python/sglang/srt/utils/async_probe.py b/python/sglang/srt/utils/async_probe.py new file mode 100644 index 000000000000..aefc63bcc0eb --- /dev/null +++ b/python/sglang/srt/utils/async_probe.py @@ -0,0 +1,48 @@ +"""Async invariant probes — fire torch._assert_async without CPU sync. + +All probes are gated on SGLANG_ENABLE_ASYNC_ASSERT (default off in prod). +When the gate is on, a violation surfaces as an assertion at the next CUDA +sync point instead of as a silent NaN cascade or illegal-address crash. +""" + +import torch + +from sglang.srt.environ import envs + + +def maybe_detect_nan(tensor: torch.Tensor, msg: str = ""): + """Async NaN check — no GPU-CPU sync, error surfaces at next sync point.""" + if not envs.SGLANG_ENABLE_ASYNC_ASSERT.get(): + return + torch._assert_async(~torch.any(torch.isnan(tensor)), f"NaN detected! {msg}") + + +def maybe_detect_inf(tensor: torch.Tensor, msg: str = ""): + """Async Inf check — fp16 overflow surfaces as Inf before NaN.""" + if not envs.SGLANG_ENABLE_ASYNC_ASSERT.get(): + return + torch._assert_async(~torch.any(torch.isinf(tensor)), f"Inf detected! {msg}") + + +def maybe_detect_oob(indices: torch.Tensor, low: int, high: int, msg: str): + """Async OOB check — no GPU-CPU sync, error surfaces at next sync point.""" + if not envs.SGLANG_ENABLE_ASYNC_ASSERT.get(): + return + if indices.numel() == 0: + return + torch._assert_async( + (indices.min() >= low) & (indices.max() < high), + f"OOB indices not in [{low}, {high}): {msg}", + ) + + +def maybe_detect_page_aligned(indices: torch.Tensor, page_size: int, msg: str): + """Async page-alignment check on slot ids.""" + if not envs.SGLANG_ENABLE_ASYNC_ASSERT.get(): + return + if indices.numel() == 0 or page_size <= 1: + return + torch._assert_async( + (indices % page_size == 0).all(), + f"page-misaligned indices (page_size={page_size}): {msg}", + ) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 19b190e1b150..23c7e7bec21a 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -47,6 +47,7 @@ import types import uuid import warnings +from array import array from collections import OrderedDict, defaultdict from contextlib import contextmanager from dataclasses import dataclass @@ -103,6 +104,21 @@ torch_release = pkg_version.parse(torch.__version__).release +def flatten_arrays_to_int64_tensor( + parts: List[array[int]], device, pin: bool +) -> torch.Tensor: + """Flatten a list of array.array('q') buffers into one int64 tensor. + + Uses NumPy here to speed up the conversion by using memcpy + instead of a per-element PyLong-to-int64 walk. + """ + combined = np.concatenate([np.frombuffer(p, dtype=np.int64) for p in parts]) + cpu_t = torch.from_numpy(combined) + if pin: + cpu_t = cpu_t.pin_memory() + return cpu_t.to(device, non_blocking=True) + + # https://pytorch.org/docs/stable/notes/hip.html#checking-for-hip @lru_cache(maxsize=1) def is_hip() -> bool: @@ -1071,6 +1087,33 @@ def suppress_noisy_warnings(): category=FutureWarning, ) + # cutlass-dsl emits these inside `catch_warnings()+simplefilter("always")`, + # which bypasses filterwarnings; override showwarning to drop them too. + cutlass_dsl_noisy = { + ( + DeprecationWarning, + "Use explicit `struct.scalar.ptr` for pointer instead.", + ), + ( + UserWarning, + "NamedBarrier wait also arrives on the barrier. " + "Routing call to NamedBarrier.arrive_and_wait().", + ), + } + for cat, msg in cutlass_dsl_noisy: + warnings.filterwarnings("ignore", message=re.escape(msg), category=cat) + + if not getattr(warnings.showwarning, "_sglang_patched_cutlass_dsl", False): + prev_showwarning = warnings.showwarning + + def _filtered_showwarning(message, category, *args, **kwargs): + if (category, str(message)) in cutlass_dsl_noisy: + return + prev_showwarning(message, category, *args, **kwargs) + + _filtered_showwarning._sglang_patched_cutlass_dsl = True + warnings.showwarning = _filtered_showwarning + # Suppress noisy third-party HTTP loggers. # huggingface_hub uses httpx which logs every HTTP request at INFO level. for name in ("httpx", "httpcore"): @@ -3129,6 +3172,13 @@ def require_attn_tp_gather(server_args: ServerArgs): """ Check if the input of attention is scattered. """ + # Opt-out for models that manage SP scatter/gather at the model level + # and do not consume the upstream gathered_buffer. Without this, the + # cuda graph runner pads num_tokens to attn_tp_size, which can cause + # autotuners to pick suboptimal kernel variants at small batches. + if server_args.disable_attn_tp_gather: + return False + from sglang.srt.layers.moe.utils import get_moe_a2a_backend assert server_args.moe_dense_tp_size in [1, None] diff --git a/python/sglang/srt/utils/hf_transformers/tokenizer.py b/python/sglang/srt/utils/hf_transformers/tokenizer.py index 9a0fafb0fcbf..40b1693a36a5 100644 --- a/python/sglang/srt/utils/hf_transformers/tokenizer.py +++ b/python/sglang/srt/utils/hf_transformers/tokenizer.py @@ -105,7 +105,7 @@ def _load_tokenizer_by_declared_class(tokenizer_name, *args, **kwargs): if tok_cls is None: return None - logger.info( + logger.debug( "Loading tokenizer for %s directly as %s (bypassing AutoTokenizer)", tokenizer_name, tok_class_name, @@ -208,7 +208,7 @@ def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs): ``tokenizer_config.json``. May still return a ``TokenizersBackend`` if all retries fail (with a warning). """ - logger.warning( + logger.debug( "Tokenizer loaded as generic TokenizersBackend for %s, " "retrying with use_fast=False", tokenizer_name, @@ -239,7 +239,7 @@ def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs): tokenizer_name, ) else: - logger.warning( + logger.debug( "Tokenizer for %s loaded as generic TokenizersBackend. " "Set --trust-remote-code to load the model-specific tokenizer.", tokenizer_name, diff --git a/python/sglang/test/server_fixtures/disaggregation_fixture.py b/python/sglang/test/server_fixtures/disaggregation_fixture.py index 6ab5ec481411..7c1a1d68a9e7 100644 --- a/python/sglang/test/server_fixtures/disaggregation_fixture.py +++ b/python/sglang/test/server_fixtures/disaggregation_fixture.py @@ -14,6 +14,7 @@ is_in_ci, popen_launch_pd_server, popen_with_error_check, + start_subprocess_fail_fast_watcher, ) from sglang.utils import wait_for_http_ready @@ -39,6 +40,7 @@ def setUpClass(cls): f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=} {cls.bootstrap_port=}" ) cls.process_lb, cls.process_decode, cls.process_prefill = None, None, None + cls._fail_fast_stop = None # config transfer backend and rdma devices if is_in_ci(): @@ -110,6 +112,13 @@ def launch_all(cls): cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() + cls._fail_fast_stop = start_subprocess_fail_fast_watcher( + [ + ("prefill", cls.process_prefill), + ("decode", cls.process_decode), + ("lb", cls.process_lb), + ] + ) @classmethod def launch_lb(cls): @@ -141,6 +150,11 @@ def wait_server_ready( @classmethod def tearDownClass(cls): + # Stop the watcher BEFORE killing processes: kill_process_tree + # below makes them exit with a negative signal rc, which would + # otherwise trip the watcher and os._exit out of pytest mid-teardown. + if cls._fail_fast_stop is not None: + cls._fail_fast_stop.set() os.environ.pop("MC_TCP_ENABLE_CONNECTION_POOL") for process in [cls.process_lb, cls.process_decode, cls.process_prefill]: if process: diff --git a/python/sglang/test/server_fixtures/eagle_fixture.py b/python/sglang/test/server_fixtures/eagle_fixture.py index 512f6d9767de..b628831de468 100644 --- a/python/sglang/test/server_fixtures/eagle_fixture.py +++ b/python/sglang/test/server_fixtures/eagle_fixture.py @@ -37,10 +37,7 @@ class EagleServerBase(CustomTestCase): @classmethod def setUpClass(cls): cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), - ): + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): cls.process = popen_launch_server( cls.target_model, cls.base_url, diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index f34c73a4d387..a9d3670d64ff 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -574,6 +574,41 @@ def _run_and_check(): return process +def start_subprocess_fail_fast_watcher( + named_procs: list[tuple[str, subprocess.Popen]], +) -> threading.Event: + """Abort the test runner the moment any watched subprocess exits non-zero. + + Caller must `.set()` the returned Event before intentional teardown.""" + stop = threading.Event() + + def watcher(): + while not stop.is_set(): + for name, proc in named_procs: + rc = proc.poll() if proc else None + if rc is None or rc == 0: + continue + if stop.is_set(): + return + sys.stderr.write( + f"[FIXTURE FAIL-FAST] {name} (pid={proc.pid}) exited " + f"rc={rc}; aborting.\n" + ) + sys.stderr.flush() + for _, sib in named_procs: + if sib and sib is not proc: + try: + kill_process_tree(sib.pid, wait_timeout=10) + except Exception: + pass + # POSIX: signal N -> 128+N (os._exit masks negatives via & 0xff). + os._exit(rc if rc >= 0 else 128 + (-rc)) + time.sleep(0.1) + + threading.Thread(target=watcher, daemon=True, name="SubprocFailFastWatcher").start() + return stop + + def _try_enable_offline_mode_if_cache_complete( model_name_or_path: str, env: dict, other_args: Optional[list[str]] = None ) -> Optional[str]: diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index d37d2286e454..daad43e64d30 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -56,13 +56,16 @@ configure_environment() { [ "$(command -v python3)" = "$UV_VENV/bin/python3" ] || { echo "FATAL: python3 still resolves outside venv (got $(command -v python3))"; exit 1; } if [ -n "${GITHUB_ENV:-}" ]; then - echo "VIRTUAL_ENV=$UV_VENV" >> "$GITHUB_ENV" - echo "SGLANG_CI_VENV_PATH=$UV_VENV" >> "$GITHUB_ENV" - echo "BASH_ENV=$UV_VENV/env.sh" >> "$GITHUB_ENV" + # Self-heal: see install_rustup.sh for context on missing _runner_file_commands/. + mkdir -p "$(dirname "$GITHUB_ENV")" 2>/dev/null || true + echo "VIRTUAL_ENV=$UV_VENV" >> "$GITHUB_ENV" || true + echo "SGLANG_CI_VENV_PATH=$UV_VENV" >> "$GITHUB_ENV" || true + echo "BASH_ENV=$UV_VENV/env.sh" >> "$GITHUB_ENV" || true touch "$UV_VENV/env.sh" fi if [ -n "${GITHUB_PATH:-}" ]; then - echo "$UV_VENV/bin" >> "$GITHUB_PATH" + mkdir -p "$(dirname "$GITHUB_PATH")" 2>/dev/null || true + echo "$UV_VENV/bin" >> "$GITHUB_PATH" || true fi else echo "USE_VENV=0: skipping uv venv creation, installing into system Python" @@ -416,11 +419,11 @@ stabilize_flashinfer_jit_paths() { install_extra_deps() { if [ "$CU_MAJOR" = "13" ]; then - MOONCAKE_PKG="mooncake-transfer-engine-cuda13==0.3.10.post2" + MOONCAKE_PKG="mooncake-transfer-engine-cuda13==0.3.11.post1" MOONCAKE_STALE_PKG="mooncake-transfer-engine" EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc" else - MOONCAKE_PKG="mooncake-transfer-engine==0.3.10.post2" + MOONCAKE_PKG="mooncake-transfer-engine==0.3.11.post1" MOONCAKE_STALE_PKG="mooncake-transfer-engine-cuda13" EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc-cu12" fi diff --git a/scripts/ci/musa/rename_wheels_musa.sh b/scripts/ci/musa/rename_wheels_musa.sh index 23ea57f2bf91..f3816548ad0b 100755 --- a/scripts/ci/musa/rename_wheels_musa.sh +++ b/scripts/ci/musa/rename_wheels_musa.sh @@ -1,46 +1,102 @@ #!/usr/bin/env bash -set -euo pipefail - -# Rename MUSA wheels to include a +musa build tag. +# Align MUSA wheel filenames (+musa43/...) with internal METADATA Version and +# WHEEL tags after build. Two drifts need fixing in lockstep: +# - METADATA `Version:` must carry the `+musa` local version, or +# recent pip versions reject the wheel with "inconsistent version". +# - WHEEL `Tag:` must be `manylinux2014_*` when the filename says so; +# leaving it as `linux_*` can trip installers that re-derive the platform. +# Unpack → patch WHEEL/METADATA → wheel pack (RECORD regenerated; no hand-editing). +# # Usage: # rename_wheels_musa.sh [wheel_dir] # Example: # rename_wheels_musa.sh 43 sgl-kernel/dist +set -euxo pipefail if [[ $# -lt 1 || $# -gt 2 ]]; then echo "Usage: $0 [wheel_dir]" >&2 exit 1 fi -MUSA_SUFFIX="$1" +MUSA_SUFFIX="+musa$1" WHEEL_DIR="${2:-dist}" -wheel_files=("$WHEEL_DIR"/*.whl) +patch_wheel_platform_tags() { + local wheel_file="$1" + # Line-end anchors: "linux_x86_64" is a substring of "manylinux2014_x86_64", so + # unanchored global replace corrupts tags on a second run. + sed -i \ + -e 's/-linux_x86_64$/-manylinux2014_x86_64/' \ + -e 's/-linux_aarch64$/-manylinux2014_aarch64/' \ + "$wheel_file" +} +wheel_files=("$WHEEL_DIR"/*.whl) if [[ ! -e "${wheel_files[0]}" ]]; then echo "No wheel files found in ${WHEEL_DIR}/, nothing to rename." exit 0 fi for wheel in "${wheel_files[@]}"; do - # Normalize platform tag to manylinux2014 - intermediate_wheel="${wheel/linux/manylinux2014}" - - # Extract Python ABI version (e.g. cp310) - if [[ $intermediate_wheel =~ -cp([0-9]+)- ]]; then - cp_version="${BASH_REMATCH[1]}" - else - echo "Could not extract Python version from wheel name: $intermediate_wheel" >&2 - continue - fi - - # Insert +musa before the Python ABI tag - new_wheel="${intermediate_wheel/-cp${cp_version}/+musa${MUSA_SUFFIX}-cp${cp_version}}" - - if [[ "$wheel" != "$new_wheel" ]]; then - echo "Renaming $wheel -> $new_wheel" - mv -- "$wheel" "$new_wheel" - fi + [[ -f "$wheel" ]] || continue + + intermediate_wheel="$wheel" + case "$wheel" in + *-linux_x86_64.whl) + intermediate_wheel="${wheel%-linux_x86_64.whl}-manylinux2014_x86_64.whl" + ;; + *-linux_aarch64.whl) + intermediate_wheel="${wheel%-linux_aarch64.whl}-manylinux2014_aarch64.whl" + ;; + esac + if [[ "$wheel" != "$intermediate_wheel" ]]; then + mv -- "$wheel" "$intermediate_wheel" + wheel="$intermediate_wheel" + fi + + TMPDIR=$(mktemp -d) + trap 'rm -rf -- "$TMPDIR"' ERR + + "${PYTHON:-python3}" -m wheel unpack "$wheel" --dest "$TMPDIR" + # `find | head -1` succeeds with empty stdout when there are no matches — + # `set -e` won't catch that. Assert each path is real so a malformed wheel + # surfaces with a useful message instead of a downstream `sed: /WHEEL` error. + UNPACKED=$(find "$TMPDIR" -mindepth 1 -maxdepth 1 -type d | head -1) + [[ -d "$UNPACKED" ]] || { echo "ERROR: wheel unpack produced no top-level dir for $wheel" >&2; exit 1; } + DIST_INFO=$(find "$UNPACKED" -maxdepth 1 -type d -name "*.dist-info" | head -1) + [[ -d "$DIST_INFO" ]] || { echo "ERROR: no *.dist-info under $UNPACKED (malformed wheel?): $wheel" >&2; exit 1; } + WHEEL_META="${DIST_INFO}/WHEEL" + METADATA_FILE="${DIST_INFO}/METADATA" + [[ -f "$WHEEL_META" && -f "$METADATA_FILE" ]] || { echo "ERROR: missing WHEEL or METADATA in $DIST_INFO" >&2; exit 1; } + + patch_wheel_platform_tags "$WHEEL_META" + + ORIG_VERSION=$(grep '^Version:' "$METADATA_FILE" | head -1 | sed 's/^Version:[[:space:]]*//') + # Empty ORIG_VERSION would fall through the `+musa` check below and silently + # produce `Version: +musa43` — a broken release. Fail loud instead. + [[ -n "$ORIG_VERSION" ]] || { echo "ERROR: no 'Version:' line in $METADATA_FILE" >&2; exit 1; } + if [[ "$ORIG_VERSION" == *"$MUSA_SUFFIX"* ]]; then + echo "Skipping $wheel: version in METADATA is already suffixed." + rm -rf "$TMPDIR" + trap - ERR + continue + fi + NEW_VERSION="${ORIG_VERSION}${MUSA_SUFFIX}" + sed -i "s/^Version:.*/Version: ${NEW_VERSION}/" "$METADATA_FILE" + # `sed -i` exits 0 even when the pattern matched zero lines. Verify the + # rewrite actually landed before we publish. + grep -qx "Version: ${NEW_VERSION}" "$METADATA_FILE" || { echo "ERROR: METADATA Version rewrite did not land in $METADATA_FILE" >&2; exit 1; } + + OLD_BASE=$(basename "$DIST_INFO") + NEW_BASE="${OLD_BASE/${ORIG_VERSION}/${NEW_VERSION}}" + # `${var/pat/repl}` silently leaves var unchanged if pat is empty or absent. + [[ "$NEW_BASE" != "$OLD_BASE" ]] || { echo "ERROR: dist-info dir '$OLD_BASE' did not contain ORIG_VERSION='$ORIG_VERSION'" >&2; exit 1; } + mv "$DIST_INFO" "${UNPACKED}/${NEW_BASE}" + + rm -f "$wheel" + "${PYTHON:-python3}" -m wheel pack "$UNPACKED" --dest-dir "$WHEEL_DIR" + rm -rf "$TMPDIR" + trap - ERR done echo "MUSA wheel renaming completed." diff --git a/scripts/ci/utils/diffusion/comparison_configs.json b/scripts/ci/utils/diffusion/comparison_configs.json index aa4171aca704..c8c4b25612f6 100644 --- a/scripts/ci/utils/diffusion/comparison_configs.json +++ b/scripts/ci/utils/diffusion/comparison_configs.json @@ -13,7 +13,7 @@ "num_gpus": 1, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup --dit-layerwise-offload false", + "serve_args": "--warmup --dit-layerwise-offload false", "extra_env": {} } } @@ -29,7 +29,7 @@ "num_gpus": 1, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup --dit-layerwise-offload false", + "serve_args": "--warmup --dit-layerwise-offload false", "extra_env": {} } } @@ -45,7 +45,7 @@ "num_gpus": 1, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup", + "serve_args": "--warmup", "extra_env": {} } } @@ -62,7 +62,7 @@ "num_gpus": 1, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup", + "serve_args": "--warmup", "extra_env": {} } } @@ -78,7 +78,7 @@ "num_gpus": 1, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup", + "serve_args": "--warmup", "extra_env": {} } } @@ -95,7 +95,7 @@ "num_gpus": 4, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory", + "serve_args": "--warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory", "extra_env": {} } } @@ -113,7 +113,7 @@ "num_gpus": 1, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup", + "serve_args": "--warmup", "extra_env": {} } } @@ -130,7 +130,7 @@ "num_gpus": 2, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline", + "serve_args": "--warmup --enable-cfg-parallel --pipeline-class-name LTX2TwoStagePipeline", "extra_env": {} } } @@ -148,7 +148,7 @@ "num_gpus": 2, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup --pipeline-class-name LTX2TwoStagePipeline --cfg-parallel-size 2", + "serve_args": "--warmup --pipeline-class-name LTX2TwoStagePipeline --cfg-parallel-size 2", "extra_env": {} } } @@ -166,7 +166,7 @@ "num_gpus": 4, "frameworks": { "sglang": { - "serve_args": "--enable-torch-compile --warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory", + "serve_args": "--warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory", "extra_env": {} } } diff --git a/scripts/ci/utils/diffusion/publish_diffusion_gt.py b/scripts/ci/utils/diffusion/publish_diffusion_gt.py index 9912a2c1983c..b2b3745b1231 100644 --- a/scripts/ci/utils/diffusion/publish_diffusion_gt.py +++ b/scripts/ci/utils/diffusion/publish_diffusion_gt.py @@ -4,13 +4,19 @@ """ import argparse +import base64 import hashlib +import io import json import os import sys +from dataclasses import dataclass from pathlib import Path from urllib.error import HTTPError +import numpy as np +from PIL import Image, ImageFilter + # Reuse GitHub API helpers from publish_traces. # Support both direct script execution and package-style imports. if __package__: @@ -47,6 +53,32 @@ DEFAULT_TARGET_DIR = "diffusion-ci/consistency_gt/sglang_generated" IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"} +QUALITY_MAX_SIDE = 256 +LOW_DETAIL_STD_THRESHOLD = 0.075 +LOW_DETAIL_ENTROPY_THRESHOLD = 0.55 +LOW_DETAIL_BLUR_RESIDUAL_THRESHOLD = 0.035 +LOW_DETAIL_GRADIENT_P95_THRESHOLD = 0.045 +RANDOM_NOISE_CORRELATION_THRESHOLD = 0.55 +RANDOM_NOISE_LOW_FREQUENCY_THRESHOLD = 0.20 +RANDOM_NOISE_BLUR_RESIDUAL_THRESHOLD = 0.045 +OLD_NEW_MIN_SSIM = 0.20 +OLD_NEW_MAX_MEAN_ABS_DIFF = 45.0 + + +@dataclass(frozen=True) +class ImageQualityMetrics: + luminance_std: float + entropy: float + blur_residual: float + gradient_p95: float + neighbor_correlation: float + low_frequency_ratio: float + + +@dataclass(frozen=True) +class OldNewMetrics: + ssim: float + mean_abs_diff: float def collect_images(source_dir, target_dir): @@ -72,6 +104,15 @@ def git_blob_sha(content): def get_remote_blob_shas(repo_owner, repo_name, target_dir, token): + return { + path: item["sha"] + for path, item in get_remote_image_entries( + repo_owner, repo_name, target_dir, token + ).items() + } + + +def get_remote_image_entries(repo_owner, repo_name, target_dir, token): url = ( f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents/" f"{target_dir}?ref={BRANCH}" @@ -84,9 +125,11 @@ def get_remote_blob_shas(repo_owner, repo_name, target_dir, token): raise entries = json.loads(response) return { - item["path"]: item["sha"] + item["path"]: item for item in entries - if item.get("type") == "file" and "sha" in item + if item.get("type") == "file" + and "sha" in item + and os.path.splitext(item["path"])[1].lower() in IMAGE_EXTENSIONS } @@ -98,6 +141,237 @@ def filter_changed_files(files, remote_blob_shas): ] +def get_remote_blob_content(repo_owner, repo_name, blob_sha, token): + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs/{blob_sha}" + response = make_github_request(url, token) + blob = json.loads(response) + if blob.get("encoding") != "base64": + raise ValueError( + f"Unexpected blob encoding for {blob_sha}: {blob.get('encoding')}" + ) + return base64.b64decode(blob["content"]) + + +def _load_quality_image(content): + with Image.open(io.BytesIO(content)) as image: + image = image.convert("RGB") + image.thumbnail((QUALITY_MAX_SIDE, QUALITY_MAX_SIDE), Image.Resampling.BICUBIC) + return image.copy() + + +def _image_to_rgb_array(image): + return np.asarray(image, dtype=np.float32) + + +def _luminance(rgb): + return 0.299 * rgb[..., 0] + 0.587 * rgb[..., 1] + 0.114 * rgb[..., 2] + + +def _neighbor_correlation(luma): + def corr(a, b): + a = a.ravel() + b = b.ravel() + if a.std() < 1e-6 or b.std() < 1e-6: + return 1.0 + return float(np.corrcoef(a, b)[0, 1]) + + return (corr(luma[:, 1:], luma[:, :-1]) + corr(luma[1:, :], luma[:-1, :])) / 2 + + +def _low_frequency_ratio(luma): + centered = luma - luma.mean() + power = np.abs(np.fft.fftshift(np.fft.fft2(centered))) ** 2 + total_power = power.sum() + if total_power < 1e-12: + return 0.0 + + height, width = luma.shape + y, x = np.ogrid[:height, :width] + center_y = height // 2 + center_x = width // 2 + radius = np.sqrt((y - center_y) ** 2 + (x - center_x) ** 2) + low_frequency_radius = min(height, width) * 0.08 + return float(power[radius <= low_frequency_radius].sum() / total_power) + + +def compute_image_quality_metrics(content): + image = _load_quality_image(content) + rgb = _image_to_rgb_array(image) + luma = _luminance(rgb) / 255.0 + + gradients = np.concatenate( + [ + np.abs(np.diff(luma, axis=1)).ravel(), + np.abs(np.diff(luma, axis=0)).ravel(), + ] + ) + histogram, _ = np.histogram(luma, bins=32, range=(0, 1)) + probabilities = histogram / histogram.sum() + nonzero_probabilities = probabilities[probabilities > 0] + entropy = float( + -(nonzero_probabilities * np.log2(nonzero_probabilities)).sum() / 5.0 + ) + blurred = _image_to_rgb_array(image.filter(ImageFilter.GaussianBlur(radius=3))) + + return ImageQualityMetrics( + luminance_std=float(luma.std()), + entropy=entropy, + blur_residual=float(np.mean(np.abs(rgb - blurred)) / 255.0), + gradient_p95=float(np.percentile(gradients, 95)), + neighbor_correlation=_neighbor_correlation(luma), + low_frequency_ratio=_low_frequency_ratio(luma), + ) + + +def get_quality_failure_reasons(metrics): + reasons = [] + low_detail_static = ( + metrics.luminance_std < LOW_DETAIL_STD_THRESHOLD + and metrics.entropy < LOW_DETAIL_ENTROPY_THRESHOLD + and ( + metrics.blur_residual < LOW_DETAIL_BLUR_RESIDUAL_THRESHOLD + or metrics.gradient_p95 < LOW_DETAIL_GRADIENT_P95_THRESHOLD + ) + ) + high_frequency_noise = ( + metrics.neighbor_correlation < RANDOM_NOISE_CORRELATION_THRESHOLD + and metrics.low_frequency_ratio < RANDOM_NOISE_LOW_FREQUENCY_THRESHOLD + and metrics.blur_residual > RANDOM_NOISE_BLUR_RESIDUAL_THRESHOLD + ) + if low_detail_static: + reasons.append("low-contrast low-detail output") + if high_frequency_noise: + reasons.append("high-frequency random noise") + return reasons + + +def _resize_for_old_new_compare(content, size=None): + with Image.open(io.BytesIO(content)) as image: + image = image.convert("RGB") + if size is None: + image.thumbnail( + (QUALITY_MAX_SIDE, QUALITY_MAX_SIDE), Image.Resampling.BICUBIC + ) + else: + image = image.resize(size, Image.Resampling.BICUBIC) + return _image_to_rgb_array(image) + + +def compute_old_new_metrics(old_content, new_content): + old_rgb = _resize_for_old_new_compare(old_content) + new_rgb = _resize_for_old_new_compare( + new_content, size=(old_rgb.shape[1], old_rgb.shape[0]) + ) + old_luma = _luminance(old_rgb) / 255.0 + new_luma = _luminance(new_rgb) / 255.0 + + old_mean = old_luma.mean() + new_mean = new_luma.mean() + old_variance = old_luma.var() + new_variance = new_luma.var() + covariance = ((old_luma - old_mean) * (new_luma - new_mean)).mean() + c1 = 0.01**2 + c2 = 0.03**2 + ssim = ( + (2 * old_mean * new_mean + c1) + * (2 * covariance + c2) + / ((old_mean**2 + new_mean**2 + c1) * (old_variance + new_variance + c2)) + ) + + return OldNewMetrics( + ssim=float(ssim), + mean_abs_diff=float(np.abs(old_rgb - new_rgb).mean()), + ) + + +def _format_quality_metrics(metrics): + return ( + f"std={metrics.luminance_std:.4f}, entropy={metrics.entropy:.4f}, " + f"blur_residual={metrics.blur_residual:.4f}, " + f"gradient_p95={metrics.gradient_p95:.4f}, " + f"neighbor_corr={metrics.neighbor_correlation:.4f}, " + f"low_freq={metrics.low_frequency_ratio:.4f}" + ) + + +def _format_old_new_metrics(metrics): + return f"ssim={metrics.ssim:.4f}, mean_abs_diff={metrics.mean_abs_diff:.2f}" + + +def validate_gt_files(files_to_upload, changed_files, remote_image_entries, token): + failures = [] + for path, content in files_to_upload: + quality_metrics = compute_image_quality_metrics(content) + quality_reasons = get_quality_failure_reasons(quality_metrics) + if quality_reasons: + failures.append( + f"{path}: {', '.join(quality_reasons)} " + f"({_format_quality_metrics(quality_metrics)})" + ) + + for path, content in changed_files: + remote_entry = remote_image_entries.get(path) + if not remote_entry: + continue + + old_content = get_remote_blob_content( + REPO_OWNER, REPO_NAME, remote_entry["sha"], token + ) + old_quality_metrics = compute_image_quality_metrics(old_content) + old_quality_reasons = get_quality_failure_reasons(old_quality_metrics) + if old_quality_reasons: + print( + f"Skipping old/new drift check for {path} because existing GT is " + f"already suspicious: {', '.join(old_quality_reasons)} " + f"({_format_quality_metrics(old_quality_metrics)})" + ) + continue + + old_new_metrics = compute_old_new_metrics(old_content, content) + if ( + old_new_metrics.ssim < OLD_NEW_MIN_SSIM + and old_new_metrics.mean_abs_diff > OLD_NEW_MAX_MEAN_ABS_DIFF + ): + failures.append( + f"{path}: changed too far from existing GT " + f"({_format_old_new_metrics(old_new_metrics)})" + ) + + if not failures: + print( + f"GT quality gate passed for {len(files_to_upload)} generated image(s) " + f"and {len(changed_files)} changed image(s)." + ) + return + + print("GT quality gate failed; refusing to publish suspicious image updates:") + for failure in failures: + print(f" - {failure}") + sys.exit(1) + + +def check_quality(source_dir, target_dir=None): + target_dir = target_dir or DEFAULT_TARGET_DIR + token = os.getenv("GITHUB_TOKEN") + if not token: + print("Error: GITHUB_TOKEN environment variable not set") + sys.exit(1) + + files_to_upload = collect_images(source_dir, target_dir) + if not files_to_upload: + print(f"No image files found in {source_dir}") + return + + remote_image_entries = get_remote_image_entries( + REPO_OWNER, REPO_NAME, target_dir, token + ) + remote_blob_shas = { + path: item["sha"] for path, item in remote_image_entries.items() + } + changed_files = filter_changed_files(files_to_upload, remote_blob_shas) + validate_gt_files(files_to_upload, changed_files, remote_image_entries, token) + + def publish(source_dir, target_dir=None): target_dir = target_dir or DEFAULT_TARGET_DIR token = os.getenv("GITHUB_TOKEN") @@ -129,10 +403,16 @@ def publish(source_dir, target_dir=None): try: branch_sha = get_branch_sha(REPO_OWNER, REPO_NAME, BRANCH, token) tree_sha = get_tree_sha(REPO_OWNER, REPO_NAME, branch_sha, token) - remote_blob_shas = get_remote_blob_shas( + remote_image_entries = get_remote_image_entries( REPO_OWNER, REPO_NAME, target_dir, token ) + remote_blob_shas = { + path: item["sha"] for path, item in remote_image_entries.items() + } changed_files = filter_changed_files(files_to_upload, remote_blob_shas) + validate_gt_files( + files_to_upload, changed_files, remote_image_entries, token + ) if not changed_files: print("No image changes to publish.") return @@ -211,8 +491,16 @@ def main(): default=None, help=f"Target directory in the remote repo (default: {DEFAULT_TARGET_DIR})", ) + parser.add_argument( + "--check-only", + action="store_true", + help="Validate generated GT images without publishing them", + ) args = parser.parse_args() - publish(args.source_dir, args.target_dir) + if args.check_only: + check_quality(args.source_dir, args.target_dir) + else: + publish(args.source_dir, args.target_dir) if __name__ == "__main__": diff --git a/scripts/ci/utils/diffusion/run_comparison.py b/scripts/ci/utils/diffusion/run_comparison.py index c25df75ce239..eadd091c42b0 100644 --- a/scripts/ci/utils/diffusion/run_comparison.py +++ b/scripts/ci/utils/diffusion/run_comparison.py @@ -43,9 +43,7 @@ DEFAULT_PORT = 30000 SGLANG_MASTER_PORT_OFFSET = 5 SGLANG_SCHEDULER_PORT_OFFSET = 55 -HEALTH_TIMEOUT = ( - 2400 # seconds (40 min — FLUX.2-dev needs ~10 min download + torch.compile) -) +HEALTH_TIMEOUT = 2400 # seconds (40 min — keep large model download/warmup headroom) REQUEST_TIMEOUT = 1200 # seconds GPU_CLEAR_WAIT = 15 # seconds between framework runs SERVER_FATAL_ERROR_PATTERNS = ( diff --git a/scripts/ci/utils/install_rustup.sh b/scripts/ci/utils/install_rustup.sh index 71748c05a345..478f48da228a 100755 --- a/scripts/ci/utils/install_rustup.sh +++ b/scripts/ci/utils/install_rustup.sh @@ -8,7 +8,11 @@ set -euxo pipefail # GitHub Actions steps in the same job. export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" if [ -n "${GITHUB_PATH:-}" ]; then - echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> "${GITHUB_PATH}" + # Self-heal if _runner_file_commands/ disappears mid-job on some self-hosted + # runners; the runner reads this file by its registered UUID at step end, so + # recreating the path keeps PATH propagation working for subsequent steps. + mkdir -p "$(dirname "${GITHUB_PATH}")" 2>/dev/null || true + echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> "${GITHUB_PATH}" || true fi if command -v cargo >/dev/null 2>&1 && command -v rustc >/dev/null 2>&1; then diff --git a/sgl-kernel/CMakeLists.txt b/sgl-kernel/CMakeLists.txt index 8ede4482c0ed..86d315783a67 100644 --- a/sgl-kernel/CMakeLists.txt +++ b/sgl-kernel/CMakeLists.txt @@ -258,6 +258,7 @@ set(SOURCES "csrc/elementwise/activation.cu" "csrc/elementwise/concat_mla.cu" "csrc/elementwise/copy.cu" + "csrc/elementwise/dsv4_norm_rope.cu" "csrc/elementwise/fused_add_rms_norm_kernel.cu" "csrc/elementwise/pos_enc.cu" "csrc/elementwise/topk.cu" diff --git a/sgl-kernel/benchmark/bench_dsv4_norm_rope.py b/sgl-kernel/benchmark/bench_dsv4_norm_rope.py new file mode 100644 index 000000000000..3e52c415df45 --- /dev/null +++ b/sgl-kernel/benchmark/bench_dsv4_norm_rope.py @@ -0,0 +1,75 @@ +"""Benchmark for DeepSeek-V4 fused norm + RoPE kernels.""" + +import itertools + +import sgl_kernel +import torch +import triton +import triton.testing + +try: + from sglang.utils import is_in_ci + + IS_CI = is_in_ci() +except ImportError: + IS_CI = False + +batch_sizes = [1] if IS_CI else [1, 4, 16, 64, 256] +num_heads_list = [8] if IS_CI else [8, 16, 64] +head_dims = [192] if IS_CI else [128, 192] + +configs = list(itertools.product(batch_sizes, num_heads_list, head_dims)) + + +def torch_rmsnorm_rope( + q: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, eps: float +) -> torch.Tensor: + """Naive PyTorch reference: RMSNorm + RoPE.""" + rms = torch.sqrt(q.float().pow(2).mean(dim=-1, keepdim=True) + eps) + q_normed = (q.float() / rms).to(q.dtype) + return q_normed + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "num_heads", "head_dim"], + x_vals=configs, + line_arg="provider", + line_vals=["sglang", "torch"], + line_names=["SGL Kernel", "PyTorch"], + styles=[("green", "-"), ("red", "--")], + ylabel="µs (median)", + plot_name="dsv4-q-norm-rope-performance", + args={}, + ) +) +def benchmark_q_norm_rope(batch_size, num_heads, head_dim, provider): + torch.manual_seed(42) + eps = 1e-6 + max_pos = 8192 + rope_dim = 64 + + q_input = torch.randn( + batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda" + ) + q_output = torch.empty_like(q_input) + freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda") + positions = torch.randint( + 0, max_pos, (batch_size,), dtype=torch.int32, device="cuda" + ) + + if provider == "sglang": + fn = lambda: sgl_kernel.dsv4_fused_q_norm_rope( + q_input, freqs_cis, positions, eps, q_output + ) + else: + fn = lambda: torch_rmsnorm_rope(q_input, freqs_cis, positions, eps) + + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + fn, quantiles=[0.5, 0.2, 0.8] + ) + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + benchmark_q_norm_rope.run(print_data=True) diff --git a/sgl-kernel/benchmark/bench_moe_topk_softmax.py b/sgl-kernel/benchmark/bench_moe_topk_softmax.py index 451ae8d80d63..9ae15d6c96cb 100644 --- a/sgl-kernel/benchmark/bench_moe_topk_softmax.py +++ b/sgl-kernel/benchmark/bench_moe_topk_softmax.py @@ -144,7 +144,7 @@ def calculate_diff(num_tokens, num_experts, topk): else: num_tokens_range = [128, 512, 1024, 2048, 4096, 8192, 16384, 32768] num_experts_range = [32, 64, 128, 256, 12, 512] - topk_range = [1, 2, 4, 8] + topk_range = [1, 2, 4, 8, 10] configs = list(itertools.product(num_tokens_range, num_experts_range, topk_range)) diff --git a/sgl-kernel/cmake/flashmla.cmake b/sgl-kernel/cmake/flashmla.cmake index b67ace6afed1..564d5a6f9ae6 100644 --- a/sgl-kernel/cmake/flashmla.cmake +++ b/sgl-kernel/cmake/flashmla.cmake @@ -1,10 +1,8 @@ -include(FetchContent) - # flash_mla FetchContent_Declare( repo-flashmla GIT_REPOSITORY https://github.com/sgl-project/FlashMLA - GIT_TAG abb54777d4e08c8054c238f59889b52d4e9f0896 + GIT_TAG df022ebafb88578eab9f0300606ee765608d8b5c GIT_SHALLOW OFF ) FetchContent_Populate(repo-flashmla) diff --git a/sgl-kernel/csrc/common_extension.cc b/sgl-kernel/csrc/common_extension.cc index b7c01a08327e..b50687abcf59 100644 --- a/sgl-kernel/csrc/common_extension.cc +++ b/sgl-kernel/csrc/common_extension.cc @@ -214,6 +214,21 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.def("apply_shuffle_mul_sum(Tensor input, Tensor output, Tensor permutation, Tensor? factors) -> ()"); m.impl("apply_shuffle_mul_sum", torch::kCUDA, &apply_shuffle_mul_sum); + // DeepSeek-V4 fused norm + rope + m.def( + "dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()"); + m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope); + + m.def( + "dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, " + "Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()"); + m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla); + + m.def( + "dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, " + "Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()"); + m.impl("dsv4_fused_q_indexer_rope_hadamard_quant", torch::kCUDA, &dsv4_fused_q_indexer_rope_hadamard_quant); + m.def( "fused_qk_norm_rope(Tensor! qkv, int num_heads_q, " "int num_heads_k, int num_heads_v, int head_dim, float eps, " diff --git a/sgl-kernel/csrc/common_extension_rocm.cc b/sgl-kernel/csrc/common_extension_rocm.cc index 1c7265b2ba96..cad9c95d95c5 100644 --- a/sgl-kernel/csrc/common_extension_rocm.cc +++ b/sgl-kernel/csrc/common_extension_rocm.cc @@ -47,6 +47,25 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) { "topk_indices_offset, Tensor ? row_starts) -> ()"); m.impl("fast_topk_transform_ragged_fused", torch::kCUDA, &fast_topk_transform_ragged_interface); + m.def( + "deepseek_v4_topk_transform_512(Tensor scores, Tensor seq_lens, Tensor page_table, Tensor! " + "page_indices, int page_size, Tensor!? raw_indices) -> ()"); + m.impl("deepseek_v4_topk_transform_512", torch::kCUDA, &deepseek_v4_topk_transform_512); + + m.def( + "dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()"); + m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope); + + m.def( + "dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, " + "Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()"); + m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla); + + m.def( + "dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, " + "Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()"); + m.impl("dsv4_fused_q_indexer_rope_hadamard_quant", torch::kCUDA, &dsv4_fused_q_indexer_rope_hadamard_quant); + /* * From csrc/allreduce */ diff --git a/sgl-kernel/csrc/cpu/common.h b/sgl-kernel/csrc/cpu/common.h index 139121859faa..0da10f0bccdd 100644 --- a/sgl-kernel/csrc/cpu/common.h +++ b/sgl-kernel/csrc/cpu/common.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #if defined(_OPENMP) @@ -44,6 +45,14 @@ namespace { } \ }() +// Half + BFloat16, plus one extra scalar type +#define AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, ...) \ + AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(__VA_ARGS__) \ + AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) + +#define AT_DISPATCH_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES_AND(SCALARTYPE, __VA_ARGS__)) + // dispatch: bfloat16, float16, int8_t, fp8_e4m3, uint8_t(mxfp4/int4) #define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \ [&] { \ diff --git a/sgl-kernel/csrc/cpu/kvcache.cpp b/sgl-kernel/csrc/cpu/kvcache.cpp new file mode 100644 index 000000000000..ca2dc81b86e5 --- /dev/null +++ b/sgl-kernel/csrc/cpu/kvcache.cpp @@ -0,0 +1,130 @@ +#include "common.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ dst, const scalar_t* __restrict__ src, int size) { + int d = 0; +#if defined(CPU_CAPABILITY_AVX512) + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + + for (; d <= size - kVecSize; d += kVecSize) { + Vec data = Vec::loadu(src + d); + data.store(dst + d); + } +#endif + for (; d < size; ++d) { + dst[d] = src[d]; + } +} + +template +void store_cache_kernel_impl( + const scalar_t* __restrict__ k, + const scalar_t* __restrict__ v, + scalar_t* __restrict__ k_cache, + scalar_t* __restrict__ v_cache, + const index_t* __restrict__ indices, + int64_t batch_size, + int64_t num_pages, + int64_t row_dim, + int64_t k_stride, + int64_t v_stride, + int64_t kc_stride, + int64_t vc_stride) { + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + for (int64_t bs = begin; bs < end; ++bs) { + const int64_t idx = static_cast(indices[bs]); + const scalar_t* k_ptr = k + bs * k_stride; + const scalar_t* v_ptr = v + bs * v_stride; + scalar_t* kc_ptr = k_cache + idx * kc_stride; + scalar_t* vc_ptr = v_cache + idx * vc_stride; + copy_stub(kc_ptr, k_ptr, row_dim); + copy_stub(vc_ptr, v_ptr, row_dim); + } + }); +} + +} // anonymous namespace + +// check tensor last two dimensions are contiguous +#define CHECK_LAST2_DIM_CONTIGUOUS(x, ndim) \ + do { \ + const auto& _x = (x); \ + const auto _ndim = _x.dim(); \ + const auto _strides = _x.strides(); \ + const auto _sizes = _x.sizes(); \ + TORCH_CHECK(_ndim == ndim, #x " must have " #ndim " dimensions"); \ + TORCH_CHECK( \ + _ndim >= 2 && _strides[_ndim - 1] == 1 && _strides[_ndim - 2] == _sizes[_ndim - 1], \ + #x " must be contiguous at the last two dimensions"); \ + } while (0) + +// [NB]: store_cache takes 3 dimension tensors, +// This is to avoid the overhead of creating a new TensorImpl +// from .view(-1, row_dim) +// +// k : [batch_size, num_heads, head_size] -> [batch_size, row_dim] +// v : [batch_size, num_heads, head_size] -> [batch_size, row_dim] +// k_cache : [num_pages, num_heads, head_size] -> [num_pages, row_dim] +// v_cache : [num_pages, num_heads, head_size] -> [num_pages, row_dim] +// indices : [batch_size] +// +void store_cache_cpu( + const at::Tensor& k, + const at::Tensor& v, + const at::Tensor& k_cache, + const at::Tensor& v_cache, + const at::Tensor& indices, + std::optional row_dim) { + CHECK_LAST2_DIM_CONTIGUOUS(k, 3); + CHECK_LAST2_DIM_CONTIGUOUS(v, 3); + CHECK_LAST2_DIM_CONTIGUOUS(k_cache, 3); + CHECK_LAST2_DIM_CONTIGUOUS(v_cache, 3); + CHECK_INPUT(indices); + + int64_t batch_size = k.size(0); + int64_t num_heads = k.size(1); + int64_t head_size = k.size(2); + int64_t num_pages = k_cache.size(0); + int64_t row_dim_value = num_heads * head_size; + if (row_dim.has_value()) { + CHECK_EQ(row_dim.value(), row_dim_value); + } + CHECK_EQ(indices.size(0), batch_size); + + // strides: batch dimension (dim 0) stride in elements + int64_t k_stride = k.stride(0); + int64_t v_stride = v.stride(0); + int64_t kc_stride = k_cache.stride(0); + int64_t vc_stride = v_cache.stride(0); + + const auto dtype = k.scalar_type(); + TORCH_CHECK( + dtype == v.scalar_type() && dtype == k_cache.scalar_type() && dtype == v_cache.scalar_type(), + "store_cache_cpu: input tensors must have the same dtype"); + const auto index_dtype = indices.scalar_type(); + TORCH_CHECK(index_dtype == at::kLong || index_dtype == at::kInt, "indices must be int64 or int32"); + + // dtype : [bfloat16, float16, uint8] for fp8 KV stored as uint8 + // index_dtype : [int64, int32] + AT_DISPATCH_REDUCED_FLOATING_TYPES_AND(at::ScalarType::Byte, dtype, "store_cache_cpu", [&] { + AT_DISPATCH_INDEX_TYPES(index_dtype, "store_cache_cpu_index", [&] { + store_cache_kernel_impl( + k.data_ptr(), + v.data_ptr(), + k_cache.data_ptr(), + v_cache.data_ptr(), + indices.data_ptr(), + batch_size, + num_pages, + row_dim_value, + k_stride, + v_stride, + kc_stride, + vc_stride); + }); + }); +} diff --git a/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp b/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp index e86d249daca1..5ed291986aaa 100644 --- a/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp +++ b/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp @@ -410,6 +410,15 @@ std::tuple image_preprocess_cpu( bool disable_grouping, at::ScalarType out_dtype); +// kvcache +void store_cache_cpu( + const at::Tensor& k, + const at::Tensor& v, + const at::Tensor& k_cache, + const at::Tensor& v_cache, + const at::Tensor& indices, + std::optional row_dim); + // [NOTE] When registering kernels, we should accurately describe the in-place information. // Taking fused_add_rmsnorm_cpu as an example, add `Tensor(a!)` modifier to all tensors that // will be modified in-place to avoid incorrect fusing and execution order on graph mode. @@ -658,6 +667,12 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { "image_std, int patch_size, int temporal_patch_size, int merge_size, bool disable_grouping, ScalarType " "out_dtype) -> (Tensor, Tensor)"); m.impl("image_preprocess_cpu", torch::kCPU, &image_preprocess_cpu); + + // kvcache + m.def( + "store_cache_cpu(Tensor k, Tensor v, Tensor(a!) k_cache, Tensor(a!) v_cache, Tensor indices, int? row_dim) -> " + "()"); + m.impl("store_cache_cpu", torch::kCPU, &store_cache_cpu); } TORCH_LIBRARY_IMPL(sgl_kernel, CatchAll, m) { diff --git a/sgl-kernel/csrc/elementwise/deepseek_v4_topk.cu b/sgl-kernel/csrc/elementwise/deepseek_v4_topk.cu new file mode 100644 index 000000000000..5262af88e0af --- /dev/null +++ b/sgl-kernel/csrc/elementwise/deepseek_v4_topk.cu @@ -0,0 +1,372 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +constexpr uint32_t kMaxTopK = 1024; +constexpr uint32_t kBlockSize = 512; + +#ifdef SGL_TOPK_DYNAMIC_SMEM_BYTES +constexpr size_t kSMEM = static_cast(SGL_TOPK_DYNAMIC_SMEM_BYTES); +#else +constexpr size_t kSMEM = 48 * 1024; // bytes +#endif +static_assert(kSMEM % (2 * sizeof(int32_t)) == 0, "kSMEM must be a multiple of 8 bytes."); + +struct TopKParams { + const float* __restrict__ scores; + const int32_t* __restrict__ seq_lens; + const int32_t* __restrict__ page_table; + int32_t* __restrict__ page_indices; + int32_t* __restrict__ raw_indices; + int64_t score_stride; + int64_t page_table_stride; + uint32_t page_bits; + uint32_t topk; + int64_t output_stride; +}; + +__device__ __forceinline__ uint8_t convert_to_uint8(float x) { + __half h = __float2half_rn(x); + uint16_t bits = __half_as_ushort(h); + uint16_t key = (bits & 0x8000) ? static_cast(~bits) : static_cast(bits | 0x8000); + return static_cast(key >> 8); +} + +__device__ __forceinline__ uint32_t convert_to_uint32(float x) { + uint32_t bits = __float_as_uint(x); + return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u); +} + +__device__ __forceinline__ int32_t +page_to_slot(const int32_t* __restrict__ page_table, uint32_t i, uint32_t page_bits) { + const uint32_t mask = (1u << page_bits) - 1u; + return (page_table[i >> page_bits] << page_bits) | static_cast(i & mask); +} + +__device__ void naive_paged_transform( + int32_t length, + uint32_t topk, + uint32_t page_bits, + const int32_t* __restrict__ page_table, + int32_t* __restrict__ page_indices_out, + int32_t* __restrict__ raw_indices_out) { + for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) { + if (i < static_cast(length)) { + page_indices_out[i] = page_to_slot(page_table, i, page_bits); + if (raw_indices_out != nullptr) { + raw_indices_out[i] = static_cast(i); + } + } else { + page_indices_out[i] = -1; + if (raw_indices_out != nullptr) { + raw_indices_out[i] = -1; + } + } + } +} + +__device__ void +radix_topk(const float* __restrict__ input, int32_t* __restrict__ output, uint32_t length, uint32_t topk) { + constexpr uint32_t RADIX = 256; + constexpr uint32_t BLOCK_SIZE = kBlockSize; + constexpr uint32_t SMEM_INPUT_SIZE = kSMEM / (2 * sizeof(int32_t)); + + alignas(128) __shared__ uint32_t _s_histogram_buf[2][RADIX + 32]; + alignas(128) __shared__ uint32_t s_counter; + alignas(128) __shared__ uint32_t s_threshold_bin_id; + alignas(128) __shared__ uint32_t s_num_input[2]; + alignas(128) __shared__ int32_t s_last_remain; + + extern __shared__ uint32_t s_input_idx[][SMEM_INPUT_SIZE]; + + const uint32_t tx = threadIdx.x; + uint32_t remain_topk = topk; + auto& s_histogram = _s_histogram_buf[0]; + + const auto run_cumsum = [&] { +#pragma unroll 8 + for (int32_t i = 0; i < 8; ++i) { + static_assert(1 << 8 == RADIX); + if (tx < RADIX) { + const auto j = 1 << i; + const auto k = i & 1; + auto value = _s_histogram_buf[k][tx]; + if (tx + j < RADIX) { + value += _s_histogram_buf[k][tx + j]; + } + _s_histogram_buf[k ^ 1][tx] = value; + } + __syncthreads(); + } + }; + + // stage 1: 8bit coarse histogram + if (tx < RADIX + 1) s_histogram[tx] = 0; + __syncthreads(); + for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { + const auto bin = convert_to_uint8(input[idx]); + ::atomicAdd(&s_histogram[bin], 1); + } + __syncthreads(); + run_cumsum(); + if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) { + s_threshold_bin_id = tx; + s_num_input[0] = 0; + s_counter = 0; + } + __syncthreads(); + + { + const auto threshold_bin = s_threshold_bin_id; + remain_topk -= s_histogram[threshold_bin + 1]; + if (remain_topk == 0) { + for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { + const uint32_t bin = convert_to_uint8(input[idx]); + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + output[pos] = static_cast(idx); + } + } + __syncthreads(); + return; + } + __syncthreads(); + if (tx < RADIX + 1) s_histogram[tx] = 0; + __syncthreads(); + + for (uint32_t idx = tx; idx < length; idx += BLOCK_SIZE) { + const float raw_input = input[idx]; + const uint32_t bin = convert_to_uint8(raw_input); + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + output[pos] = static_cast(idx); + } else if (bin == threshold_bin) { + const auto pos = ::atomicAdd(&s_num_input[0], 1); + if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { + s_input_idx[0][pos] = idx; + const auto bin32 = convert_to_uint32(raw_input); + const auto sub_bin = (bin32 >> 24) & 0xFF; + ::atomicAdd(&s_histogram[sub_bin], 1); + } + } + } + __syncthreads(); + } + + // stage 2: refine with 8bit radix passes +#pragma unroll 4 + for (int round = 0; round < 4; ++round) { + const auto r_idx = round % 2; + + const auto raw_num_input = s_num_input[r_idx]; + const auto num_input = raw_num_input < SMEM_INPUT_SIZE ? raw_num_input : SMEM_INPUT_SIZE; + + run_cumsum(); + if (tx < RADIX && s_histogram[tx] > remain_topk && s_histogram[tx + 1] <= remain_topk) { + s_threshold_bin_id = tx; + s_num_input[r_idx ^ 1] = 0; + s_last_remain = static_cast(remain_topk - s_histogram[tx + 1]); + } + __syncthreads(); + + const auto threshold_bin = s_threshold_bin_id; + remain_topk -= s_histogram[threshold_bin + 1]; + + if (remain_topk == 0) { + for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) { + const auto idx = s_input_idx[r_idx][i]; + const auto offset = 24 - round * 8; + const auto bin = (convert_to_uint32(input[idx]) >> offset) & 0xFF; + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + output[pos] = static_cast(idx); + } + } + __syncthreads(); + break; + } + __syncthreads(); + if (tx < RADIX + 1) s_histogram[tx] = 0; + __syncthreads(); + for (uint32_t i = tx; i < num_input; i += BLOCK_SIZE) { + const auto idx = s_input_idx[r_idx][i]; + const auto raw_input = input[idx]; + const auto offset = 24 - round * 8; + const auto bin = (convert_to_uint32(raw_input) >> offset) & 0xFF; + if (bin > threshold_bin) { + const auto pos = ::atomicAdd(&s_counter, 1); + output[pos] = static_cast(idx); + } else if (bin == threshold_bin) { + if (round == 3) { + const auto pos = ::atomicAdd(&s_last_remain, -1); + if (pos > 0) { + output[topk - pos] = static_cast(idx); + } + } else { + const auto pos = ::atomicAdd(&s_num_input[r_idx ^ 1], 1); + if (C10_LIKELY(pos < SMEM_INPUT_SIZE)) { + s_input_idx[r_idx ^ 1][pos] = idx; + const auto bin32 = convert_to_uint32(raw_input); + const auto sub_bin = (bin32 >> (offset - 8)) & 0xFF; + ::atomicAdd(&s_histogram[sub_bin], 1); + } + } + } + } + __syncthreads(); + } +} + +__global__ __launch_bounds__(kBlockSize) void deepseek_v4_topk_transform_kernel(const TopKParams params) { + const auto bid = blockIdx.x; + const auto seq_len = params.seq_lens[bid]; + const auto topk = params.topk; + const auto score_ptr = params.scores + bid * params.score_stride; + const auto page_ptr = params.page_table + bid * params.page_table_stride; + const auto indices_ptr = params.page_indices + bid * params.output_stride; + const auto raw_indices_ptr = + params.raw_indices != nullptr ? params.raw_indices + bid * params.output_stride : nullptr; + + if (seq_len <= static_cast(topk)) { + naive_paged_transform(seq_len, topk, params.page_bits, page_ptr, indices_ptr, raw_indices_ptr); + return; + } + + __shared__ int32_t s_topk_indices[kMaxTopK]; + radix_topk(score_ptr, s_topk_indices, static_cast(seq_len), topk); + + __syncthreads(); + for (uint32_t i = threadIdx.x; i < topk; i += kBlockSize) { + const auto raw = s_topk_indices[i]; + indices_ptr[i] = page_to_slot(page_ptr, static_cast(raw), params.page_bits); + if (raw_indices_ptr != nullptr) { + raw_indices_ptr[i] = raw; + } + } +} + +template +void setup_kernel_smem_once() { + [[maybe_unused]] + static const auto result = [] { +#ifdef USE_ROCM + return ::cudaFuncSetAttribute( + reinterpret_cast(f), ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM); +#else + return ::cudaFuncSetAttribute(f, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM); +#endif + }(); + TORCH_CHECK( + result == cudaSuccess, "deepseek_v4_topk_transform: cudaFuncSetAttribute failed: ", ::cudaGetErrorString(result)); +} + +} // namespace + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +void deepseek_v4_topk_transform_512( + const at::Tensor& scores, + const at::Tensor& seq_lens, + const at::Tensor& page_table, + at::Tensor& page_indices, + int64_t page_size, + std::optional raw_indices_opt) { + CHECK_CUDA(scores); + CHECK_CUDA(seq_lens); + CHECK_CUDA(page_table); + CHECK_CUDA(page_indices); + if (raw_indices_opt.has_value()) { + CHECK_CUDA(raw_indices_opt.value()); + } + + TORCH_CHECK( + scores.dim() == 2 && scores.scalar_type() == at::kFloat, "scores must be float32 with shape [B, max_seq_len]"); + TORCH_CHECK(scores.stride(1) == 1, "scores must be contiguous along the last dim"); + + TORCH_CHECK( + seq_lens.dim() == 1 && seq_lens.is_contiguous() && seq_lens.scalar_type() == at::kInt, + "seq_lens must be int32 with shape [B], contiguous"); + + TORCH_CHECK( + page_table.dim() == 2 && page_table.scalar_type() == at::kInt, + "page_table must be int32 with shape [B, num_pages]"); + TORCH_CHECK(page_table.stride(1) == 1, "page_table must be contiguous along the last dim"); + + const auto topk = page_indices.size(1); + TORCH_CHECK( + page_indices.dim() == 2 && page_indices.is_contiguous() && page_indices.scalar_type() == at::kInt, + "page_indices must be int32 with shape [B, topk], contiguous"); + TORCH_CHECK( + topk > 0 && topk <= static_cast(kMaxTopK), + "page_indices last dim must be in [1, ", + kMaxTopK, + "], got ", + topk); + + const auto B = scores.size(0); + TORCH_CHECK( + seq_lens.size(0) == B && page_table.size(0) == B && page_indices.size(0) == B, + "batch sizes must match across scores, seq_lens, page_table, page_indices"); + + TORCH_CHECK( + page_size > 0 && (page_size & (page_size - 1)) == 0, "page_size must be a positive power of 2, got ", page_size); + const auto page_bits = static_cast(__builtin_ctzll(static_cast(page_size))); + + int32_t* raw_ptr = nullptr; + if (raw_indices_opt.has_value()) { + auto& raw = raw_indices_opt.value(); + TORCH_CHECK( + raw.dim() == 2 && raw.is_contiguous() && raw.scalar_type() == at::kInt, + "raw_indices must be int32 with shape [B, topk], contiguous"); + TORCH_CHECK(raw.size(0) == B && raw.size(1) == topk, "raw_indices shape must match page_indices [B, ", topk, "]"); + raw_ptr = raw.data_ptr(); + } + + const TopKParams params{ + .scores = scores.data_ptr(), + .seq_lens = seq_lens.data_ptr(), + .page_table = page_table.data_ptr(), + .page_indices = page_indices.data_ptr(), + .raw_indices = raw_ptr, + .score_stride = scores.stride(0), + .page_table_stride = page_table.stride(0), + .page_bits = page_bits, + .topk = static_cast(topk), + .output_stride = topk, + }; + + const auto stream = at::cuda::getCurrentCUDAStream().stream(); + const dim3 grid(static_cast(B)); + const dim3 block(kBlockSize); + + setup_kernel_smem_once(); + deepseek_v4_topk_transform_kernel<<>>(params); + + const auto err = cudaGetLastError(); + TORCH_CHECK(err == cudaSuccess, "deepseek_v4_topk_transform kernel launch failed: ", ::cudaGetErrorString(err)); +} diff --git a/sgl-kernel/csrc/elementwise/dsv4_norm_rope.cu b/sgl-kernel/csrc/elementwise/dsv4_norm_rope.cu new file mode 100644 index 000000000000..935278d4b84f --- /dev/null +++ b/sgl-kernel/csrc/elementwise/dsv4_norm_rope.cu @@ -0,0 +1,700 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// DeepSeek-V4 fused norm + RoPE kernels, ported from JIT kernel +// python/sglang/jit_kernel/csrc/deepseek_v4/main_norm_rope.cuh +// to sgl-kernel AOT compilation with CUDA + HIP (ROCm) support. + +#ifndef USE_ROCM +#include +#include +#include +#include +#else +#include +#include +#include +#endif + +#include +#include +#include + +#include + +#include "utils.h" + +// ============================================================================ +// Platform-compatible type aliases +// ============================================================================ +#ifndef USE_ROCM +using bf16_t = __nv_bfloat16; +using bf16x2_t = __nv_bfloat162; +using fp8x2_e4m3_t = __nv_fp8x2_e4m3; +#else +using bf16_t = __hip_bfloat16; +using bf16x2_t = __hip_bfloat162; +using fp8x2_e4m3_t = uint16_t; +#ifndef __grid_constant__ +#define __grid_constant__ +#endif +#endif + +// ============================================================================ +// Utility helpers (inlined, no external header dependency) +// ============================================================================ + +static constexpr uint32_t kWarpSize = 32; + +template +__device__ __forceinline__ float warp_reduce_sum(float val) { +#pragma unroll + for (uint32_t mask = kNumThreads / 2; mask > 0; mask >>= 1) + val += SGLANG_SHFL_XOR_SYNC(FULL_MASK, val, mask); + return val; +} + +__device__ __forceinline__ float warp_reduce_max(float val) { +#pragma unroll + for (uint32_t mask = kWarpSize / 2; mask > 0; mask >>= 1) + val = fmaxf(val, SGLANG_SHFL_XOR_SYNC(FULL_MASK, val, mask)); + return val; +} + +// Aligned vector for coalesced memory access. +template +struct alignas(sizeof(T) * N) AlignedVec { + T data[N]; + __device__ __forceinline__ T& operator[](int i) { + return data[i]; + } + __device__ __forceinline__ T operator[](int i) const { + return data[i]; + } + __device__ __forceinline__ void load(const void* ptr, int64_t offset = 0) { + *this = reinterpret_cast(ptr)[offset]; + } + __device__ __forceinline__ void store(void* ptr, int64_t offset = 0) const { + reinterpret_cast(ptr)[offset] = *this; + } +}; + +__device__ __forceinline__ float bf16_to_float(bf16_t v) { + return __bfloat162float(v); +} + +__device__ __forceinline__ bf16_t float_to_bf16(float v) { +#ifndef USE_ROCM + return __float2bfloat16_rn(v); +#else + return __float2bfloat16(v); +#endif +} + +// ============================================================================ +// FP8 E4M3 helpers (portable CUDA + HIP) +// ============================================================================ + +// UE8M0 scale: round a positive float to the nearest power-of-two +// representable in UE8M0 (unsigned 8-bit exponent, no mantissa). +__device__ __forceinline__ int32_t cast_to_ue8m0(float x) { + uint32_t u = __float_as_uint(x); + int32_t exp = static_cast((u >> 23) & 0xFFu); + uint32_t mant = u & 0x7FFFFFu; + return exp + (mant != 0); +} + +__device__ __forceinline__ float inv_scale_ue8m0(int32_t exp) { + return __uint_as_float(static_cast((127 + 127 - exp) << 23)); +} + +static constexpr float kFP8Max = 448.0f; + +#ifndef USE_ROCM +__device__ __forceinline__ fp8x2_e4m3_t pack_fp8(float x, float y) { + x = fmaxf(fminf(x, kFP8Max), -kFP8Max); + y = fmaxf(fminf(y, kFP8Max), -kFP8Max); + return __nv_fp8x2_e4m3(float2{x, y}); +} +#else +// Software float -> FP8 E4M3 conversion for ROCm +__device__ __forceinline__ uint8_t cvt_float_to_fp8_e4m3(float val) { + constexpr float kMax = kFP8Max; + val = fmaxf(fminf(val, kMax), -kMax); + if (val == 0.0f) return 0; + + uint32_t f32 = __float_as_uint(val); + uint8_t sign = static_cast((f32 >> 24) & 0x80u); + f32 &= 0x7FFFFFFFu; + + int32_t exp32 = static_cast((f32 >> 23) & 0xFFu); + uint32_t mant32 = f32 & 0x7FFFFFu; + + // FP8 E4M3 bias=7, FP32 bias=127, offset=120 + int32_t exp8 = exp32 - 120; + + if (exp8 <= 0) { + mant32 |= 0x800000u; + int32_t shift = 1 - exp8; + if (shift > 24) return sign; + uint32_t shifted = mant32 >> (20 + shift); + uint32_t rbit = (shift <= 23) ? ((mant32 >> (19 + shift)) & 1u) : 0u; + uint32_t sbit = (shift <= 23) ? ((mant32 & ((1u << (19 + shift)) - 1u)) != 0) : 0u; + shifted += (rbit && (sbit || (shifted & 1u))); + return sign | static_cast(shifted & 0x7u); + } + if (exp8 >= 15) return sign | 0x7Eu; + + uint32_t mant3 = (mant32 >> 20) & 0x7u; + uint32_t rbit = (mant32 >> 19) & 1u; + uint32_t sbit = (mant32 & 0x7FFFFu) != 0; + mant3 += (rbit && (sbit || (mant3 & 1u))); + if (mant3 > 7) { + mant3 = 0; + exp8++; + if (exp8 >= 15) return sign | 0x7Eu; + } + return sign | (static_cast(exp8) << 3) | static_cast(mant3); +} + +__device__ __forceinline__ fp8x2_e4m3_t pack_fp8(float x, float y) { + uint8_t x8 = cvt_float_to_fp8_e4m3(x); + uint8_t y8 = cvt_float_to_fp8_e4m3(y); + return static_cast(x8) | (static_cast(y8) << 8); +} +#endif + +// ============================================================================ +// Kernel 1: Fused Q Norm + RoPE +// warp-per-(token, head), rmsnorm-self (no weight) + RoPE + write to q_out. +// ============================================================================ + +namespace { + +constexpr uint32_t kFusedQBlockSize = 128; +constexpr uint32_t kFusedQNumWarps = kFusedQBlockSize / kWarpSize; + +constexpr uint32_t kFusedKBlockSize = 256; +constexpr uint32_t kFusedKNumWarps = kFusedKBlockSize / kWarpSize; + +struct FusedQNormRopeParams { + const void* __restrict__ q_input; + void* __restrict__ q_output; + const float* __restrict__ freqs_cis; + const int32_t* __restrict__ positions; + int64_t q_input_stride_batch; + int64_t q_output_stride_batch; + uint32_t batch_size; + uint32_t num_q_heads; + float eps; +}; + +// Compute the largest power-of-2 vec size that divides both kHeadDim and +// fits in 16 bytes, while also dividing kRopeDim. +template +struct QKernelTraits { + static constexpr int64_t kMaxVecSize = 16 / sizeof(bf16_t); // 8 + // Use kRopeDim/kWarpSize (=2 for kRopeDim=64) as the vec size. + // This guarantees kRopeDim % kVecSize == 0 and works for all head dims + // that are multiples of kWarpSize*kVecSize. + static constexpr int64_t kVecSize = kRopeDim / kWarpSize; // 2 + static constexpr int64_t kLocalSize = kHeadDim / (kWarpSize * kVecSize); + static constexpr uint32_t kRopeSize = kRopeDim / kVecSize; + static_assert(kHeadDim % (kWarpSize * kVecSize) == 0); + static_assert(kRopeDim % kVecSize == 0); + static_assert(kRopeDim == kWarpSize * 2, "1 (real, imag) pair per lane"); +}; + +template +__global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_norm_rope_kernel( + const __grid_constant__ FusedQNormRopeParams params) { + using Traits = QKernelTraits; + constexpr int64_t kVecSize = Traits::kVecSize; + constexpr int64_t kLocalSize = Traits::kLocalSize; + constexpr uint32_t kRopeSize = Traits::kRopeSize; + + using Storage = AlignedVec; + using Float2 = AlignedVec; + + const auto warp_id = threadIdx.x / kWarpSize; + const auto lane_id = threadIdx.x % kWarpSize; + const auto work_id = blockIdx.x * kFusedQNumWarps + warp_id; + + const uint32_t total_works = params.batch_size * params.num_q_heads; + if (work_id >= total_works) return; + + const uint32_t batch_id = work_id / params.num_q_heads; + const uint32_t head_id = work_id % params.num_q_heads; + const auto input_ptr = + static_cast(params.q_input) + batch_id * params.q_input_stride_batch + head_id * kHeadDim; + const auto output_ptr = + static_cast(params.q_output) + batch_id * params.q_output_stride_batch + head_id * kHeadDim; + const auto position = params.positions[batch_id]; + + __shared__ Storage s_rope[kFusedQNumWarps][kRopeSize]; + + // Prefetch freq pair. + Float2 freq; + freq.load(params.freqs_cis + position * kRopeDim, lane_id); + + // Part 1: rmsnorm-self (no weight). + Storage input_vec[kLocalSize]; +#pragma unroll + for (int i = 0; i < kLocalSize; ++i) { + input_vec[i].load(input_ptr, lane_id + i * kWarpSize); + } + + float sum_of_squares = 0.0f; +#pragma unroll + for (int i = 0; i < kLocalSize; ++i) { +#pragma unroll + for (int j = 0; j < kVecSize; ++j) { + float x = bf16_to_float(input_vec[i][j]); + sum_of_squares += x * x; + } + } + sum_of_squares = warp_reduce_sum(sum_of_squares); + const float norm_factor = rsqrtf(sum_of_squares / static_cast(kHeadDim) + params.eps); + +#pragma unroll + for (int i = 0; i < kLocalSize; ++i) { +#pragma unroll + for (int j = 0; j < kVecSize; ++j) { + float x = bf16_to_float(input_vec[i][j]); + input_vec[i][j] = float_to_bf16(x * norm_factor); + } + } + + // Stash rope tail into shared memory; write nope tiles to gmem. + const bool is_rope_lane = lane_id >= kWarpSize - kRopeSize; +#pragma unroll + for (int i = 0; i < kLocalSize; ++i) { + if (i == kLocalSize - 1 && is_rope_lane) { + const auto rope_id = lane_id - (kWarpSize - kRopeSize); + s_rope[warp_id][rope_id] = input_vec[i]; + } else { + input_vec[i].store(output_ptr, lane_id + i * kWarpSize); + } + } + __syncwarp(); + + // Part 2: RoPE on all 32 lanes -- one (real, imag) bf16x2 pair per lane. + auto elem_ptr = reinterpret_cast(&s_rope[warp_id][0]); + bf16x2_t elem = elem_ptr[lane_id]; +#ifndef USE_ROCM + float2 elem_f = __bfloat1622float2(elem); + float x_real = elem_f.x, x_imag = elem_f.y; +#else + float x_real = __bfloat162float(elem.x), x_imag = __bfloat162float(elem.y); +#endif + float freq_real = freq[0], freq_imag = freq[1]; + float rot_real = x_real * freq_real - x_imag * freq_imag; + float rot_imag = x_real * freq_imag + x_imag * freq_real; + bf16x2_t rotated = __float22bfloat162_rn(make_float2(rot_real, rot_imag)); + auto out_elem = reinterpret_cast(output_ptr + (kHeadDim - kRopeDim)); + out_elem[lane_id] = rotated; +} + +// ============================================================================ +// Kernel 2: Fused K Norm + RoPE + FlashMLA Store +// block-per-token, rmsnorm (with kv_weight) + RoPE + FP8 quantized store. +// ============================================================================ + +struct FusedKNormRopeFlashMLAParams { + const void* __restrict__ kv; + const void* __restrict__ kv_weight; + const float* __restrict__ freqs_cis; + const int32_t* __restrict__ positions; + const int32_t* __restrict__ out_loc; + uint8_t* __restrict__ kvcache; + int64_t kv_stride_batch; + uint32_t batch_size; + float eps; +}; + +template +__global__ __launch_bounds__(kFusedKBlockSize, 8) void fused_k_norm_rope_flashmla_kernel( + const __grid_constant__ FusedKNormRopeFlashMLAParams params) { + constexpr int64_t kVecSize = 2; + constexpr uint32_t kRopeWarp = kFusedKNumWarps - 1; + constexpr int64_t kPageBytes = ((584ll << kPageBits) + 575) / 576 * 576; + static_assert(kHeadDim == kFusedKBlockSize * kVecSize); + static_assert(kRopeDim == kWarpSize * kVecSize); + + using Storage = AlignedVec; + + const auto tx = threadIdx.x; + const auto warp_id = tx / kWarpSize; + const auto lane_id = tx % kWarpSize; + const auto work_id = blockIdx.x; + if (work_id >= params.batch_size) return; + + const auto input_ptr = static_cast(params.kv) + work_id * params.kv_stride_batch; + const auto position = params.positions[work_id]; + const auto out_loc = params.out_loc[work_id]; + const auto freqs_cis = params.freqs_cis + position * kRopeDim; + + AlignedVec data, freq; + + // Part 1: norm with block-wide reduction. + { + __shared__ float partial_sums[kFusedKNumWarps]; + + Storage input_vec, weight_vec; + input_vec.load(input_ptr, tx); + weight_vec.load(params.kv_weight, tx); + if (warp_id == kRopeWarp) freq.load(freqs_cis, lane_id); + + float sum_of_squares = 0.0f; +#pragma unroll + for (int i = 0; i < kVecSize; ++i) { + float x = bf16_to_float(input_vec[i]); + sum_of_squares += x * x; + } + const float warp_sum = warp_reduce_sum(sum_of_squares); + if (lane_id == 0) partial_sums[warp_id] = warp_sum; + __syncthreads(); + sum_of_squares = warp_reduce_sum(partial_sums[lane_id % kFusedKNumWarps]); + const float norm_factor = rsqrtf(sum_of_squares / static_cast(kHeadDim) + params.eps); + +#pragma unroll + for (int i = 0; i < kVecSize; ++i) { + float x = bf16_to_float(input_vec[i]); + float w = bf16_to_float(weight_vec[i]); + data[i] = x * norm_factor * w; + } + } + + const int32_t page = out_loc >> kPageBits; + const int32_t offset = out_loc & ((1 << kPageBits) - 1); + const auto page_ptr = params.kvcache + page * kPageBytes; + const auto value_ptr = page_ptr + offset * 576; + + // Part 2: rope on last warp (BF16 store), per-warp UE8M0 quant + store on others. + if (warp_id == kRopeWarp) { + float x_real = data[0], x_imag = data[1]; + float freq_real = freq[0], freq_imag = freq[1]; + float rot_real = x_real * freq_real - x_imag * freq_imag; + float rot_imag = x_real * freq_imag + x_imag * freq_real; + bf16x2_t result = __float22bfloat162_rn(make_float2(rot_real, rot_imag)); + auto rope_ptr = value_ptr + 448; + reinterpret_cast(rope_ptr)[lane_id] = result; + } else { + float x = data[0], y = data[1]; + float abs_max = warp_reduce_max(fmaxf(fabsf(x), fabsf(y))); + float scale_raw = fmaxf(1e-4f, abs_max) / kFP8Max; + int32_t scale_ue8m0 = cast_to_ue8m0(scale_raw); + float inv_scale = inv_scale_ue8m0(scale_ue8m0); + fp8x2_e4m3_t result = pack_fp8(x * inv_scale, y * inv_scale); + auto scale_ptr = page_ptr + (576ll << kPageBits) + offset * 8; + reinterpret_cast(value_ptr)[tx] = result; + if (lane_id == 0) static_cast(scale_ptr)[warp_id] = static_cast(scale_ue8m0); + } +} + +// ============================================================================ +// Kernel 3: Fused Q Indexer RoPE + Hadamard + FP8 Quantization +// warp-per-(token, head), no norm, RoPE + Hadamard + fp8 act-quant. +// ============================================================================ + +struct FusedQIndexerRopeHadamardQuantParams { + const void* __restrict__ q_input; + void* __restrict__ q_fp8; + const void* __restrict__ weight; + float* __restrict__ weights_out; + float weight_scale; + const float* __restrict__ freqs_cis; + const int32_t* __restrict__ positions; + uint32_t batch_size; + uint32_t num_heads; +}; + +__global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_indexer_rope_hadamard_quant_kernel( + const __grid_constant__ FusedQIndexerRopeHadamardQuantParams params) { + constexpr int64_t kHeadDim = 128; + constexpr int64_t kRopeDim = 64; + constexpr int64_t kVecSize = 4; + constexpr uint32_t kRopeSize = kRopeDim / kVecSize; + static_assert(kHeadDim == kWarpSize * kVecSize); + + using Storage = AlignedVec; + using Float4 = AlignedVec; + using OutStorage = AlignedVec; + + const auto warp_id = threadIdx.x / kWarpSize; + const auto lane_id = threadIdx.x % kWarpSize; + const auto work_id = blockIdx.x * kFusedQNumWarps + warp_id; + const bool is_rope_lane = lane_id >= kWarpSize - kRopeSize; + + const uint32_t total_works = params.batch_size * params.num_heads; + if (work_id >= total_works) return; + + const uint32_t batch_id = work_id / params.num_heads; + const auto input_ptr = static_cast(params.q_input) + work_id * kHeadDim; + const auto position = params.positions[batch_id]; + const auto freqs_cis = params.freqs_cis + position * kRopeDim; + + Float4 data, freq; + const float weight_val = bf16_to_float(static_cast(params.weight)[work_id]); + + // Part 1: load (no norm). + { + Storage input_vec; + input_vec.load(input_ptr, lane_id); + if (is_rope_lane) freq.load(freqs_cis, lane_id - (kWarpSize - kRopeSize)); +#pragma unroll + for (int i = 0; i < kVecSize; ++i) + data[i] = bf16_to_float(input_vec[i]); + } + + // Part 2: rope on rope lanes. + if (is_rope_lane) { + float x_r = data[0], x_i = data[1], y_r = data[2], y_i = data[3]; + float fxr = freq[0], fxi = freq[1], fyr = freq[2], fyi = freq[3]; + data[0] = x_r * fxr - x_i * fxi; + data[1] = x_r * fxi + x_i * fxr; + data[2] = y_r * fyr - y_i * fyi; + data[3] = y_r * fyi + y_i * fyr; + } + + // Part 3: 128-point Hadamard (2 local + 5 cross-lane stages). + { + { + float a0 = data[0], a1 = data[1], a2 = data[2], a3 = data[3]; + data[0] = a0 + a1; + data[1] = a0 - a1; + data[2] = a2 + a3; + data[3] = a2 - a3; + } + { + float a0 = data[0], a1 = data[1], a2 = data[2], a3 = data[3]; + data[0] = a0 + a2; + data[1] = a1 + a3; + data[2] = a0 - a2; + data[3] = a1 - a3; + } +#pragma unroll + for (uint32_t mask = 1; mask < kWarpSize; mask <<= 1) { +#pragma unroll + for (int i = 0; i < kVecSize; ++i) { + float other = SGLANG_SHFL_XOR_SYNC_WIDTH(FULL_MASK, data[i], mask, kWarpSize); + data[i] = (lane_id & mask) ? (other - data[i]) : (data[i] + other); + } + } + const float kHadamardScale = rsqrtf(static_cast(kHeadDim)); +#pragma unroll + for (int i = 0; i < kVecSize; ++i) + data[i] *= kHadamardScale; + } + + // Part 4: per-warp FP8 quant + store. + { + float local_max = fabsf(data[0]); +#pragma unroll + for (int i = 1; i < kVecSize; ++i) + local_max = fmaxf(local_max, fabsf(data[i])); + float abs_max = warp_reduce_max(local_max); + float scale = fmaxf(1e-4f, abs_max) / kFP8Max; + float inv_scale = 1.0f / scale; + + OutStorage result; + result[0] = pack_fp8(data[0] * inv_scale, data[1] * inv_scale); + result[1] = pack_fp8(data[2] * inv_scale, data[3] * inv_scale); + + auto out_row = static_cast(params.q_fp8) + work_id * kHeadDim; + result.store(out_row, lane_id); + params.weights_out[work_id] = weight_val * params.weight_scale * scale; + } +} + +} // anonymous namespace + +// ============================================================================ +// Host-side launchers (PyTorch C++ extension API) +// ============================================================================ + +void dsv4_fused_q_norm_rope( + const at::Tensor& q_input, + at::Tensor& q_output, + const at::Tensor& freqs_cis, + const at::Tensor& positions, + double eps) { + TORCH_CHECK(q_input.is_cuda(), "q_input must be a CUDA tensor"); + TORCH_CHECK(q_output.is_cuda(), "q_output must be a CUDA tensor"); + TORCH_CHECK(q_input.scalar_type() == at::ScalarType::BFloat16, "q_input must be bfloat16"); + TORCH_CHECK(q_output.scalar_type() == at::ScalarType::BFloat16, "q_output must be bfloat16"); + TORCH_CHECK(q_input.dim() == 3, "q_input must be 3D: (B, H, D)"); + TORCH_CHECK(q_output.dim() == 3, "q_output must be 3D: (B, H, D)"); + TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32"); + + const int64_t B = q_input.size(0); + const int64_t H = q_input.size(1); + const int64_t D = q_input.size(2); + TORCH_CHECK( + q_output.size(0) == B && q_output.size(1) == H && q_output.size(2) == D, "q_output shape must match q_input"); + TORCH_CHECK(q_input.stride(2) == 1 && q_output.stride(2) == 1, "last dim must be contiguous"); + TORCH_CHECK(q_input.stride(1) == D && q_output.stride(1) == D, "head dim must be contiguous"); + + if (B == 0) return; + + const auto stream = at::cuda::getCurrentCUDAStream(q_input.get_device()); + const auto params = FusedQNormRopeParams{ + .q_input = q_input.data_ptr(), + .q_output = q_output.data_ptr(), + .freqs_cis = freqs_cis.data_ptr(), + .positions = positions.data_ptr(), + .q_input_stride_batch = q_input.stride(0), + .q_output_stride_batch = q_output.stride(0), + .batch_size = static_cast(B), + .num_q_heads = static_cast(H), + .eps = static_cast(eps), + }; + const uint32_t total_works = static_cast(B * H); + const uint32_t num_blocks = CEILDIV(total_works, kFusedQNumWarps); + + // Dispatch on head_dim. DeepSeek V4 uses D=192 with kRopeDim=64. + constexpr int64_t kRopeDim = 64; + switch (D) { + case 128: + fused_q_norm_rope_kernel<128, kRopeDim><<>>(params); + break; + case 192: + fused_q_norm_rope_kernel<192, kRopeDim><<>>(params); + break; + default: + TORCH_CHECK(false, "Unsupported head_dim for dsv4_fused_q_norm_rope: ", D); + } +} + +void dsv4_fused_k_norm_rope_flashmla( + const at::Tensor& kv, + const at::Tensor& kv_weight, + const at::Tensor& freqs_cis, + const at::Tensor& positions, + const at::Tensor& out_loc, + at::Tensor& kvcache, + double eps, + int64_t page_size) { + TORCH_CHECK(kv.is_cuda(), "kv must be a CUDA tensor"); + TORCH_CHECK(kv.scalar_type() == at::ScalarType::BFloat16, "kv must be bfloat16"); + TORCH_CHECK(kv.dim() == 2, "kv must be 2D: (B, D)"); + TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32"); + TORCH_CHECK(out_loc.scalar_type() == at::ScalarType::Int, "out_loc must be int32"); + + const int64_t B = kv.size(0); + const int64_t D = kv.size(1); + TORCH_CHECK(D == 512, "kv head_dim must be 512 for FlashMLA"); + TORCH_CHECK(kv_weight.size(0) == D, "kv_weight size must match head_dim"); + + if (B == 0) return; + + const auto stream = at::cuda::getCurrentCUDAStream(kv.get_device()); + const auto params = FusedKNormRopeFlashMLAParams{ + .kv = kv.data_ptr(), + .kv_weight = kv_weight.data_ptr(), + .freqs_cis = freqs_cis.data_ptr(), + .positions = positions.data_ptr(), + .out_loc = out_loc.data_ptr(), + .kvcache = static_cast(kvcache.data_ptr()), + .kv_stride_batch = kv.stride(0), + .batch_size = static_cast(B), + .eps = static_cast(eps), + }; + + constexpr int64_t kHeadDim = 512; + constexpr int64_t kRopeDim = 64; + + // Dispatch on page_size (must be power of 2). + TORCH_CHECK(page_size > 0 && (page_size & (page_size - 1)) == 0, "page_size must be a power of 2"); + +#define LAUNCH_K_KERNEL(PAGE_BITS) \ + fused_k_norm_rope_flashmla_kernel \ + <<(B), kFusedKBlockSize, 0, stream>>>(params) + + switch (page_size) { + case 1: + LAUNCH_K_KERNEL(0); + break; + case 2: + LAUNCH_K_KERNEL(1); + break; + case 4: + LAUNCH_K_KERNEL(2); + break; + case 8: + LAUNCH_K_KERNEL(3); + break; + case 16: + LAUNCH_K_KERNEL(4); + break; + case 32: + LAUNCH_K_KERNEL(5); + break; + case 64: + LAUNCH_K_KERNEL(6); + break; + case 128: + LAUNCH_K_KERNEL(7); + break; + case 256: + LAUNCH_K_KERNEL(8); + break; + default: + TORCH_CHECK(false, "Unsupported page_size: ", page_size); + } +#undef LAUNCH_K_KERNEL +} + +void dsv4_fused_q_indexer_rope_hadamard_quant( + const at::Tensor& q_input, + at::Tensor& q_fp8, + const at::Tensor& weight, + at::Tensor& weights_out, + double weight_scale, + const at::Tensor& freqs_cis, + const at::Tensor& positions) { + TORCH_CHECK(q_input.is_cuda(), "q_input must be a CUDA tensor"); + TORCH_CHECK(q_input.scalar_type() == at::ScalarType::BFloat16, "q_input must be bfloat16"); + TORCH_CHECK(q_input.dim() == 3, "q_input must be 3D: (B, H, D)"); + + const int64_t B = q_input.size(0); + const int64_t H = q_input.size(1); + constexpr int64_t kHeadDim = 128; + TORCH_CHECK(q_input.size(2) == kHeadDim, "q_input head_dim must be 128 for indexer"); + TORCH_CHECK( + q_input.stride(2) == 1 && q_input.stride(1) == kHeadDim, "q_input must be contiguous in (head, elem) dims"); + TORCH_CHECK(q_input.stride(0) == H * kHeadDim, "q_input must be contiguous (B, H, D)"); + TORCH_CHECK(q_fp8.stride(0) == H * kHeadDim, "q_fp8 must be contiguous (B, H, D)"); + TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32"); + + if (B == 0) return; + + const auto stream = at::cuda::getCurrentCUDAStream(q_input.get_device()); + const auto params = FusedQIndexerRopeHadamardQuantParams{ + .q_input = q_input.data_ptr(), + .q_fp8 = q_fp8.data_ptr(), + .weight = weight.data_ptr(), + .weights_out = weights_out.data_ptr(), + .weight_scale = static_cast(weight_scale), + .freqs_cis = freqs_cis.data_ptr(), + .positions = positions.data_ptr(), + .batch_size = static_cast(B), + .num_heads = static_cast(H), + }; + const uint32_t total_works = static_cast(B * H); + const uint32_t num_blocks = CEILDIV(total_works, kFusedQNumWarps); + + fused_q_indexer_rope_hadamard_quant_kernel<<>>(params); +} diff --git a/sgl-kernel/csrc/flashmla_extension.cc b/sgl-kernel/csrc/flashmla_extension.cc index 12b09524a0b6..b9f2fe00357c 100644 --- a/sgl-kernel/csrc/flashmla_extension.cc +++ b/sgl-kernel/csrc/flashmla_extension.cc @@ -16,8 +16,61 @@ limitations under the License. #include #include +#include "api/dense_decode.h" +#include "api/sparse_decode.h" +#include "api/sparse_fwd.h" #include "sgl_kernel_ops.h" +static std::tuple, std::optional> sgl_sparse_decode_fwd( + const at::Tensor& q, + const at::Tensor& kv, + const at::Tensor& indices, + const std::optional& topk_length, + const std::optional& attn_sink, + std::optional tile_scheduler_metadata, + std::optional num_splits, + const std::optional& extra_kv, + const std::optional& extra_indices, + const std::optional& extra_topk_length, + int64_t d_v, + double sm_scale) { + return sparse_attn_decode_interface( + q, + kv, + indices, + topk_length, + attn_sink, + tile_scheduler_metadata, + num_splits, + extra_kv, + extra_indices, + extra_topk_length, + static_cast(d_v), + static_cast(sm_scale)); +} + +static std::tuple, std::optional> sgl_dense_decode_fwd( + at::Tensor q, + const at::Tensor& kcache, + int64_t head_size_v, + const at::Tensor& seqlens_k, + const at::Tensor& block_table, + double softmax_scale, + bool is_causal, + std::optional tile_scheduler_metadata, + std::optional num_splits) { + return dense_attn_decode_interface( + q, + kcache, + static_cast(head_size_v), + seqlens_k, + block_table, + static_cast(softmax_scale), + is_causal, + tile_scheduler_metadata, + num_splits); +} + TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { /* * From FlashMLA @@ -32,7 +85,9 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.def( "fwd_kvcache_mla(Tensor q, Tensor kv_cache, int head_size_v, Tensor seqlens_k, Tensor block_table, float " - "softmax_scale, bool is_causal, Tensor tile_scheduler_metadata, Tensor num_splits, bool is_fp8, Tensor? indices) " + "softmax_scale, bool is_causal, Tensor tile_scheduler_metadata, Tensor num_splits, bool is_fp8, Tensor? indices, " + "Tensor? attn_sink, Tensor? extra_k_cache, Tensor? extra_indices_in_kvcache, Tensor? topk_length, Tensor? " + "extra_topk_length) " "-> Tensor[]"); m.impl("fwd_kvcache_mla", torch::kCUDA, &fwd_kvcache_mla); @@ -44,7 +99,22 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.impl("dense_prefill_fwd", torch::kCUDA, &FMHACutlassSM100FwdRun); #endif - m.def("sparse_prefill_fwd(Tensor q, Tensor kv, Tensor indices, float sm_scale, int d_v) -> Tensor[]"); + m.def( + "sparse_decode_fwd(Tensor q, Tensor kv, Tensor indices, Tensor? topk_length, Tensor? attn_sink, " + "Tensor? tile_scheduler_metadata, Tensor? num_splits, Tensor? extra_kv, Tensor? extra_indices, " + "Tensor? extra_topk_length, int d_v, float sm_scale) -> (Tensor, Tensor, Tensor?, Tensor?)"); + m.impl("sparse_decode_fwd", torch::kCUDA, &sgl_sparse_decode_fwd); + + m.def( + "dense_decode_fwd(Tensor q, Tensor kcache, int head_size_v, Tensor seqlens_k, Tensor block_table, float " + "softmax_scale, bool is_causal, Tensor? tile_scheduler_metadata, Tensor? num_splits) -> (Tensor, Tensor, " + "Tensor?, " + "Tensor?)"); + m.impl("dense_decode_fwd", torch::kCUDA, &sgl_dense_decode_fwd); + + m.def( + "sparse_prefill_fwd(Tensor q, Tensor kv, Tensor indices, float sm_scale, int d_v, Tensor? attn_sink=None, " + "Tensor? topk_length=None) -> Tensor[]"); m.impl("sparse_prefill_fwd", torch::kCUDA, &sparse_prefill_fwd); m.def( diff --git a/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu index 82f8b89fcf95..44cf9b3148c9 100644 --- a/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu +++ b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu @@ -696,6 +696,9 @@ void topkGatingSoftmaxKernelLauncher( case 256: LAUNCH_SOFTMAX(T, 256, WARPS_PER_TB); break; + case 512: + LAUNCH_SOFTMAX(T, 512, WARPS_PER_TB); + break; default: { TORCH_CHECK( softmax_workspace != nullptr, @@ -751,7 +754,7 @@ void topk_softmax( const int topk = static_cast(topk_weights.size(-1)); const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0); - const bool needs_workspace = !is_pow_2 || num_experts > 256; + const bool needs_workspace = !is_pow_2 || num_experts > 512; const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); diff --git a/sgl-kernel/include/sgl_kernel_ops.h b/sgl-kernel/include/sgl_kernel_ops.h index 5c0261b643df..b162520076d7 100644 --- a/sgl-kernel/include/sgl_kernel_ops.h +++ b/sgl-kernel/include/sgl_kernel_ops.h @@ -22,6 +22,7 @@ limitations under the License. #include #include +#include #include #include @@ -172,8 +173,45 @@ void fast_topk_transform_ragged_interface( #ifdef USE_ROCM void gelu_quick(at::Tensor& out, const at::Tensor& input); + +void deepseek_v4_topk_transform_512( + const at::Tensor& scores, + const at::Tensor& seq_lens, + const at::Tensor& page_table, + at::Tensor& page_indices, + int64_t page_size, + std::optional raw_indices_opt = std::nullopt); #endif +/* + * From csrc/elementwise (DeepSeek-V4 norm + rope) + */ +void dsv4_fused_q_norm_rope( + const at::Tensor& q_input, + at::Tensor& q_output, + const at::Tensor& freqs_cis, + const at::Tensor& positions, + double eps); + +void dsv4_fused_k_norm_rope_flashmla( + const at::Tensor& kv, + const at::Tensor& kv_weight, + const at::Tensor& freqs_cis, + const at::Tensor& positions, + const at::Tensor& out_loc, + at::Tensor& kvcache, + double eps, + int64_t page_size); + +void dsv4_fused_q_indexer_rope_hadamard_quant( + const at::Tensor& q_input, + at::Tensor& q_fp8, + const at::Tensor& weight, + at::Tensor& weights_out, + double weight_scale, + const at::Tensor& freqs_cis, + const at::Tensor& positions); + /* * From csrc/gemm */ @@ -331,6 +369,35 @@ void apply_shuffle_mul_sum( const torch::Tensor& permutation, const std::optional& factors); +/* + * From csrc/elementwise (DeepSeek-V4 norm + rope) + */ +void dsv4_fused_q_norm_rope( + const at::Tensor& q_input, + at::Tensor& q_output, + const at::Tensor& freqs_cis, + const at::Tensor& positions, + double eps); + +void dsv4_fused_k_norm_rope_flashmla( + const at::Tensor& kv, + const at::Tensor& kv_weight, + const at::Tensor& freqs_cis, + const at::Tensor& positions, + const at::Tensor& out_loc, + at::Tensor& kvcache, + double eps, + int64_t page_size); + +void dsv4_fused_q_indexer_rope_hadamard_quant( + const at::Tensor& q_input, + at::Tensor& q_fp8, + const at::Tensor& weight, + at::Tensor& weights_out, + double weight_scale, + const at::Tensor& freqs_cis, + const at::Tensor& positions); + void fused_qk_norm_rope( torch::Tensor& qkv, int64_t num_heads_q, @@ -811,8 +878,12 @@ std::vector fwd_kvcache_mla( const at::Tensor& tile_scheduler_metadata, // num_sm_parts x TileSchedulerMetaDataSize const at::Tensor& num_splits, // batch_size + 1 const bool& is_fp8, - const std::optional& indices // None, or batch_size x seqlen_q x topk -); + const std::optional& indices, // None, or batch_size x seqlen_q x topk + const std::optional& attn_sink, + const std::optional& extra_k_cache, + const std::optional& extra_indices_in_kvcache, + const std::optional& topk_length, + const std::optional& extra_topk_length); void FMHACutlassSM100FwdRun( at::Tensor workspace_buffer, @@ -829,8 +900,14 @@ void FMHACutlassSM100FwdRun( int64_t max_seqlen_kv, bool is_varlen); -std::vector -sparse_prefill_fwd(const at::Tensor& q, const at::Tensor& kv, const at::Tensor& indices, double sm_scale, int64_t d_v); +std::vector sparse_prefill_fwd( + const at::Tensor& q, + const at::Tensor& kv, + const at::Tensor& indices, + double sm_scale, + int64_t d_v, + const std::optional& attn_sink, + const std::optional& topk_length); std::vector fwd_kvcache_mla_fp8( at::Tensor& q, // batch_size x seqlen_q x num_heads x head_size diff --git a/sgl-kernel/pyproject.toml b/sgl-kernel/pyproject.toml index 4b263f07e6d5..cb2877fe4883 100644 --- a/sgl-kernel/pyproject.toml +++ b/sgl-kernel/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "scikit_build_core.build" [project] name = "sglang-kernel" -version = "0.4.2.post2" +version = "0.4.3" authors = [ { name="SGLang Kernel Team", email="sglang@lmsys.org" }, ] diff --git a/sgl-kernel/pyproject_cpu.toml b/sgl-kernel/pyproject_cpu.toml index 3e17322e8828..a3e684fc2f6a 100644 --- a/sgl-kernel/pyproject_cpu.toml +++ b/sgl-kernel/pyproject_cpu.toml @@ -8,7 +8,7 @@ build-backend = "scikit_build_core.build" [project] name = "sglang-kernel-cpu" -version = "0.4.2.post2" +version = "0.4.3" description = "Kernel Library for SGLang" readme = "README.md" requires-python = ">=3.10" diff --git a/sgl-kernel/pyproject_musa.toml b/sgl-kernel/pyproject_musa.toml index c9b7907607bb..c7c182d3e7de 100644 --- a/sgl-kernel/pyproject_musa.toml +++ b/sgl-kernel/pyproject_musa.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "sglang-kernel" -version = "0.4.2.post2" +version = "0.4.3" description = "Kernel Library for SGLang" readme = "README.md" requires-python = ">=3.10" diff --git a/sgl-kernel/pyproject_rocm.toml b/sgl-kernel/pyproject_rocm.toml index ecb4c5804688..1478bbf34125 100644 --- a/sgl-kernel/pyproject_rocm.toml +++ b/sgl-kernel/pyproject_rocm.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "sglang-kernel" -version = "0.4.2.post2" +version = "0.4.3" description = "Kernel Library for SGLang" readme = "README.md" requires-python = ">=3.10" diff --git a/sgl-kernel/python/sgl_kernel/__init__.py b/sgl-kernel/python/sgl_kernel/__init__.py index 1b97271f2282..ce6e9d049760 100644 --- a/sgl-kernel/python/sgl_kernel/__init__.py +++ b/sgl-kernel/python/sgl_kernel/__init__.py @@ -36,6 +36,9 @@ concat_mla_absorb_q, concat_mla_k, copy_to_gpu_no_ce, + dsv4_fused_k_norm_rope_flashmla, + dsv4_fused_q_indexer_rope_hadamard_quant, + dsv4_fused_q_norm_rope, fused_add_rmsnorm, gelu_and_mul, gelu_tanh_and_mul, @@ -125,6 +128,7 @@ if torch.version.hip is not None: from sgl_kernel.elementwise import gelu_quick + from sgl_kernel.top_k import deepseek_v4_topk_transform_512 if hasattr(torch.version, "musa") and torch.version.musa is not None: from sgl_kernel.musa import ( @@ -152,6 +156,9 @@ "cutlass_mla_get_workspace_size", "dsv3_fused_a_gemm", "dsv3_router_gemm", + "dsv4_fused_k_norm_rope_flashmla", + "dsv4_fused_q_indexer_rope_hadamard_quant", + "dsv4_fused_q_norm_rope", "es_fp8_blockwise_scaled_grouped_mm", "es_sm100_mxfp8_blockscaled_grouped_mm", "es_sm100_mxfp8_blockscaled_grouped_quant", @@ -205,6 +212,7 @@ if torch.version.hip is not None: _DEBUG_EXPORT_NAMES.append("gelu_quick") + _DEBUG_EXPORT_NAMES.append("deepseek_v4_topk_transform_512") for _name in _DEBUG_EXPORT_NAMES: if _name in globals(): diff --git a/sgl-kernel/python/sgl_kernel/elementwise.py b/sgl-kernel/python/sgl_kernel/elementwise.py index 48fa5584c6b5..aa325b2779d6 100644 --- a/sgl-kernel/python/sgl_kernel/elementwise.py +++ b/sgl-kernel/python/sgl_kernel/elementwise.py @@ -324,6 +324,85 @@ def gelu_quick(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: return out +def dsv4_fused_q_norm_rope( + q_input: torch.Tensor, + freqs_cis: torch.Tensor, + positions: torch.Tensor, + eps: float = 1e-6, + q_output: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """DeepSeek-V4 fused Q RMSNorm (no weight) + RoPE. + + Parameters + ---------- + q_input : (B, num_q_heads, head_dim) bfloat16 + freqs_cis: (max_pos, rope_dim) float32, re/im interleaved + positions: (B,) int32 + eps : RMSNorm epsilon + q_output : optional pre-allocated output tensor + """ + if q_output is None: + q_output = torch.empty_like(q_input) + torch.ops.sgl_kernel.dsv4_fused_q_norm_rope.default( + q_input, q_output, freqs_cis, positions, eps + ) + return q_output + + +def dsv4_fused_k_norm_rope_flashmla( + kv: torch.Tensor, + kv_weight: torch.Tensor, + freqs_cis: torch.Tensor, + positions: torch.Tensor, + out_loc: torch.Tensor, + kvcache: torch.Tensor, + eps: float = 1e-6, + page_size: int = 1, +) -> None: + """DeepSeek-V4 fused K RMSNorm + RoPE + FlashMLA FP8 store. + + Parameters + ---------- + kv : (B, 512) bfloat16 + kv_weight: (512,) bfloat16 + freqs_cis: (max_pos, 64) float32 + positions: (B,) int32 + out_loc : (B,) int32 cache slot ids + kvcache : (npages, page_bytes) uint8 + eps : RMSNorm epsilon + page_size: page size (power of 2) + """ + torch.ops.sgl_kernel.dsv4_fused_k_norm_rope_flashmla.default( + kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, page_size + ) + + +def dsv4_fused_q_indexer_rope_hadamard_quant( + q_input: torch.Tensor, + q_fp8: torch.Tensor, + weight: torch.Tensor, + weights_out: torch.Tensor, + weight_scale: float, + freqs_cis: torch.Tensor, + positions: torch.Tensor, +) -> None: + """DeepSeek-V4 fused Q indexer: RoPE + Hadamard + FP8 quant. + + Parameters + ---------- + q_input : (B, num_heads, 128) bfloat16 + q_fp8 : (B, num_heads, 128) fp8_e4m3 output + weight : (B, num_heads) bfloat16 + weights_out: (B, num_heads, 1) float32 output + weight_scale: scalar + freqs_cis : (max_pos, 64) float32 + positions : (B,) int32 + """ + torch.ops.sgl_kernel.dsv4_fused_q_indexer_rope_hadamard_quant.default( + q_input, q_fp8, weight, weights_out, weight_scale, freqs_cis, positions + ) + + def rotary_embedding( positions: torch.Tensor, query: torch.Tensor, diff --git a/sgl-kernel/python/sgl_kernel/flash_mla.py b/sgl-kernel/python/sgl_kernel/flash_mla.py index 3dd062ac6d61..df24e8f68f84 100644 --- a/sgl-kernel/python/sgl_kernel/flash_mla.py +++ b/sgl-kernel/python/sgl_kernel/flash_mla.py @@ -1,4 +1,5 @@ # ENGRAM_MODIFIED — Kernel fork tweak +import dataclasses from typing import Optional, Tuple import torch @@ -16,10 +17,33 @@ ) +@dataclasses.dataclass +class FlashMLASchedMeta: + """Tile scheduler metadata for the newer FlashMLA Python API.""" + + @dataclasses.dataclass + class Config: + b: int + s_q: int + h_q: int + page_block_size: int + h_k: int + causal: bool + is_fp8_kvcache: bool + topk: Optional[int] + extra_page_block_size: Optional[int] + extra_topk: Optional[int] + + have_initialized: bool = False + config: Optional[Config] = None + tile_scheduler_metadata: Optional[torch.Tensor] = None + num_splits: Optional[torch.Tensor] = None + + def get_mla_metadata( - cache_seqlens: torch.Tensor, - num_q_tokens_per_head_k: int, - num_heads_k: int, + cache_seqlens: Optional[torch.Tensor] = None, + num_q_tokens_per_head_k: Optional[int] = None, + num_heads_k: Optional[int] = None, num_heads_q: Optional[int] = None, is_fp8_kvcache: bool = False, topk: Optional[int] = None, @@ -40,6 +64,12 @@ def get_mla_metadata( if _flashmla_import_error is not None: raise _IMPORT_ERROR from _flashmla_import_error + if cache_seqlens is None: + return FlashMLASchedMeta(), None + + assert num_q_tokens_per_head_k is not None + assert num_heads_k is not None + if is_fp8_kvcache and topk is None: return torch.ops.sgl_kernel.get_mla_decoding_metadata_dense_fp8.default( cache_seqlens, @@ -59,17 +89,22 @@ def get_mla_metadata( def flash_mla_with_kvcache( q: torch.Tensor, k_cache: torch.Tensor, - block_table: torch.Tensor, - cache_seqlens: torch.Tensor, + block_table: Optional[torch.Tensor], + cache_seqlens: Optional[torch.Tensor], head_dim_v: int, - tile_scheduler_metadata: torch.Tensor, - num_splits: torch.Tensor, + tile_scheduler_metadata: torch.Tensor | FlashMLASchedMeta, + num_splits: Optional[torch.Tensor] = None, softmax_scale: Optional[float] = None, causal: bool = False, descale_q: torch.Tensor | None = None, descale_k: torch.Tensor | None = None, is_fp8_kvcache: bool = False, indices: Optional[torch.Tensor] = None, + attn_sink: Optional[torch.Tensor] = None, + extra_k_cache: Optional[torch.Tensor] = None, + extra_indices_in_kvcache: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + extra_topk_length: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """ Arguments: @@ -96,6 +131,34 @@ def flash_mla_with_kvcache( if softmax_scale is None: softmax_scale = q.shape[-1] ** (-0.5) + if isinstance(tile_scheduler_metadata, FlashMLASchedMeta): + return _flash_mla_with_kvcache_sched_meta( + q=q, + k_cache=k_cache, + block_table=block_table, + cache_seqlens=cache_seqlens, + head_dim_v=head_dim_v, + sched_meta=tile_scheduler_metadata, + num_splits=num_splits, + softmax_scale=softmax_scale, + causal=causal, + is_fp8_kvcache=is_fp8_kvcache, + indices=indices, + attn_sink=attn_sink, + extra_k_cache=extra_k_cache, + extra_indices_in_kvcache=extra_indices_in_kvcache, + topk_length=topk_length, + extra_topk_length=extra_topk_length, + ) + + assert num_splits is not None + assert block_table is not None + assert cache_seqlens is not None + assert attn_sink is None + assert extra_k_cache is None + assert extra_indices_in_kvcache is None + assert topk_length is None + assert extra_topk_length is None if indices is not None: assert causal == False, "causal must be `false` if sparse attention is enabled." assert (descale_q is None) == ( @@ -129,16 +192,131 @@ def flash_mla_with_kvcache( num_splits, is_fp8_kvcache, indices, + attn_sink, + extra_k_cache, + extra_indices_in_kvcache, + topk_length, + extra_topk_length, ) return out, softmax_lse +def _flash_mla_with_kvcache_sched_meta( + q: torch.Tensor, + k_cache: torch.Tensor, + block_table: Optional[torch.Tensor], + cache_seqlens: Optional[torch.Tensor], + head_dim_v: int, + sched_meta: FlashMLASchedMeta, + num_splits: Optional[torch.Tensor], + softmax_scale: float, + causal: bool, + is_fp8_kvcache: bool, + indices: Optional[torch.Tensor], + attn_sink: Optional[torch.Tensor], + extra_k_cache: Optional[torch.Tensor], + extra_indices_in_kvcache: Optional[torch.Tensor], + topk_length: Optional[torch.Tensor], + extra_topk_length: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + assert num_splits is None, "num_splits must be None with FlashMLASchedMeta" + + topk = indices.shape[-1] if indices is not None else None + extra_page_block_size = ( + extra_k_cache.shape[1] if extra_k_cache is not None else None + ) + extra_topk = ( + extra_indices_in_kvcache.shape[-1] + if extra_indices_in_kvcache is not None + else None + ) + + if not sched_meta.have_initialized: + sched_meta.have_initialized = True + sched_meta.config = FlashMLASchedMeta.Config( + b=q.shape[0], + s_q=q.shape[1], + h_q=q.shape[2], + page_block_size=k_cache.shape[1], + h_k=k_cache.shape[2], + causal=causal, + is_fp8_kvcache=is_fp8_kvcache, + topk=topk, + extra_page_block_size=extra_page_block_size, + extra_topk=extra_topk, + ) + else: + helper_msg = ( + " Input arguments are inconsistent with FlashMLASchedMeta. Reuse a " + "scheduler only for matching tensor shapes and sparse settings." + ) + assert sched_meta.config is not None + assert sched_meta.config.b == q.shape[0], helper_msg + assert sched_meta.config.s_q == q.shape[1], helper_msg + assert sched_meta.config.h_q == q.shape[2], helper_msg + assert sched_meta.config.page_block_size == k_cache.shape[1], helper_msg + assert sched_meta.config.h_k == k_cache.shape[2], helper_msg + assert sched_meta.config.causal == causal, helper_msg + assert sched_meta.config.is_fp8_kvcache == is_fp8_kvcache, helper_msg + assert sched_meta.config.topk == topk, helper_msg + assert ( + sched_meta.config.extra_page_block_size == extra_page_block_size + ), helper_msg + assert sched_meta.config.extra_topk == extra_topk, helper_msg + + if topk is not None: + assert not causal, "causal must be False when sparse attention is enabled" + assert is_fp8_kvcache, "is_fp8_kvcache must be True for sparse attention" + out, lse, new_tile_scheduler_metadata, new_num_splits = ( + torch.ops.sgl_kernel.sparse_decode_fwd.default( + q, + k_cache, + indices, + topk_length, + attn_sink, + sched_meta.tile_scheduler_metadata, + sched_meta.num_splits, + extra_k_cache, + extra_indices_in_kvcache, + extra_topk_length, + head_dim_v, + softmax_scale, + ) + ) + else: + assert block_table is not None and cache_seqlens is not None + assert attn_sink is None + assert extra_k_cache is None + assert extra_indices_in_kvcache is None + assert topk_length is None + assert extra_topk_length is None + out, lse, new_tile_scheduler_metadata, new_num_splits = ( + torch.ops.sgl_kernel.dense_decode_fwd.default( + q, + k_cache, + head_dim_v, + cache_seqlens, + block_table, + softmax_scale, + causal, + sched_meta.tile_scheduler_metadata, + sched_meta.num_splits, + ) + ) + + sched_meta.tile_scheduler_metadata = new_tile_scheduler_metadata + sched_meta.num_splits = new_num_splits + return out, lse + + def flash_mla_sparse_fwd( q: torch.Tensor, kv: torch.Tensor, indices: torch.Tensor, sm_scale: float, d_v: int = 512, + attn_sink: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Sparse attention prefill kernel @@ -161,6 +339,6 @@ def flash_mla_sparse_fwd( raise _IMPORT_ERROR from _flashmla_import_error results = torch.ops.sgl_kernel.sparse_prefill_fwd.default( - q, kv, indices, sm_scale, d_v + q, kv, indices, sm_scale, d_v, attn_sink, topk_length ) return results diff --git a/sgl-kernel/python/sgl_kernel/top_k.py b/sgl-kernel/python/sgl_kernel/top_k.py index 77fd5e5e5295..4b842499a743 100644 --- a/sgl-kernel/python/sgl_kernel/top_k.py +++ b/sgl-kernel/python/sgl_kernel/top_k.py @@ -80,6 +80,38 @@ def fast_topk_transform_fused( return dst_page_table +def deepseek_v4_topk_transform_512( + scores: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + page_indices: torch.Tensor, + page_size: int, + raw_indices: Optional[torch.Tensor] = None, +) -> None: + """ + Performs the DeepSeek-V4 indexer top-k selection and writes the paged + physical slot indices into ``page_indices``. Supports topk up to 1024. + Optionally also writes the row-relative raw token positions into + ``raw_indices`` for hisparse capture. + + Args: + scores: float32 ``[B, max_seq_len]`` indexer logits, contiguous on dim 1. + seq_lens: int32 ``[B]``, true KV length per batch row. + page_table: int32 ``[B, num_pages]``, logical->physical page table, + contiguous on dim 1. + page_indices: int32 ``[B, topk]``, output buffer, contiguous. Filled + with paged physical slots; -1 for padding entries. + page_size: power-of-2 page size. + raw_indices: optional int32 ``[B, topk]``, contiguous. If provided, + filled with raw token positions within each row. + """ + if raw_indices is not None: + assert raw_indices.dim() == 2 + torch.ops.sgl_kernel.deepseek_v4_topk_transform_512( + scores, seq_lens, page_table, page_indices, page_size, raw_indices + ) + + def fast_topk_transform_ragged_fused( score: torch.Tensor, lengths: torch.Tensor, diff --git a/sgl-kernel/python/sgl_kernel/version.py b/sgl-kernel/python/sgl_kernel/version.py index 615d4c40d178..f6b7e267c1ed 100644 --- a/sgl-kernel/python/sgl_kernel/version.py +++ b/sgl-kernel/python/sgl_kernel/version.py @@ -1 +1 @@ -__version__ = "0.4.2.post2" +__version__ = "0.4.3" diff --git a/sgl-kernel/setup_rocm.py b/sgl-kernel/setup_rocm.py index 8dddf027ec35..9762787aa76d 100644 --- a/sgl-kernel/setup_rocm.py +++ b/sgl-kernel/setup_rocm.py @@ -46,6 +46,8 @@ def _get_version(): "csrc/allreduce/quick_all_reduce.cu", "csrc/common_extension_rocm.cc", "csrc/elementwise/activation.cu", + "csrc/elementwise/deepseek_v4_topk.cu", + "csrc/elementwise/dsv4_norm_rope.cu", "csrc/elementwise/topk.cu", "csrc/grammar/apply_token_bitmask_inplace_cuda.cu", "csrc/moe/moe_align_kernel.cu", diff --git a/sgl-kernel/tests/test_dsv4_norm_rope.py b/sgl-kernel/tests/test_dsv4_norm_rope.py new file mode 100644 index 000000000000..4dbbbdc054dc --- /dev/null +++ b/sgl-kernel/tests/test_dsv4_norm_rope.py @@ -0,0 +1,130 @@ +"""Tests for DeepSeek-V4 fused norm + RoPE kernels.""" + +import math + +import pytest +import sgl_kernel +import torch + + +def _ref_rmsnorm_self(x: torch.Tensor, eps: float) -> torch.Tensor: + """Reference: RMSNorm without weight (identity weight).""" + rms = torch.sqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + eps) + return (x.float() / rms).to(x.dtype) + + +def _ref_rope_interleaved( + x: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, rope_dim: int +) -> torch.Tensor: + """Reference: apply RoPE to the last `rope_dim` elements (interleaved re/im).""" + out = x.clone() + B = x.size(0) + head_dim = x.size(-1) + nope_dim = head_dim - rope_dim + + for b in range(B): + pos = positions[b].item() + freq = freqs_cis[pos] # (rope_dim,) interleaved [re0, im0, re1, im1, ...] + rope_part = out[b, ..., nope_dim:].float() + # Reshape to pairs + pairs = rope_part.reshape(*rope_part.shape[:-1], rope_dim // 2, 2) + x_real = pairs[..., 0] + x_imag = pairs[..., 1] + freq_pairs = freq.reshape(rope_dim // 2, 2) + f_real = freq_pairs[:, 0] + f_imag = freq_pairs[:, 1] + rot_real = x_real * f_real - x_imag * f_imag + rot_imag = x_real * f_imag + x_imag * f_real + result = torch.stack([rot_real, rot_imag], dim=-1).reshape(rope_part.shape) + out[b, ..., nope_dim:] = result.to(x.dtype) + return out + + +@pytest.mark.parametrize("batch_size", [1, 4, 16]) +@pytest.mark.parametrize("num_heads", [1, 8]) +@pytest.mark.parametrize("head_dim", [128, 192]) +def test_fused_q_norm_rope_correctness(batch_size, num_heads, head_dim): + """Test Q norm + rope against reference.""" + torch.manual_seed(42) + rope_dim = 64 + max_pos = 512 + eps = 1e-6 + + q_input = torch.randn( + batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda" + ) + freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda") + positions = torch.randint( + 0, max_pos, (batch_size,), dtype=torch.int32, device="cuda" + ) + + q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions, eps) + + # Reference + normed = _ref_rmsnorm_self(q_input, eps) + expected = _ref_rope_interleaved(normed, freqs_cis, positions, rope_dim) + + torch.testing.assert_close(q_output.float(), expected.float(), rtol=1e-2, atol=1e-2) + + +def test_fused_q_norm_rope_zero_batch(): + """Empty batch should not crash.""" + q_input = torch.empty(0, 8, 192, dtype=torch.bfloat16, device="cuda") + freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda") + positions = torch.empty(0, dtype=torch.int32, device="cuda") + q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions) + assert q_output.shape == q_input.shape + + +def test_fused_q_norm_rope_preallocated_output(): + """Test with pre-allocated output tensor.""" + torch.manual_seed(42) + B, H, D = 4, 8, 192 + q_input = torch.randn(B, H, D, dtype=torch.bfloat16, device="cuda") + freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda") + positions = torch.randint(0, 512, (B,), dtype=torch.int32, device="cuda") + q_output = torch.empty_like(q_input) + + result = sgl_kernel.dsv4_fused_q_norm_rope( + q_input, freqs_cis, positions, q_output=q_output + ) + assert result is q_output + + +@pytest.mark.parametrize("batch_size", [1, 8]) +def test_fused_q_indexer_rope_hadamard_quant_runs(batch_size): + """Smoke test: kernel runs without errors and produces finite results.""" + torch.manual_seed(42) + num_heads = 4 + head_dim = 128 + rope_dim = 64 + max_pos = 256 + + q_input = torch.randn( + batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda" + ) + q_fp8 = torch.empty( + batch_size, num_heads, head_dim, dtype=torch.uint8, device="cuda" + ) + weight = torch.randn(batch_size, num_heads, dtype=torch.bfloat16, device="cuda") + weights_out = torch.empty( + batch_size, num_heads, 1, dtype=torch.float32, device="cuda" + ) + freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda") + positions = torch.randint( + 0, max_pos, (batch_size,), dtype=torch.int32, device="cuda" + ) + weight_scale = 0.5 + + sgl_kernel.dsv4_fused_q_indexer_rope_hadamard_quant( + q_input, q_fp8, weight, weights_out, weight_scale, freqs_cis, positions + ) + + assert torch.isfinite(weights_out).all(), "weights_out contains non-finite values" + assert q_fp8.any(), "q_fp8 should not be all zeros" + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/sgl-kernel/tests/test_moe_topk_softmax.py b/sgl-kernel/tests/test_moe_topk_softmax.py index 1a8bfb93cab0..77ffe8a4605c 100644 --- a/sgl-kernel/tests/test_moe_topk_softmax.py +++ b/sgl-kernel/tests/test_moe_topk_softmax.py @@ -18,7 +18,7 @@ def compare_topk_values(gating_output, topk_indices_ref, topk_indices): itertools.product( [1, 16, 128, 512, 1024, 2048], # num_tokens [512], # num_experts - [1, 2, 3, 4, 5, 8], # topk + [1, 2, 3, 4, 5, 8, 10], # topk ) ), ) diff --git a/sgl-model-gateway/src/service_discovery.rs b/sgl-model-gateway/src/service_discovery.rs index 0ca248d8c92e..3ff23bf551dd 100644 --- a/sgl-model-gateway/src/service_discovery.rs +++ b/sgl-model-gateway/src/service_discovery.rs @@ -113,13 +113,12 @@ impl PodInfo { pub fn should_include(pod: &Pod, config: &ServiceDiscoveryConfig) -> bool { if config.pd_mode { - if config.prefill_selector.is_empty() && config.decode_selector.is_empty() { - if !(config.igw_mode && !config.selector.is_empty()) { - warn!( - "PD mode enabled but both prefill_selector and decode_selector are empty" - ); - return false; - } + if config.prefill_selector.is_empty() + && config.decode_selector.is_empty() + && (!config.igw_mode || config.selector.is_empty()) + { + warn!("PD mode enabled but both prefill_selector and decode_selector are empty"); + return false; } let matches_pd = Self::matches_selector(pod, &config.prefill_selector) || Self::matches_selector(pod, &config.decode_selector); diff --git a/test/manual/attention/test_flashattn_backend.py b/test/manual/attention/test_flashattn_backend.py index 16b7b68fd0e0..466871df6250 100644 --- a/test/manual/attention/test_flashattn_backend.py +++ b/test/manual/attention/test_flashattn_backend.py @@ -11,6 +11,10 @@ from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + set_forward_context, +) from sglang.test.test_utils import CustomTestCase @@ -87,6 +91,7 @@ def __init__( device=self.device, enable_memory_saver=False, ) + self.hisparse_coordinator = None @unittest.skipIf(not torch.cuda.is_available(), "Test requires CUDA") @@ -109,6 +114,9 @@ def _init_model_runner(self, page_size=1): self.backend = FlashAttentionBackend(self.model_runner) self.ref_backend = TorchNativeAttnBackend(self.model_runner) self.model_runner.model_config.num_attention_heads = self.num_heads + # Publish the backend for any RadixAttention.forward path the tests + # exercise; tearDown is unnecessary here since each test re-inits. + set_forward_context(ForwardContext(attn_backend=self.backend)) def _mock_write_to_req_to_token_pool(self, batch_size, seq_len, page_size): # if page_size > 1, the token pool stores the index to the page. @@ -223,7 +231,6 @@ def _create_forward_batch( extend_seq_lens_cpu=torch.tensor( [q_len] * self.batch_size, device="cpu" ), - attn_backend=self.backend, ) if attn_cp_size > 1: forward_batch.attn_cp_metadata = type( @@ -273,16 +280,11 @@ def _create_forward_batch( [total_len] * self.batch_size, device=self.device ), seq_lens_cpu=torch.tensor([total_len] * self.batch_size, device="cpu"), - attn_backend=self.backend, ) - # Add token pool - forward_batch.req_to_token_pool = self.model_runner.req_to_token_pool - - # Write current batch's req_to_token to req_to_token_pool + # Pool refs are resolved via the active ForwardContext (published in + # setUp). Write the test fixture's req_to_token mapping. self._mock_write_to_req_to_token_pool(self.batch_size, total_len, page_size) - # Add kv pool for this forward batch - forward_batch.token_to_kv_pool = self.model_runner.token_to_kv_pool return forward_batch @@ -307,7 +309,7 @@ def _setup_kv_cache(self, forward_batch, layer, cache_len): ) # Set the prefix KV cache - forward_batch.token_to_kv_pool.set_kv_buffer( + self.model_runner.token_to_kv_pool.set_kv_buffer( layer, torch.arange(self.batch_size * cache_len, device=self.device), cache_k, diff --git a/test/manual/attention/test_flashattn_mla_backend.py b/test/manual/attention/test_flashattn_mla_backend.py index 98eaa5913577..fb44bf0f3969 100644 --- a/test/manual/attention/test_flashattn_mla_backend.py +++ b/test/manual/attention/test_flashattn_mla_backend.py @@ -8,6 +8,10 @@ from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + set_forward_context, +) from sglang.test.test_utils import CustomTestCase @@ -72,6 +76,7 @@ def __init__( device=self.device, enable_memory_saver=False, ) + self.hisparse_coordinator = None class MockReqToTokenPool: @@ -112,6 +117,8 @@ def setUp(self): self.backend = FlashAttentionBackend(self.model_runner) self.ref_backend = TorchNativeAttnBackend(self.model_runner) self.num_local_heads = 2 + # Publish the backend so RadixAttention.forward resolves correctly. + set_forward_context(ForwardContext(attn_backend=self.backend)) def _init_model_runner(self): self.model_runner = MockModelRunner( @@ -192,7 +199,6 @@ def _create_forward_batch(self, mode, q_len=None, prefix_len=0): extend_seq_lens_cpu=torch.tensor( [q_len] * self.batch_size, device="cpu" ), - attn_backend=self.backend, ) else: # ForwardMode.DECODE @@ -216,15 +222,10 @@ def _create_forward_batch(self, mode, q_len=None, prefix_len=0): [total_len] * self.batch_size, device=self.device ), seq_lens_cpu=torch.tensor([total_len] * self.batch_size, device="cpu"), - attn_backend=self.backend, ) - # Add token pool from model runner to forward batch - forward_batch.req_to_token_pool = self.model_runner.req_to_token_pool - - # Add KV cache from model runner to forward batch - forward_batch.token_to_kv_pool = self.model_runner.token_to_kv_pool - + # Pool refs are resolved via the active ForwardContext (published in + # setUp); the fixture no longer needs to attach them to forward_batch. return forward_batch def _setup_kv_cache(self, forward_batch, layer, cache_len): @@ -250,7 +251,7 @@ def _setup_kv_cache(self, forward_batch, layer, cache_len): ) # Set the prefix KV cache using MLA-specific method - forward_batch.token_to_kv_pool.set_mla_kv_buffer( + self.model_runner.token_to_kv_pool.set_mla_kv_buffer( layer, torch.arange(self.batch_size * cache_len, device=self.device), cache_k_nope, diff --git a/test/manual/attention/test_prefix_chunk_info.py b/test/manual/attention/test_prefix_chunk_info.py index 5002a0b09516..523e6b20fb97 100644 --- a/test/manual/attention/test_prefix_chunk_info.py +++ b/test/manual/attention/test_prefix_chunk_info.py @@ -110,12 +110,10 @@ def __init__(self, batch_size, seq_len, device): # Test correctness of triton kernel for computing kv indices -def check_kv_indices(forward_batch): +def check_kv_indices(forward_batch, req_to_token_pool): for i in range(forward_batch.num_prefix_chunks): computed_kv_indices = forward_batch.prefix_chunk_kv_indices[i] - req_to_token = forward_batch.req_to_token_pool.req_to_token[ - : forward_batch.batch_size, : - ] + req_to_token = req_to_token_pool.req_to_token[: forward_batch.batch_size, :] ref_kv_indices = torch.empty( forward_batch.prefix_chunk_num_tokens[i], dtype=torch.int32, @@ -205,8 +203,20 @@ def test_prefix_chunk_info(self): extend_prefix_lens=prefix_lens, extend_prefix_lens_cpu=prefix_lens_cpu, ) - forward_batch.req_to_token_pool = self.req_to_token_pool - forward_batch.token_to_kv_pool = self.token_to_kv_pool + # Pool refs are resolved via the active ForwardContext; mock an + # attn_backend that carries the pools (Pattern A invariant). + from types import SimpleNamespace + + from sglang.srt.model_executor.forward_context import ( + ForwardContext, + set_forward_context, + ) + + mock_backend = SimpleNamespace( + req_to_token_pool=self.req_to_token_pool, + token_to_kv_pool=self.token_to_kv_pool, + ) + set_forward_context(ForwardContext(attn_backend=mock_backend)) forward_batch.prepare_chunked_prefix_cache_info(self.device) assert forward_batch.get_max_chunk_capacity() == max_chunk_capacity @@ -221,7 +231,7 @@ def test_prefix_chunk_info(self): test_case["prefix_chunk_seq_lens"].to(self.device), ) - check_kv_indices(forward_batch) + check_kv_indices(forward_batch, self.req_to_token_pool) if __name__ == "__main__": diff --git a/test/manual/attention/test_trtllm_mla_backend.py b/test/manual/attention/test_trtllm_mla_backend.py index 6ba9a14c05d1..25cfe2a6d971 100755 --- a/test/manual/attention/test_trtllm_mla_backend.py +++ b/test/manual/attention/test_trtllm_mla_backend.py @@ -20,6 +20,10 @@ from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.forward_context import ( + ForwardContext, + set_forward_context, +) from sglang.srt.server_args import ( ServerArgs, get_global_server_args, @@ -264,6 +268,7 @@ def __init__(self, config): device=self.device, enable_memory_saver=False, ) + self.hisparse_coordinator = None def compare_outputs(trtllm_out, reference_out, tolerance=1e-2): @@ -434,10 +439,9 @@ def _create_forward_batch( req_pool_indices=torch.arange(batch_size, device=config["device"]), seq_lens=seq_lens, seq_lens_cpu=seq_lens.cpu(), - attn_backend=backend, ) - fb.req_to_token_pool = model_runner.req_to_token_pool - fb.token_to_kv_pool = model_runner.token_to_kv_pool + # Publish backend for RadixAttention dispatch. + set_forward_context(ForwardContext(attn_backend=backend)) # Add position information for RoPE fb.positions = torch.arange(batch_size, device=config["device"]) @@ -1167,10 +1171,9 @@ def _create_forward_batch_prefill( seq_lens_cpu=seq_lens.cpu(), attn_attend_prefix_cache=False, mha_return_lse=False, - attn_backend=backend, ) - fb.req_to_token_pool = model_runner.req_to_token_pool - fb.token_to_kv_pool = model_runner.token_to_kv_pool + # Publish backend for RadixAttention dispatch. + set_forward_context(ForwardContext(attn_backend=backend)) # Add position information for RoPE fb.positions = torch.arange(batch_size, device=config["device"]) diff --git a/test/manual/dsv4/test_fused_compress_attn_hip.py b/test/manual/dsv4/test_fused_compress_attn_hip.py new file mode 100644 index 000000000000..0e6da091dc61 --- /dev/null +++ b/test/manual/dsv4/test_fused_compress_attn_hip.py @@ -0,0 +1,465 @@ +"""Unit tests for the fused compressor attention Triton kernel on HIP. + +Validates numerical parity between the fused single-kernel path (plan-driven +Triton) and the reference per-seq Python implementation. + +Usage: + python -m pytest test/manual/dsv4/test_fused_compress_attn_hip.py -v + # or directly: + python test/manual/dsv4/test_fused_compress_attn_hip.py +""" + +import unittest +from dataclasses import dataclass + +import numpy as np +import torch + + +@dataclass +class FusedCompressPlan: + compress_plan_gpu: torch.Tensor + write_plan_gpu: torch.Tensor + num_compress: int + num_write: int + + +def write_current_token_to_state( + kv_score_input: torch.Tensor, + write_plan: torch.Tensor, + state_pool_buffer: torch.Tensor, + head_dim: int, + overlap: bool, + ratio: int, +) -> None: + """Reference write path used by this manual test. + + Plan row layout: [ragged_id, batch_id, position, window_len, state_base]. + """ + del head_dim # layout is already encoded in kv_score_input/state_pool_buffer shape. + state_size = (2 if overlap else 1) * ratio + plan_cpu = write_plan.cpu() + for row in plan_cpu: + ragged_id = int(row[0].item()) + position = int(row[2].item()) + state_base = int(row[4].item()) + if ragged_id < 0 or position < 0: + continue + dst = state_base + (position % state_size) + if ( + 0 <= dst < state_pool_buffer.shape[0] + and 0 <= ragged_id < kv_score_input.shape[0] + ): + state_pool_buffer[dst] = kv_score_input[ragged_id] + + +def fused_compress_attn( + state_pool_buffer: torch.Tensor, + plan: torch.Tensor, + ape: torch.Tensor, + rms_weight: torch.Tensor, + rms_eps: float, + freqs_cis_real: torch.Tensor, + head_dim: int, + rope_head_dim: int, + overlap: bool, + ratio: int, + out: torch.Tensor, +) -> torch.Tensor: + """Reference compress path for manual parity tests. + + This keeps the test runnable after removing `fused_compress_kernel.py`. + """ + freqs_cis = torch.view_as_complex( + freqs_cis_real.view(freqs_cis_real.shape[0], -1, 2).contiguous() + ) + result = _ref_compress( + kv_score_input=torch.empty( + 0, device=state_pool_buffer.device, dtype=torch.float32 + ), + state_pool=state_pool_buffer, + plan=plan, + ape=ape, + rms_weight=rms_weight, + rms_eps=rms_eps, + freqs_cis=freqs_cis, + head_dim=head_dim, + rope_head_dim=rope_head_dim, + overlap=overlap, + ratio=ratio, + num_compress=plan.shape[0], + ) + out.copy_(result) + return out + + +def _make_plan_from_params( + extend_lens: list[int], + seq_lens: list[int], + ratio: int, + overlap: bool, + state_bases: list[int], + device: torch.device, +) -> FusedCompressPlan: + """Build a test plan without requiring real SWA / req_to_token tables.""" + bs = len(extend_lens) + ext = np.array(extend_lens, dtype=np.int32) + seq = np.array(seq_lens, dtype=np.int32) + total = int(ext.sum()) + + state_size = (2 if overlap else 1) * ratio + K = state_size + + batch_ids = np.repeat(np.arange(bs, dtype=np.int32), ext) + ragged_ids = np.arange(total, dtype=np.int32) + cu_extend = np.empty(bs + 1, dtype=np.int32) + cu_extend[0] = 0 + np.cumsum(ext, out=cu_extend[1:]) + j_in_seq = ragged_ids - cu_extend[batch_ids] + prefix_lens = seq - ext + positions = prefix_lens[batch_ids] + j_in_seq + + window_lens = np.maximum(0, K - np.minimum(j_in_seq + 1, K)).astype(np.int32) + state_base_arr = np.array(state_bases, dtype=np.int32) + state_base_per_token = state_base_arr[batch_ids] + + plan_rows = np.stack( + [ragged_ids, batch_ids, positions, window_lens, state_base_per_token], + axis=1, + ).astype(np.int32) + + compress_mask = (positions + 1) % ratio == 0 + compress_plan = plan_rows[compress_mask] + + write_starts = np.maximum(0, seq - K).astype(np.int32) + write_mask = positions >= write_starts[batch_ids] + write_plan = plan_rows[write_mask] + + n_compress = int(compress_plan.shape[0]) if compress_plan.size > 0 else 0 + n_write = int(write_plan.shape[0]) if write_plan.size > 0 else 0 + + compress_gpu = ( + torch.from_numpy(np.ascontiguousarray(compress_plan)).to(device) + if n_compress > 0 + else torch.empty((0, 5), dtype=torch.int32, device=device) + ) + write_gpu = ( + torch.from_numpy(np.ascontiguousarray(write_plan)).to(device) + if n_write > 0 + else torch.empty((0, 5), dtype=torch.int32, device=device) + ) + + return FusedCompressPlan( + compress_plan_gpu=compress_gpu, + write_plan_gpu=write_gpu, + num_compress=n_compress, + num_write=n_write, + ) + + +def _make_freqs_cis(max_seq: int, rope_dim: int, device: torch.device) -> torch.Tensor: + """Create test freqs_cis as complex64 [max_seq, rope_dim/2], matching production.""" + half = rope_dim // 2 + angles = torch.randn(max_seq, half, device=device, dtype=torch.float32) * 0.1 + return torch.polar(torch.ones_like(angles), angles) + + +def _freqs_to_real(freqs_cis: torch.Tensor) -> torch.Tensor: + """Convert complex64 freqs to float32 [max_seq, rope_dim] interleaved.""" + return torch.view_as_real(freqs_cis).flatten(-2).contiguous() + + +def _ref_compress( + kv_score_input: torch.Tensor, + state_pool: torch.Tensor, + plan: torch.Tensor, + ape: torch.Tensor, + rms_weight: torch.Tensor, + rms_eps: float, + freqs_cis: torch.Tensor, + head_dim: int, + rope_head_dim: int, + overlap: bool, + ratio: int, + num_compress: int, +) -> torch.Tensor: + """Pure-PyTorch reference matching SGLang compress_decode_paged semantics. + + State already has current tokens written (no APE). + APE is added to ALL K scores at compress time. + """ + if num_compress == 0: + return torch.empty(0, head_dim, dtype=torch.float32, device=state_pool.device) + + coff = 2 if overlap else 1 + half_dim = coff * head_dim + state_size = coff * ratio + K = state_size + + plan_cpu = plan[:num_compress].cpu() + out = torch.empty( + num_compress, head_dim, dtype=torch.float32, device=state_pool.device + ) + + for pid in range(num_compress): + position = int(plan_cpu[pid, 2].item()) + state_base = int(plan_cpu[pid, 4].item()) + + if position < 0: + continue + + kv_rows = [] + score_rows = [] + for k in range(K): + s = position - K + 1 + k + col_off = (head_dim if k >= ratio else 0) if overlap else 0 + ape_row = k % ratio + d_slice = slice(col_off, col_off + head_dim) + + if s < 0: + kv_rows.append( + torch.zeros(head_dim, dtype=torch.float32, device=state_pool.device) + ) + score_rows.append( + torch.full( + (head_dim,), + float("-inf"), + dtype=torch.float32, + device=state_pool.device, + ) + ) + else: + ring = s % state_size + row = state_pool[state_base + ring] + kv_rows.append(row[d_slice].float()) + # APE added to ALL scores + score_rows.append( + row[half_dim + col_off : half_dim + col_off + head_dim].float() + + ape[ape_row, d_slice].float() + ) + + kv_stack = torch.stack(kv_rows, dim=0) + sc_stack = torch.stack(score_rows, dim=0) + weights = torch.softmax(sc_stack, dim=0) + compressed = (weights * kv_stack).sum(dim=0) + + var = (compressed * compressed).mean() + normed = compressed * torch.rsqrt(var + rms_eps) * rms_weight.float() + + comp_pos = (position // ratio) * ratio + rope_seg = normed[-rope_head_dim:].clone() + freqs_row = torch.view_as_real(freqs_cis[comp_pos]).flatten() + cos_v = freqs_row[0::2].float() + sin_v = freqs_row[1::2].float() + + even = rope_seg[0::2] + odd = rope_seg[1::2] + normed[-rope_head_dim:] = torch.stack( + [even * cos_v - odd * sin_v, odd * cos_v + even * sin_v], dim=-1 + ).flatten() + + out[pid] = normed + + return out + + +class TestFusedCompressAttn(unittest.TestCase): + + def _run_test( + self, + ratio: int, + overlap: bool, + bs: int, + extend_lens: list[int], + prefix_lens: list[int], + head_dim: int = 512, + rope_head_dim: int = 64, + ): + device = torch.device("cuda") + torch.manual_seed(42) + coff = 2 if overlap else 1 + half_dim = coff * head_dim + last_dim = 2 * half_dim + state_size = coff * ratio + + seq_lens = [p + e for p, e in zip(prefix_lens, extend_lens)] + total_tokens = sum(extend_lens) + max_seq = max(seq_lens) + 128 + + kv_score_input = torch.randn( + total_tokens, last_dim, device=device, dtype=torch.float32 + ) + + pool_size = bs * state_size + 2 + state_pool = torch.randn( + pool_size, last_dim, device=device, dtype=torch.float32 + ) + state_pool[:, half_dim:] *= 0.5 # reasonable score magnitudes + + state_bases = [i * state_size for i in range(bs)] + ape = torch.randn(ratio, half_dim, device=device, dtype=torch.float32) * 0.1 + rms_weight = torch.ones(head_dim, device=device, dtype=torch.float32) + rms_eps = 1e-6 + freqs_cis = _make_freqs_cis(max_seq, rope_head_dim, device) + freqs_real = _freqs_to_real(freqs_cis) + + plan = _make_plan_from_params( + extend_lens, seq_lens, ratio, overlap, state_bases, device + ) + if plan.num_compress == 0: + return + + # Step 1: write current tokens to state (same for both paths) + state_triton = state_pool.clone() + state_ref = state_pool.clone() + + write_current_token_to_state( + kv_score_input=kv_score_input, + write_plan=plan.write_plan_gpu, + state_pool_buffer=state_triton, + head_dim=head_dim, + overlap=overlap, + ratio=ratio, + ) + # Reference: same write + write_current_token_to_state( + kv_score_input=kv_score_input, + write_plan=plan.write_plan_gpu, + state_pool_buffer=state_ref, + head_dim=head_dim, + overlap=overlap, + ratio=ratio, + ) + + # Step 2a: Triton fused compress + out_triton = torch.empty( + plan.num_compress, head_dim, device=device, dtype=torch.float32 + ) + fused_compress_attn( + state_pool_buffer=state_triton, + plan=plan.compress_plan_gpu, + ape=ape, + rms_weight=rms_weight, + rms_eps=rms_eps, + freqs_cis_real=freqs_real, + head_dim=head_dim, + rope_head_dim=rope_head_dim, + overlap=overlap, + ratio=ratio, + out=out_triton, + ) + + # Step 2b: reference compress + out_ref = _ref_compress( + kv_score_input=kv_score_input, + state_pool=state_ref, + plan=plan.compress_plan_gpu, + ape=ape, + rms_weight=rms_weight, + rms_eps=rms_eps, + freqs_cis=freqs_cis, + head_dim=head_dim, + rope_head_dim=rope_head_dim, + overlap=overlap, + ratio=ratio, + num_compress=plan.num_compress, + ) + + torch.testing.assert_close(out_triton, out_ref, atol=1e-3, rtol=1e-3) + + def test_hca_single(self): + self._run_test( + ratio=128, overlap=False, bs=1, extend_lens=[128], prefix_lens=[0] + ) + + def test_hca_multi(self): + self._run_test( + ratio=128, overlap=False, bs=2, extend_lens=[128, 256], prefix_lens=[0, 128] + ) + + def test_csa_single(self): + self._run_test(ratio=4, overlap=True, bs=1, extend_lens=[16], prefix_lens=[8]) + + def test_csa_multi(self): + self._run_test( + ratio=4, overlap=True, bs=3, extend_lens=[8, 12, 16], prefix_lens=[4, 8, 0] + ) + + def test_csa_small_dim(self): + self._run_test( + ratio=4, + overlap=True, + bs=2, + extend_lens=[8, 8], + prefix_lens=[4, 0], + head_dim=256, + ) + + +class TestStateOrdering(unittest.TestCase): + + def test_write_then_compress(self): + """Verify write-first, compress-second matches reference.""" + device = torch.device("cuda") + torch.manual_seed(123) + ratio, overlap = 4, True + coff = 2 + head_dim, rope_head_dim = 128, 64 + half_dim = coff * head_dim + last_dim = 2 * half_dim + state_size = coff * ratio + + pool_size = state_size + 2 + state_pool = torch.randn( + pool_size, last_dim, device=device, dtype=torch.float32 + ) + state_pool[:, half_dim:] *= 0.5 + + kv_score_input = torch.randn(8, last_dim, device=device, dtype=torch.float32) + ape = torch.randn(ratio, half_dim, device=device, dtype=torch.float32) * 0.1 + rms_weight = torch.ones(head_dim, device=device, dtype=torch.float32) + freqs_cis = _make_freqs_cis(64, rope_head_dim, device) + + plan = _make_plan_from_params([8], [8], ratio, overlap, [0], device) + if plan.num_compress == 0: + return + + state_before = state_pool.clone() + + # Write first + write_current_token_to_state( + kv_score_input=kv_score_input, + write_plan=plan.write_plan_gpu, + state_pool_buffer=state_pool, + head_dim=head_dim, + overlap=overlap, + ratio=ratio, + ) + + # State should now be different (tokens written) + self.assertFalse(torch.allclose(state_pool, state_before)) + + # Compress + out = torch.empty( + plan.num_compress, head_dim, device=device, dtype=torch.float32 + ) + fused_compress_attn( + state_pool_buffer=state_pool, + plan=plan.compress_plan_gpu, + ape=ape, + rms_weight=rms_weight, + rms_eps=1e-6, + freqs_cis_real=_freqs_to_real(freqs_cis), + head_dim=head_dim, + rope_head_dim=rope_head_dim, + overlap=overlap, + ratio=ratio, + out=out, + ) + + self.assertFalse(torch.any(torch.isnan(out)).item()) + self.assertFalse(torch.any(torch.isinf(out)).item()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/manual/models/test_granite_moe_hybrid.py b/test/manual/models/test_granite_moe_hybrid.py new file mode 100644 index 000000000000..a8e498523080 --- /dev/null +++ b/test/manual/models/test_granite_moe_hybrid.py @@ -0,0 +1,33 @@ +import unittest + +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin +from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin +from sglang.test.server_fixtures.default_fixture import DefaultServerBase + +GRANITE_MOE_HYBRID_MODEL = "ibm-granite/granite-4.0-h-micro" + + +class TestGraniteMoeHybrid(GSM8KMixin, DefaultServerBase): + model = GRANITE_MOE_HYBRID_MODEL + gsm8k_accuracy_thres = 0.78 + + +class TestGraniteMoeHybridExtraBuffer( + GSM8KMixin, KLDivergenceMixin, PrefixCacheBranchingMixin, DefaultServerBase +): + model = GRANITE_MOE_HYBRID_MODEL + cache_chunk_size = 256 + gsm8k_accuracy_thres = 0.78 + kl_div_thres = 0.002 + kl_div_thres_prefill = 0.02 + other_args = [ + "--mem-fraction-static", + "0.8", + "--mamba-scheduler-strategy", + "extra_buffer", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/manual/models/test_nvidia_nemotron_nano_v2.py b/test/manual/models/test_nvidia_nemotron_nano_v2.py index 7a93af0061da..770f4f5443af 100644 --- a/test/manual/models/test_nvidia_nemotron_nano_v2.py +++ b/test/manual/models/test_nvidia_nemotron_nano_v2.py @@ -2,91 +2,104 @@ from sglang.srt.utils import is_blackwell from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin +from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase +NVIDIA_NEMOTRON_NANO_V2_MODEL = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" + class TestNvidiaNemotronNanoV2BF16(GSM8KMixin, DefaultServerBase): - model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" + model = NVIDIA_NEMOTRON_NANO_V2_MODEL gsm8k_accuracy_thres = 0.87 other_args = ["--max-mamba-cache-size", "256"] class TestNvidiaNemotronNanoV2BF16PP(GSM8KMixin, DefaultServerBase): - model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" + model = NVIDIA_NEMOTRON_NANO_V2_MODEL gsm8k_accuracy_thres = 0.87 other_args = ["--max-mamba-cache-size", "256", "--pp-size", "2"] -class TestNvidiaNemotronNanoV2FP8(GSM8KMixin, DefaultServerBase): +class TestNvidiaNemotronNanoV2BF16ExtraBuffer( + GSM8KMixin, KLDivergenceMixin, PrefixCacheBranchingMixin, DefaultServerBase +): + model = NVIDIA_NEMOTRON_NANO_V2_MODEL + cache_chunk_size = 256 gsm8k_accuracy_thres = 0.87 + kl_div_thres = 0.002 + kl_div_thres_prefill = 0.01 + other_args = [ + "--max-mamba-cache-size", + "256", + "--mem-fraction-static", + "0.8", + "--mamba-scheduler-strategy", + "extra_buffer", + ] + + +class TestNvidiaNemotronNanoV2FP8(GSM8KMixin, DefaultServerBase): model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8" + gsm8k_accuracy_thres = 0.87 other_args = ["--max-mamba-cache-size", "256"] @unittest.skipIf(not is_blackwell(), "NVFP4 only supported on blackwell") class TestNvidiaNemotronNanoV2NVFP4(GSM8KMixin, DefaultServerBase): - gsm8k_accuracy_thres = 0.855 model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4" + gsm8k_accuracy_thres = 0.855 other_args = ["--max-mamba-cache-size", "256"] -@unittest.skip( - "STANDALONE speculative decoding does not yet support target and draft models " - "with different hidden sizes (Nemotron-9B: 4480, Llama-3.2-1B: 2048)" -) +SPECULATIVE_DECODING_OTHER_ARGS = [ + "--speculative-algorithm", + "STANDALONE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "3", + "--speculative-num-draft-tokens", + "5", + "--speculative-draft-model-path", + "meta-llama/Llama-3.2-1B", + "--speculative-draft-load-format", + "dummy", + "--max-running-requests", + "8", + "--max-total-tokens", + "2048", + "--json-model-override-args", + '{"vocab_size": 131072, "hidden_size": 4480}', +] + + class TestNvidiaNemotronNanoV2SpeculativeDecoding(GSM8KMixin, DefaultServerBase): + model = NVIDIA_NEMOTRON_NANO_V2_MODEL gsm8k_accuracy_thres = 0.87 - model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" - other_args = [ - "--speculative-algorithm", - "STANDALONE", - "--speculative-num-steps", - "2", - "--speculative-eagle-topk", - "3", - "--speculative-num-draft-tokens", - "5", - "--speculative-draft-model-path", - "meta-llama/Llama-3.2-1B", - "--speculative-draft-load-format", - "dummy", - "--max-running-requests", - "8", - "--max-total-tokens", - "2048", - "--json-model-override-args", - '{"vocab_size": 131072}', + other_args = SPECULATIVE_DECODING_OTHER_ARGS + [ + "--disable-radix-cache", + ] + + +class TestNvidiaNemotronNanoV2SpeculativeDecodingExtraBuffer( + GSM8KMixin, DefaultServerBase +): + model = NVIDIA_NEMOTRON_NANO_V2_MODEL + gsm8k_accuracy_thres = 0.87 + other_args = SPECULATIVE_DECODING_OTHER_ARGS + [ + "--mamba-scheduler-strategy", + "extra_buffer", ] -@unittest.skip( - "STANDALONE speculative decoding does not yet support target and draft models " - "with different hidden sizes (Nemotron-9B: 4480, Llama-3.2-1B: 2048)" -) class TestNvidiaNemotronNanoV2SpeculativeDecodingBF16Cache( GSM8KMixin, DefaultServerBase ): + model = NVIDIA_NEMOTRON_NANO_V2_MODEL gsm8k_accuracy_thres = 0.87 - model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" - other_args = [ - "--speculative-algorithm", - "STANDALONE", - "--speculative-num-steps", - "2", - "--speculative-eagle-topk", - "3", - "--speculative-num-draft-tokens", - "5", - "--speculative-draft-model-path", - "meta-llama/Llama-3.2-1B", - "--speculative-draft-load-format", - "dummy", - "--max-running-requests", - "8", - "--max-total-tokens", - "2048", - "--json-model-override-args", - '{"vocab_size": 131072}', + other_args = SPECULATIVE_DECODING_OTHER_ARGS + [ + "--disable-radix-cache", "--mamba-ssm-dtype", "bfloat16", ] diff --git a/test/manual/vlm/test_mm_utils.py b/test/manual/vlm/test_mm_utils.py index c526c43246ba..9302c40a6a73 100644 --- a/test/manual/vlm/test_mm_utils.py +++ b/test/manual/vlm/test_mm_utils.py @@ -45,6 +45,58 @@ def test_materialize_proxy(self): self.assertTrue(torch.equal(mm_inputs.mm_items[0].feature, feature_tensor)) proxy_feature.reconstruct_on_target_device.assert_called_once_with(0) + def test_materialize_precomputed_embedding_proxy_without_feature(self): + embedding_tensor = torch.tensor([[1.0, 2.0]], dtype=torch.float32) + proxy_embedding = _make_proxy_with_reconstruct_result(embedding_tensor) + mm_item = MultimodalDataItem( + modality=Modality.IMAGE, + offsets=[(0, 1)], + precomputed_embeddings=proxy_embedding, + ) + + with ( + patch.object(schedule_batch.torch.cuda, "is_available", return_value=True), + patch.object(schedule_batch.torch.cuda, "current_device", return_value=0), + patch.object( + schedule_batch.envs.SGLANG_MM_BUFFER_SIZE_MB, "get", return_value=0 + ), + ): + mm_inputs = MultimodalInputs.from_dict({"mm_items": [mm_item]}) + + self.assertTrue( + torch.equal( + mm_inputs.mm_items[0].precomputed_embeddings, + embedding_tensor, + ) + ) + proxy_embedding.reconstruct_on_target_device.assert_called_once_with(0) + + def test_materialize_model_specific_proxy_without_feature(self): + grid_tensor = torch.tensor([[1, 2, 3]], dtype=torch.int64) + proxy_grid = _make_proxy_with_reconstruct_result(grid_tensor) + mm_item = MultimodalDataItem( + modality=Modality.IMAGE, + offsets=[(0, 1)], + model_specific_data={"image_grid_thw": proxy_grid}, + ) + + with ( + patch.object(schedule_batch.torch.cuda, "is_available", return_value=True), + patch.object(schedule_batch.torch.cuda, "current_device", return_value=0), + patch.object( + schedule_batch.envs.SGLANG_MM_BUFFER_SIZE_MB, "get", return_value=0 + ), + ): + mm_inputs = MultimodalInputs.from_dict({"mm_items": [mm_item]}) + + self.assertTrue( + torch.equal( + mm_inputs.mm_items[0].model_specific_data["image_grid_thw"], + grid_tensor, + ) + ) + proxy_grid.reconstruct_on_target_device.assert_called_once_with(0) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py index 0082a1bab079..d8fa7fd92845 100644 --- a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py +++ b/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py @@ -81,6 +81,7 @@ def setUpClass(cls): "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", "HCCL_BUFFSIZE": "2100", "HCCL_OP_EXPANSION_MODE": "AIV", + "TRANSFORMERS_VERBOSITY": "error", **os.environ, }, ) diff --git a/test/registered/attention/test_chunk_gated_delta_rule.py b/test/registered/attention/test_chunk_gated_delta_rule.py index d496d679a71d..c487f25778ec 100644 --- a/test/registered/attention/test_chunk_gated_delta_rule.py +++ b/test/registered/attention/test_chunk_gated_delta_rule.py @@ -210,6 +210,16 @@ def test_multi_chunk_nt8(self): def test_large_pool(self): self._check_shape(B=4, T_per_seq=128, H=16, K=128, V=128, pool_size=512) + # ------------------------------------------------------------------ + # Long prompts (many chunks; regression test for cross-chunk errors) + # ------------------------------------------------------------------ + def test_long_prompt(self): + for B, T_per_seq in [(1, 1024), (1, 1536), (1, 2048), (2, 1024)]: + with self.subTest(T_per_seq=T_per_seq): + self._check_shape( + B=B, T_per_seq=T_per_seq, H=16, K=128, V=128, pool_size=32 + ) + # ------------------------------------------------------------------ # Combined stress # ------------------------------------------------------------------ diff --git a/test/registered/attention/test_gdn_prefill_cutedsl.py b/test/registered/attention/test_gdn_prefill_cutedsl.py new file mode 100644 index 000000000000..807be8a66f1b --- /dev/null +++ b/test/registered/attention/test_gdn_prefill_cutedsl.py @@ -0,0 +1,178 @@ +"""Correctness test for the SM100 CuTe DSL GDN prefill kernel. + +Ported from vLLM PR https://github.com/vllm-project/vllm/pull/43273. +Validates ``chunk_gated_delta_rule_cutedsl`` against the +``fused_recurrent_gated_delta_rule`` Triton reference. +""" + +import math + +import pytest +import torch +import torch.nn.functional as F + +from sglang.test.ci.ci_register import register_cuda_ci + +# CuteDSL prefill kernel only exists on Blackwell. Single-GPU kernel-unit +# suite is the right slot (matches existing jit_kernel test_*.py pattern). +register_cuda_ci(est_time=60, suite="base-b-kernel-unit-1-gpu-b200") + +if not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10): + pytest.skip( + "GDN CuteDSL prefill requires CUDA SM10x (Blackwell).", + allow_module_level=True, + ) + +from sglang.srt.layers.attention.fla.fused_recurrent import ( # noqa: E402 + fused_recurrent_gated_delta_rule, +) +from sglang.srt.layers.attention.fla.index import ( # noqa: E402 + prepare_chunk_indices, + prepare_chunk_offsets, +) +from sglang.srt.layers.attention.linear.kernels.gdn_blackwell import ( # noqa: E402 + chunk_gated_delta_rule_cutedsl, + prepare_metadata_cutedsl, +) + + +@pytest.mark.parametrize("num_seqs", [1, 5, 257]) +@pytest.mark.parametrize("state_dtype", [torch.bfloat16, torch.float32]) +def test_gdn_chunk_cutedsl_correctness(num_seqs: int, state_dtype: torch.dtype): + seq_lens = torch.randint(1, 130, (num_seqs,), dtype=torch.int32) + cu_seqlens = torch.zeros(num_seqs + 1, device="cuda", dtype=torch.int32) + cu_seqlens[1:] = seq_lens.to(device="cuda").cumsum(0) + total_tokens = int(cu_seqlens[-1].item()) + + num_k_heads = 4 + num_v_heads = 8 + head_k_dim = 128 + head_v_dim = 128 + dtype = torch.bfloat16 + + q = torch.randn( + 1, total_tokens, num_k_heads, head_k_dim, device="cuda", dtype=dtype + ) + k = torch.randn_like(q) + v = torch.randn( + 1, total_tokens, num_v_heads, head_v_dim, device="cuda", dtype=dtype + ) + q = F.normalize(q.float(), p=2, dim=-1).to(dtype) + k = F.normalize(k.float(), p=2, dim=-1).to(dtype) + a = torch.randn(1, total_tokens, num_v_heads, device="cuda", dtype=dtype) + b = torch.randn(1, total_tokens, num_v_heads, device="cuda", dtype=dtype) + + # Match upstream FLA GatedDeltaNet synthetic init. + A = torch.empty(num_v_heads, device="cuda", dtype=torch.float32).uniform_(0, 16) + A_log = torch.log(A) + dt = torch.exp( + torch.rand(num_v_heads, device="cuda", dtype=torch.float32) + * (math.log(0.1) - math.log(0.001)) + + math.log(0.001) + ) + dt = torch.clamp(dt, min=1e-4) + dt_bias = dt + torch.log(-torch.expm1(-dt)) + g = -A_log.exp().view(1, 1, num_v_heads) * F.softplus( + a.float() + dt_bias.view(1, 1, num_v_heads) + ) + beta = torch.sigmoid(b.float()) + initial_state = ( + torch.randn( + num_seqs, + num_v_heads, + head_v_dim, + head_k_dim, + device="cuda", + dtype=state_dtype, + ) + * 0.05 + ) + + # Metadata kernel matches the FLA reference helpers. + chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens) + torch.cuda.synchronize() + + expected_indices = prepare_chunk_indices(cu_seqlens, 64) + expected_offsets = prepare_chunk_offsets(cu_seqlens, 64) + total_chunks = int(expected_offsets[-1].item()) + + torch.testing.assert_close(chunk_offsets, expected_offsets.to(torch.int32)) + torch.testing.assert_close(chunk_indices[:total_chunks], expected_indices) + + # Reference: token-by-token recurrent kernel returns (o, final_state). + # Recurrent path needs float32 state, so cast initial_state for the call. + ref_o, ref_state = fused_recurrent_gated_delta_rule( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state.to(torch.float32), + output_final_state=True, + cu_seqlens=cu_seqlens.to(torch.int64), + use_qk_l2norm_in_kernel=False, + ) + + actual_core_attn_out = torch.empty( + total_tokens, num_v_heads, head_v_dim, device="cuda", dtype=dtype + ) + actual_o, actual_state = chunk_gated_delta_rule_cutedsl( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + core_attn_out=actual_core_attn_out, + ) + torch.cuda.synchronize() + + o_error = (actual_o.float() - ref_o.float()).abs() + state_error = ( + actual_state.float() - ref_state.to(actual_state.dtype).float() + ).abs() + assert o_error.max().item() < 2e-3 + assert o_error.mean().item() < 6e-5 + assert state_error.max().item() < 2e-2 + assert state_error.mean().item() < 6e-4 + core_attn_out_error = ( + actual_core_attn_out.float() - actual_o.squeeze(0).float() + ).abs() + assert core_attn_out_error.max().item() == 0 + + no_buffer_o, no_buffer_state = chunk_gated_delta_rule_cutedsl( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + ) + torch.cuda.synchronize() + + no_buffer_o_error = (no_buffer_o.float() - ref_o.float()).abs() + no_buffer_state_error = ( + no_buffer_state.float() - ref_state.to(no_buffer_state.dtype).float() + ).abs() + buffer_o_error = (no_buffer_o.float() - actual_o.float()).abs() + buffer_state_error = ( + no_buffer_state.float() - actual_state.to(no_buffer_state.dtype).float() + ).abs() + assert no_buffer_o_error.max().item() < 2e-3 + assert no_buffer_o_error.mean().item() < 6e-5 + assert no_buffer_state_error.max().item() < 2e-2 + assert no_buffer_state_error.mean().item() < 6e-4 + assert buffer_o_error.max().item() == 0 + assert buffer_state_error.max().item() == 0 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/distributed/test_flashinfer_fusion_preflight.py b/test/registered/backends/test_flashinfer_fusion_preflight.py similarity index 100% rename from test/registered/distributed/test_flashinfer_fusion_preflight.py rename to test/registered/backends/test_flashinfer_fusion_preflight.py diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index e483fa56248d..d0238f200c0e 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -155,6 +155,50 @@ def test_gsm8k(self): self.assertGreater(metrics["score"], 0.93) +class FlashinferTrtllmGenMoeBackendMXFP8MixedBF16Base: + backend = None + + @classmethod + def setUpClass(cls): + cls.model = "zianglih/JoyAI-LLM-Flash-MXFP8-last-6-BF16" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + env={**os.environ, "SGLANG_ENABLE_JIT_DEEPGEMM": "False"}, + other_args=[ + "--kv-cache-dtype", + "bf16", + "--fp8-gemm-backend", + "flashinfer_cutlass", + "--moe-runner-backend", + cls.backend, + "--tp-size", + "4", + "--trust-remote-code", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.92) + + class FlashinferTrtllmGenMoeBackendNVFP4Base: backend = None extra_env = {} @@ -217,6 +261,12 @@ class TestFlashinferTrtllmGenMoeBackendMXFP8Routed( backend = "flashinfer_trtllm_routed" +class TestFlashinferTrtllmRoutedMxfp8MixedBF16( + FlashinferTrtllmGenMoeBackendMXFP8MixedBF16Base, CustomTestCase +): + backend = "flashinfer_trtllm_routed" + + class TestFlashinferTrtllmGenMoeBackendBF16Routed( FlashinferTrtllmGenMoeBackendBF16Base, CustomTestCase ): diff --git a/test/registered/cp/test_deepseek_v3_cp_single_node.py b/test/registered/cp/test_deepseek_v3_cp_single_node.py new file mode 100644 index 000000000000..74e0258d5d6d --- /dev/null +++ b/test/registered/cp/test_deepseek_v3_cp_single_node.py @@ -0,0 +1,89 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_cuda_ci(est_time=500, stage="extra-b", runner_config="deepep-8-gpu-h200") + +DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324" + +# Matches the non-CP DSv3 production baseline in +# ``test_deepseek_v3_basic.py`` / ``test_deepseek_v3_mtp.py``. Pinning +# MLA CP to the same threshold makes this test double as a regression +# gate against the known production accuracy. +GSM8K_ACCURACY_THRESHOLD = 0.935 + + +class TestDeepseekV3CPInSeqSplit(CustomTestCase): + """tp=8, dp=2, attn-cp=4 — DP attention + DeepEP MoE + MLA CP.""" + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V3_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "2", + "--enable-prefill-context-parallel", + "--attention-backend", + "fa3", + "--mem-frac", + "0.7", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + kill_process_tree(cls.process.pid) + + # "test_a_" prefix pins alphabetical first-run ordering so this + # warms up the server before any follow-up sibling test methods. + def test_a_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=32, + num_shots=20, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_a_gsm8k (deepseek-v3-mla-cp-in-seq-split)\n" + f'{metrics["score"]=:.3f}\n' + ) + self.assertGreater(metrics["score"], GSM8K_ACCURACY_THRESHOLD) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py b/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py new file mode 100644 index 000000000000..3cb9b4e1a0a7 --- /dev/null +++ b/test/registered/cp/test_deepseek_v4_flash_fp4_b200_cp.py @@ -0,0 +1,85 @@ +"""B200 extra CI: DeepSeek-V4-Flash FP4 with attn-CP (DSA prefill CP). + +Balanced recipe (TP=4, DeepEP, EAGLE) plus --attn-cp-size=4 with the +DSA prefill-CP round-robin-split mode. Split out of +models_e2e/test_deepseek_v4_flash_fp4_b200.py so the `cp` group covers +all context-parallel tests. + +Registry: extra-b-test-4-gpu-b200 (label-gated extra CI, 4x B200) +""" + +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + try_cached_model, +) + +register_cuda_ci(est_time=235, stage="extra-b", runner_config="4-gpu-b200") + +MODEL = "deepseek-ai/DeepSeek-V4-Flash" +SERVER_LAUNCH_TIMEOUT = 3600 +DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + +_DEEPEP_ENV = { + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "1024", +} + + +class TestDSV4FlashFP4B200Balanced_CP( + BasicDecodeCorrectnessMixin, + GSM8KMixin, + CustomTestCase, +): + """Balanced recipe: TP=4, DP=4, DeepEP, EAGLE (1-step spec).""" + + gsm8k_accuracy_thres = 0.93 + + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(MODEL) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--attn-cp-size", + "4", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--enable-dsa-prefill-context-parallel", + "--dsa-prefill-cp-mode", + "round-robin-split", + "--deepep-config", + DEEPEP_CONFIG, + ], + env=_DEEPEP_ENV, + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/4-gpu-models/test_qwen3_30b.py b/test/registered/cp/test_qwen3_30b.py similarity index 98% rename from test/registered/4-gpu-models/test_qwen3_30b.py rename to test/registered/cp/test_qwen3_30b.py index b81534268d90..e21dafb5c3c9 100644 --- a/test/registered/4-gpu-models/test_qwen3_30b.py +++ b/test/registered/cp/test_qwen3_30b.py @@ -11,7 +11,7 @@ popen_launch_server, ) -register_cuda_ci(est_time=261, stage="base-c", runner_config="4-gpu-h100") +register_cuda_ci(est_time=261, stage="extra-b", runner_config="4-gpu-h100") QWEN3_30B_MODEL_PATH = "Qwen/Qwen3-30B-A3B-FP8" diff --git a/test/registered/cpu/test_store_cache.py b/test/registered/cpu/test_store_cache.py new file mode 100644 index 000000000000..80bd94e06bc7 --- /dev/null +++ b/test/registered/cpu/test_store_cache.py @@ -0,0 +1,83 @@ +import sys + +import pytest +import torch + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=25, suite="base-b-test-cpu") + +torch.manual_seed(42) + +DEVICE = "cpu" +CACHE_SIZE = 4096 + +# for fp8 KV stored as uint8, e.g. float8_e4m3fn and float8_e5m2 +DTYPES = [torch.float16, torch.bfloat16, torch.uint8] +DTYPE_IDS = ["float16", "bfloat16", "uint8"] + + +def _store_cache_cpu(k, v, k_cache, v_cache, indices): + row_dim = k.size(1) * k.size(2) + torch.ops.sgl_kernel.store_cache_cpu(k, v, k_cache, v_cache, indices, row_dim) + + +def _random_tensor(shape, dtype): + """FP8 KV is stored as uint8; randn is not implemented for Byte.""" + if dtype == torch.uint8: + return torch.randint(0, 256, shape, dtype=torch.uint8, device=DEVICE) + return torch.randn(shape, dtype=dtype, device=DEVICE) + + +@pytest.mark.parametrize("dtype", DTYPES, ids=DTYPE_IDS) +@pytest.mark.parametrize("head_dim", [64, 128]) +@pytest.mark.parametrize("num_heads", [1, 8, 16, 32]) +@pytest.mark.parametrize("batch_size", [1, 7, 133]) +def test_store_cache(batch_size, num_heads, head_dim, dtype): + shape = (batch_size, num_heads, head_dim) + cache_shape = (CACHE_SIZE, num_heads, head_dim) + k = _random_tensor(shape, dtype) + v = _random_tensor(shape, dtype) + k_cache = _random_tensor(cache_shape, dtype) + v_cache = _random_tensor(cache_shape, dtype) + indices = torch.randperm(CACHE_SIZE, device=DEVICE, dtype=torch.int64)[:batch_size] + + k_cache_ref = k_cache.clone() + v_cache_ref = v_cache.clone() + k_cache_ref[indices] = k + v_cache_ref[indices] = v + + _store_cache_cpu(k, v, k_cache, v_cache, indices) + + assert torch.equal(k_cache, k_cache_ref) + assert torch.equal(v_cache, v_cache_ref) + + +@pytest.mark.parametrize("dtype", DTYPES, ids=DTYPE_IDS) +@pytest.mark.parametrize("head_dim", [64, 128]) +@pytest.mark.parametrize("num_heads", [1, 8]) +@pytest.mark.parametrize("batch_size", [11]) +def test_store_cache_int32_indices(batch_size, num_heads, head_dim, dtype): + shape = (batch_size, num_heads, head_dim) + cache_shape = (CACHE_SIZE, num_heads, head_dim) + k = _random_tensor(shape, dtype) + v = _random_tensor(shape, dtype) + k_cache = _random_tensor(cache_shape, dtype) + v_cache = _random_tensor(cache_shape, dtype) + indices = torch.randperm(CACHE_SIZE, device=DEVICE, dtype=torch.int64)[ + :batch_size + ].to(torch.int32) + + k_cache_ref = k_cache.clone() + v_cache_ref = v_cache.clone() + k_cache_ref[indices.long()] = k + v_cache_ref[indices.long()] = v + + _store_cache_cpu(k, v, k_cache, v_cache, indices) + + assert torch.equal(k_cache, k_cache_ref) + assert torch.equal(v_cache, v_cache_ref) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/test/registered/distributed/test_disaggregation_aarch64.py b/test/registered/disaggregation/test_disaggregation_aarch64.py similarity index 100% rename from test/registered/distributed/test_disaggregation_aarch64.py rename to test/registered/disaggregation/test_disaggregation_aarch64.py diff --git a/test/registered/disaggregation/test_disaggregation_basic.py b/test/registered/disaggregation/test_disaggregation_basic.py index f2ee9de56f91..0cdbf03400f5 100644 --- a/test/registered/disaggregation/test_disaggregation_basic.py +++ b/test/registered/disaggregation/test_disaggregation_basic.py @@ -72,6 +72,32 @@ def test_logprob(self): len(input_logprobs) > 0 ), f"input_logprobs should have at least one token, but got {len(input_logprobs)}" + def test_chat_completion_top_logprobs(self): + client = openai.Client(api_key="empty", base_url=f"{self.lb_url}/v1") + response = client.chat.completions.create( + model="dummy", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + max_tokens=8, + logprobs=True, + top_logprobs=5, + ) + + self.assertIsNotNone(response.choices[0].logprobs) + content_logprobs = response.choices[0].logprobs.content + self.assertGreater(len(content_logprobs), 0) + + first_top_logprobs = next( + (item.top_logprobs for item in content_logprobs if item.top_logprobs), + None, + ) + self.assertIsNotNone(first_top_logprobs) + self.assertGreater(len(first_top_logprobs), 0) + self.assertIsInstance(first_top_logprobs[0].token, str) + def test_structured_output(self): json_schema = json.dumps( { diff --git a/test/registered/distributed/test_disaggregation_decode_radix_cache.py b/test/registered/disaggregation/test_disaggregation_decode_radix_cache.py similarity index 100% rename from test/registered/distributed/test_disaggregation_decode_radix_cache.py rename to test/registered/disaggregation/test_disaggregation_decode_radix_cache.py diff --git a/test/registered/distributed/test_disaggregation_different_tp.py b/test/registered/disaggregation/test_disaggregation_different_tp.py similarity index 100% rename from test/registered/distributed/test_disaggregation_different_tp.py rename to test/registered/disaggregation/test_disaggregation_different_tp.py diff --git a/test/registered/distributed/test_disaggregation_dp_attention.py b/test/registered/disaggregation/test_disaggregation_dp_attention.py similarity index 100% rename from test/registered/distributed/test_disaggregation_dp_attention.py rename to test/registered/disaggregation/test_disaggregation_dp_attention.py diff --git a/test/registered/distributed/test_disaggregation_dsv4.py b/test/registered/disaggregation/test_disaggregation_dsv4.py similarity index 81% rename from test/registered/distributed/test_disaggregation_dsv4.py rename to test/registered/disaggregation/test_disaggregation_dsv4.py index e20513fbc971..61e9c8efa821 100644 --- a/test/registered/distributed/test_disaggregation_dsv4.py +++ b/test/registered/disaggregation/test_disaggregation_dsv4.py @@ -1,8 +1,7 @@ import unittest -from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -35,7 +34,10 @@ ] -class TestDisaggregationDSV4(PDDisaggregationServerBase): +class TestDisaggregationDSV4(PDDisaggregationServerBase, GSM8KMixin): + + gsm8k_accuracy_thres = 0.93 + @classmethod def setUpClass(cls): super().setUpClass() @@ -70,10 +72,10 @@ def start_prefill(cls): "--cuda-graph-max-bs", "128", "--max-running-requests", - "256", - "--mem-fraction-static", - "0.7", + "128", *_EAGLE_SPEC_ARGS, + "--watchdog-timeout", + "900", ] prefill_args += cls.transfer_backend + cls.rdma_devices cls.process_prefill = popen_launch_pd_server( @@ -106,10 +108,10 @@ def start_decode(cls): "--cuda-graph-max-bs", "128", "--max-running-requests", - "256", - "--mem-fraction-static", - "0.7", + "128", *_EAGLE_SPEC_ARGS, + "--watchdog-timeout", + "900", ] decode_args += cls.transfer_backend + cls.rdma_devices cls.process_decode = popen_launch_pd_server( @@ -120,21 +122,6 @@ def start_decode(cls): env=DSV4_FLASH_ENV, ) - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(f"Evaluation metrics: {metrics}") - - self.assertGreater(metrics["score"], 0.95) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/distributed/test_disaggregation_hybrid_attention.py b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py similarity index 62% rename from test/registered/distributed/test_disaggregation_hybrid_attention.py rename to test/registered/disaggregation/test_disaggregation_hybrid_attention.py index 240f6506e7b4..a993f8b58daa 100644 --- a/test/registered/distributed/test_disaggregation_hybrid_attention.py +++ b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py @@ -16,7 +16,7 @@ @unittest.skipIf(is_in_ci(), "Temporarily disable the flaky test.") -class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase): +class TestDisaggregationHybridAttentionGDN(PDDisaggregationServerBase): @classmethod def setUpClass(cls): super().setUpClass() @@ -88,7 +88,7 @@ def test_gsm8k(self): self.assertGreater(metrics["score"], 0.93) -class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBase): +class TestDisaggregationHybridAttentionGDNExtraBuffer(PDDisaggregationServerBase): @classmethod def setUpClass(cls): super().setUpClass() @@ -165,7 +165,7 @@ def test_gsm8k(self): self.assertGreater(metrics["score"], 0.90) -class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase): +class TestDisaggregationHybridAttentionGDNDPDecode(PDDisaggregationServerBase): """Test with prefill tp=2 and decode tp=2/dp=2 with dp-attention enabled.""" @classmethod @@ -244,5 +244,153 @@ def test_gsm8k(self): self.assertGreater(metrics["score"], 0.90) +class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--disaggregation-bootstrap-port", + cls.bootstrap_port, + "--tp", + "4", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--disaggregation-bootstrap-port", + cls.bootstrap_port, + "--tp", + "4", + "--base-gpu-id", + "4", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["score"], 0.87) + + +class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--disaggregation-bootstrap-port", + cls.bootstrap_port, + "--tp", + "4", + "--mamba-scheduler-strategy", + "extra_buffer", + ] + prefill_args += cls.transfer_backend + cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--disaggregation-bootstrap-port", + cls.bootstrap_port, + "--tp", + "4", + "--base-gpu-id", + "4", + "--mamba-scheduler-strategy", + "extra_buffer", + ] + decode_args += cls.transfer_backend + cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["score"], 0.87) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/distributed/test_disaggregation_pp.py b/test/registered/disaggregation/test_disaggregation_pp.py similarity index 100% rename from test/registered/distributed/test_disaggregation_pp.py rename to test/registered/disaggregation/test_disaggregation_pp.py diff --git a/test/registered/distributed/test_epd_disaggregation.py b/test/registered/disaggregation/test_epd_disaggregation.py similarity index 99% rename from test/registered/distributed/test_epd_disaggregation.py rename to test/registered/disaggregation/test_epd_disaggregation.py index a115db259ec9..0c370fcdaab5 100644 --- a/test/registered/distributed/test_epd_disaggregation.py +++ b/test/registered/disaggregation/test_epd_disaggregation.py @@ -36,7 +36,7 @@ QWEN35_27B_MODEL = "Qwen/Qwen3.5-27B" -register_cuda_ci(est_time=97, suite="nightly-4-gpu", nightly=True) +register_cuda_ci(est_time=97, stage="base-c", runner_config="4-gpu-h100") @unittest.skipIf( diff --git a/test/registered/distributed/test_dp_attention.py b/test/registered/dp_attn/test_dp_attention.py similarity index 100% rename from test/registered/distributed/test_dp_attention.py rename to test/registered/dp_attn/test_dp_attention.py diff --git a/test/registered/distributed/test_data_parallelism.py b/test/registered/dp_engine/test_data_parallelism.py similarity index 100% rename from test/registered/distributed/test_data_parallelism.py rename to test/registered/dp_engine/test_data_parallelism.py diff --git a/test/registered/kernels/test_cp_prefix_len_fa3_parity.py b/test/registered/kernels/test_cp_prefix_len_fa3_parity.py new file mode 100644 index 000000000000..11d7acbe59d6 --- /dev/null +++ b/test/registered/kernels/test_cp_prefix_len_fa3_parity.py @@ -0,0 +1,143 @@ +""" +FA3 parity test for `prepare_context_parallel_metadata`. + +Drives the real function and feeds its `kv_len_prev/next_tensor` into FA3 +via `flash_attn_with_kvcache`. Compares per-rank CP output against a +full-sequence FA3 reference computed over the unpadded `(prefix + extend)` +KV. Any discrepancy indicates the metadata function emitted wrong +`cache_seqlens` for at least one rank. +""" + +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.layers.utils.cp_utils import prepare_context_parallel_metadata +from sglang.srt.utils.common import ceil_align +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=5, stage="extra-a", runner_config="1-gpu-large") + +_DSA_UTILS = "sglang.srt.layers.attention.dsa.utils" +_DEVICE = "cuda" +_DTYPE = torch.bfloat16 +_HEAD_NUM = 8 +_HEAD_DIM = 128 +_SCALE = _HEAD_DIM**-0.5 + + +class TestCPPrefixLenFA3Parity(CustomTestCase): + """Per-rank FA3 output under CP must match a full-sequence reference.""" + + def _run_parity(self, prefix_len: int, extend_len: int, cp_size: int): + from sgl_kernel.flash_attn import flash_attn_with_kvcache + + torch.manual_seed(extend_len * 1_000_003 + prefix_len * 101 + cp_size) + + padded_extend = ceil_align(extend_len, cp_size) + pad = padded_extend - extend_len + self.assertGreaterEqual( + padded_extend, + 2 * cp_size, + "runtime `can_cp_split` would skip this case; pick a larger extend", + ) + + # Reference: one full-sequence FA3 call over the unpadded KV. + q_full = torch.randn( + extend_len, _HEAD_NUM, _HEAD_DIM, device=_DEVICE, dtype=_DTYPE + ) + k_full = torch.randn( + prefix_len + extend_len, _HEAD_NUM, _HEAD_DIM, device=_DEVICE, dtype=_DTYPE + ) + v_full = torch.randn( + prefix_len + extend_len, _HEAD_NUM, _HEAD_DIM, device=_DEVICE, dtype=_DTYPE + ) + ref = flash_attn_with_kvcache( + q=q_full.unsqueeze(0), + k_cache=k_full.unsqueeze(0), + v_cache=v_full.unsqueeze(0), + cache_seqlens=torch.tensor( + [k_full.shape[0]], dtype=torch.int32, device=_DEVICE + ), + softmax_scale=_SCALE, + causal=True, + ).squeeze(0) + + # CP path sees tensors padded to `ceil_align(extend, cp_size)`, + # matching what `prepare_mlp_sync_batch` does in production. + zeros = torch.zeros(pad, _HEAD_NUM, _HEAD_DIM, device=_DEVICE, dtype=_DTYPE) + q_padded = torch.cat([q_full, zeros], dim=0) + k_padded = torch.cat([k_full, zeros], dim=0) + v_padded = torch.cat([v_full, zeros], dim=0) + + seqs_len = [prefix_len + extend_len] + extend_lens = [extend_len] + + def _call_meta(rank: int): + return prepare_context_parallel_metadata( + padded_extend, rank, cp_size, seqs_len, extend_lens=extend_lens + ) + + # Exercise the non-DSA branch; the DSA branch uses a separate + # `prefix_len` pathway re-added by `_get_topk_ragged_with_cp`. + with ( + patch(f"{_DSA_UTILS}.is_dsa_enable_prefill_cp", return_value=False), + patch( + f"{_DSA_UTILS}.is_dsa_prefill_cp_round_robin_split", + return_value=False, + ), + ): + meta0 = _call_meta(0) + cp_segment_num = 2 * cp_size + blocks_q = list(torch.split(q_padded, meta0.split_list, dim=0)) + outs = [None] * cp_segment_num + + for rank in range(cp_size): + meta = meta0 if rank == 0 else _call_meta(rank) + for idx, cs_tensor in ( + (rank, meta.kv_len_prev_tensor), + (cp_size * 2 - rank - 1, meta.kv_len_next_tensor), + ): + if meta0.split_list[idx] == 0: + outs[idx] = torch.empty( + 0, _HEAD_NUM, _HEAD_DIM, device=_DEVICE, dtype=_DTYPE + ) + continue + outs[idx] = flash_attn_with_kvcache( + q=blocks_q[idx].unsqueeze(0), + k_cache=k_padded.unsqueeze(0), + v_cache=v_padded.unsqueeze(0), + cache_seqlens=cs_tensor, + softmax_scale=_SCALE, + causal=True, + ).squeeze(0) + + cp_out = torch.cat(outs, dim=0) + err = (cp_out[:extend_len].float() - ref.float()).abs().max().item() + + self.assertLess( + err, + 1e-2, + f"CP output diverges from full-sequence FA3 reference by " + f"max_err={err:.5f} " + f"(prefix_len={prefix_len}, extend_len={extend_len}, " + f"cp_size={cp_size}, pad={pad})", + ) + + def test_cp2_prefix1_extend3(self): + """cp_size=2, prefix_len=1, extend_len=3 (pad=1).""" + self._run_parity(prefix_len=1, extend_len=3, cp_size=2) + + def test_cp4_prefix1_extend7(self): + """cp_size=4, prefix_len=1, extend_len=7 (pad=1).""" + self._run_parity(prefix_len=1, extend_len=7, cp_size=4) + + def test_cp8_prefix1_extend17(self): + """cp_size=8, prefix_len=1, extend_len=17 (pad=7).""" + self._run_parity(prefix_len=1, extend_len=17, cp_size=8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/test_dsa_indexer.py b/test/registered/kernels/test_dsa_indexer.py index 09021180fa17..e735030f3d70 100644 --- a/test/registered/kernels/test_dsa_indexer.py +++ b/test/registered/kernels/test_dsa_indexer.py @@ -4,6 +4,7 @@ import torch +from sglang.srt.environ import envs from sglang.srt.layers import dp_attention as _dp_attn from sglang.test.ci.ci_register import register_cuda_ci @@ -16,7 +17,15 @@ Indexer, rotate_activation, ) -from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend +from sglang.srt.layers.attention.dsa.dsa_topk_backend import ( + DSATopKBackend, + TopkTransformMethod, +) +from sglang.srt.layers.attention.dsa_backend import ( + DeepseekSparseAttnBackend, + DSAIndexerMetadata, + DSAMetadata, +) from sglang.srt.layers.layernorm import LayerNorm from sglang.srt.layers.linear import LinearBase from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool @@ -250,8 +259,10 @@ def __init__(self, config=None): "enable_deterministic_inference": False, "dsa_prefill_backend": "flashmla_sparse", "dsa_decode_backend": "fa3", + "dsa_topk_backend": "sgl-kernel", }, )() + self.hisparse_coordinator = None @unittest.skipIf(not torch.cuda.is_available(), "Test requires CUDA") @@ -360,7 +371,6 @@ def _create_forward_batch( ), extend_seq_lens=torch.tensor([q_len] * batch_size, device=self.device), extend_seq_lens_cpu=torch.tensor([q_len] * batch_size, device="cpu"), - attn_backend=self.backend, ) else: # ForwardMode.DECODE decode_len = 1 @@ -379,12 +389,18 @@ def _create_forward_batch( req_pool_indices=torch.arange(batch_size, device=self.device), seq_lens=torch.tensor([total_len] * batch_size, device=self.device), seq_lens_cpu=torch.tensor([total_len] * batch_size, device="cpu"), - attn_backend=self.backend, ) - # Add token pools - forward_batch.req_to_token_pool = self.model_runner.req_to_token_pool - forward_batch.token_to_kv_pool = self.model_runner.token_to_kv_pool + # Pool refs + attn_backend are now resolved via the ForwardContext; + # publish ``self.backend`` for the duration of this fixture call so + # ``get_attn_backend()`` / ``get_token_to_kv_pool()`` / + # ``get_req_to_token_pool()`` resolve correctly. + from sglang.srt.model_executor.forward_context import ( + ForwardContext, + set_forward_context, + ) + + set_forward_context(ForwardContext(attn_backend=self.backend)) # Mock write to req_to_token_pool page_size = self.model_runner.page_size @@ -417,6 +433,271 @@ def _verify_topk_output(self, topk_indices, batch_size, q_len, topk): "Output should have padding or exact topk size", ) + def _make_tie_free_logits( + self, batch_size: int, max_score_len: int + ) -> torch.Tensor: + perm = torch.argsort( + torch.randn( + batch_size, max_score_len, dtype=torch.float32, device=self.device + ), + dim=-1, + ) + return torch.gather( + torch.arange(max_score_len, device=self.device, dtype=torch.float32) + .unsqueeze(0) + .expand(batch_size, -1), + dim=1, + index=perm, + ) + + def _run_unfused_topk_backend_validity_test( + self, + batch_size: int, + max_score_len: int, + topk: int, + topk_backend: DSATopKBackend, + with_row_starts: bool, + ): + logits = self._make_tie_free_logits(batch_size, max_score_len) + + if with_row_starts: + row_starts = torch.randint( + 0, + max_score_len - 1, + (batch_size,), + dtype=torch.int32, + device=self.device, + ) + max_lengths = max_score_len - row_starts + random_lengths = torch.randint( + 0, + max_score_len - 1, + (batch_size,), + dtype=torch.int32, + device=self.device, + ) + seq_lens_expanded = torch.minimum(max_lengths, random_lengths) + else: + row_starts = None + seq_lens_expanded = torch.randint( + 0, + max_score_len - 1, + (batch_size,), + dtype=torch.int32, + device=self.device, + ) + + seq_lens_expanded = seq_lens_expanded.to(dtype=torch.int32, device=self.device) + max_seq_len_k = int(seq_lens_expanded.max().item()) + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=self.device + ) + dsa_cu_seqlens_k = torch.zeros( + batch_size + 1, dtype=torch.int32, device=self.device + ) + dsa_cu_seqlens_k[1:] = torch.cumsum(seq_lens_expanded, dim=0) + page_table_1 = ( + torch.arange(max_seq_len_k, dtype=torch.int32, device=self.device) + .unsqueeze(0) + .expand(batch_size, -1) + .contiguous() + ) + metadata = DSAIndexerMetadata( + attn_metadata=DSAMetadata( + page_size=1, + cache_seqlens_int32=seq_lens_expanded.clone(), + max_seq_len_q=1, + max_seq_len_k=max_seq_len_k, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_q.clone(), + page_table_1=page_table_1, + real_page_table=page_table_1, + dsa_cache_seqlens_int32=seq_lens_expanded.clone(), + dsa_cu_seqlens_q=cu_seqlens_q.clone(), + dsa_cu_seqlens_k=dsa_cu_seqlens_k, + dsa_extend_seq_lens_list=seq_lens_expanded.cpu().tolist(), + dsa_seqlens_expanded=seq_lens_expanded, + ), + topk_transform_method=TopkTransformMethod.PAGED, + topk_backend=topk_backend, + ) + + with envs.SGLANG_DSA_FUSE_TOPK.override(False): + topk_test = metadata.topk_transform(logits, topk, ks=row_starts) + self.assertEqual(topk_test.shape, (batch_size, topk)) + self.assertEqual(topk_test.dtype, torch.int32) + expected_valid = torch.minimum( + seq_lens_expanded, + torch.full_like(seq_lens_expanded, topk), + ) + actual_valid = (topk_test >= 0).sum(dim=-1).to(torch.int32) + self.assertTrue(torch.equal(actual_valid, expected_valid)) + + starts = ( + row_starts.to(torch.int32) + if row_starts is not None + else torch.zeros( + (topk_test.shape[0],), dtype=torch.int32, device=topk_test.device + ) + ) + for row in range(topk_test.shape[0]): + test_row = topk_test[row] + valid_test = test_row[test_row >= 0] + expected_k = int(expected_valid[row].item()) + self.assertEqual(valid_test.numel(), expected_k) + if expected_k == 0: + continue + start = int(starts[row].item()) + row_len = int(seq_lens_expanded[row].item()) + self.assertTrue(torch.all((valid_test >= 0) & (valid_test < row_len))) + self.assertEqual(torch.unique(valid_test).numel(), valid_test.numel()) + + row_scores = logits[row, start : start + row_len] + ref_topk = torch.topk(row_scores, expected_k, dim=-1, sorted=False).indices + self.assertTrue( + torch.equal( + torch.sort(valid_test.to(torch.int32)).values, + torch.sort(ref_topk.to(torch.int32)).values, + ) + ) + + def _run_fused_topk_backend_equivalence_test( + self, + batch_size: int, + max_score_len: int, + topk: int, + topk_transform_method: TopkTransformMethod, + with_row_starts: bool, + query_lens: Optional[List[int]] = None, + ): + num_rows = sum(query_lens) if query_lens is not None else batch_size + logits = self._make_tie_free_logits(num_rows, max_score_len) + + if with_row_starts: + row_starts = torch.randint( + 0, + max_score_len - 1, + (num_rows,), + dtype=torch.int32, + device=self.device, + ) + max_lengths = max_score_len - row_starts + random_lengths = torch.randint( + 1, + max_score_len, + (num_rows,), + dtype=torch.int32, + device=self.device, + ) + seq_lens_expanded = torch.minimum(max_lengths, random_lengths) + else: + row_starts = None + seq_lens_expanded = torch.randint( + 1, + max_score_len, + (num_rows,), + dtype=torch.int32, + device=self.device, + ) + + topk_indices_offset = ( + torch.arange(num_rows, dtype=torch.int32, device=self.device) + * max_score_len + ) + if query_lens is None: + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=self.device + ) + q_lens = None + batch_idx_list = None + else: + q_lens = torch.tensor(query_lens, dtype=torch.int32, device=self.device) + cu_seqlens_q = torch.zeros( + batch_size + 1, dtype=torch.int32, device=self.device + ) + cu_seqlens_q[1:] = torch.cumsum(q_lens, dim=0) + batch_idx_list = list(range(batch_size)) + cu_seqlens_k = torch.zeros( + batch_size + 1, dtype=torch.int32, device=self.device + ) + dsa_cu_seqlens_k = torch.zeros( + num_rows + 1, dtype=torch.int32, device=self.device + ) + dsa_cu_seqlens_k[1:] = torch.cumsum(seq_lens_expanded, dim=0) + + page_table_1 = ( + ( + torch.arange(max_score_len, dtype=torch.int32, device=self.device) + .unsqueeze(0) + .expand(batch_size, -1) + ) + + ( + torch.arange( + batch_size, dtype=torch.int32, device=self.device + ).unsqueeze(1) + * max_score_len + ) + ).contiguous() + + attn_metadata = DSAMetadata( + page_size=1, + cache_seqlens_int32=seq_lens_expanded.clone(), + max_seq_len_q=1, + max_seq_len_k=max_score_len, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + page_table_1=page_table_1, + real_page_table=page_table_1, + dsa_cache_seqlens_int32=seq_lens_expanded.clone(), + dsa_cu_seqlens_q=cu_seqlens_q.clone(), + dsa_cu_seqlens_k=dsa_cu_seqlens_k, + dsa_extend_seq_lens_list=seq_lens_expanded.cpu().tolist(), + dsa_seqlens_expanded=seq_lens_expanded, + topk_indices_offset=( + topk_indices_offset + if topk_transform_method == TopkTransformMethod.RAGGED + else None + ), + ) + + metadata_sgl = DSAIndexerMetadata( + attn_metadata=attn_metadata, + topk_transform_method=topk_transform_method, + topk_backend=DSATopKBackend.SGL_KERNEL, + ) + metadata_flashinfer = DSAIndexerMetadata( + attn_metadata=attn_metadata, + topk_transform_method=topk_transform_method, + topk_backend=DSATopKBackend.FLASHINFER, + ) + + with envs.SGLANG_DSA_FUSE_TOPK.override(True): + out_sgl = metadata_sgl.topk_transform( + logits, + topk, + ks=row_starts, + cu_seqlens_q=q_lens, + batch_idx_list=batch_idx_list, + ) + out_flashinfer = metadata_flashinfer.topk_transform( + logits, + topk, + ks=row_starts, + cu_seqlens_q=q_lens, + batch_idx_list=batch_idx_list, + ) + + self.assertEqual(out_sgl.shape, out_flashinfer.shape) + self.assertEqual(out_sgl.dtype, out_flashinfer.dtype) + self.assertEqual(out_sgl.dtype, torch.int32) + + self.assertTrue( + torch.equal( + torch.sort(out_sgl, dim=-1).values, + torch.sort(out_flashinfer, dim=-1).values, + ) + ) + @patch("sglang.srt.layers.attention.dsa.dsa_indexer.deep_gemm") def test_indexer_basic_creation(self, mock_deep_gemm): """Test basic indexer creation and initialization.""" @@ -626,6 +907,86 @@ def test_indexer_metadata_interface(self): topk_indices = metadata.topk_transform(logits, topk) self.assertEqual(topk_indices.shape, (batch_size, topk)) + def test_topk_unfused_backends_valid_selection(self): + batch_size = 8 + max_score_len = 16 * 1024 + topk = 2048 + for topk_backend in [ + DSATopKBackend.SGL_KERNEL, + DSATopKBackend.TORCH, + DSATopKBackend.FLASHINFER, + ]: + tie_break_values = ( + [None, "small", "large"] + if topk_backend == DSATopKBackend.FLASHINFER + else [None] + ) + for tie_break in tie_break_values: + for with_row_starts in [False, True]: + with self.subTest( + topk_backend=topk_backend.value, + tie_break=tie_break, + with_row_starts=with_row_starts, + ): + with envs.SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK.override( + tie_break + ): + self._run_unfused_topk_backend_validity_test( + batch_size, + max_score_len, + topk, + topk_backend=topk_backend, + with_row_starts=with_row_starts, + ) + + def test_topk_fused_backends_equivalence(self): + batch_size = 8 + max_score_len = 16 * 1024 + topk = 2048 + for tie_break in [None, "small", "large"]: + for topk_transform_method in [ + TopkTransformMethod.PAGED, + TopkTransformMethod.RAGGED, + ]: + for with_row_starts in [False, True]: + if ( + topk_transform_method == TopkTransformMethod.PAGED + and with_row_starts + ): + # The synthetic paged fixture uses the decode-like row mapping. + # Ragged fused and unfused cases cover shifted row windows. + continue + with self.subTest( + tie_break=tie_break, + topk_transform_method=topk_transform_method.name, + with_row_starts=with_row_starts, + ): + with envs.SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK.override( + tie_break + ): + self._run_fused_topk_backend_equivalence_test( + batch_size=batch_size, + max_score_len=max_score_len, + topk=topk, + topk_transform_method=topk_transform_method, + with_row_starts=with_row_starts, + ) + with self.subTest( + tie_break=tie_break, + topk_transform_method=TopkTransformMethod.PAGED.name, + with_row_starts=False, + query_lens="multi", + ): + with envs.SGLANG_DSA_TOPK_FLASHINFER_TIE_BREAK.override(tie_break): + self._run_fused_topk_backend_equivalence_test( + batch_size=batch_size, + max_score_len=max_score_len, + topk=topk, + topk_transform_method=TopkTransformMethod.PAGED, + with_row_starts=False, + query_lens=[1, 2, 3, 1, 2, 1, 3, 2], + ) + # TODO: enable this test after indexer accuracy aligned # @patch("sglang.srt.layers.attention.dsa.dsa_indexer.deep_gemm") # def test_indexer_with_different_topk(self, mock_deep_gemm): diff --git a/test/registered/kernels/test_mla_cp_fa3_parity.py b/test/registered/kernels/test_mla_cp_fa3_parity.py new file mode 100644 index 000000000000..9e2a4564aead --- /dev/null +++ b/test/registered/kernels/test_mla_cp_fa3_parity.py @@ -0,0 +1,203 @@ +"""FA3 numerical parity for MLA prefill CP. + +Verifies the rank-local zigzag-split FA3 path (``_mla_cp_attn`` + +``cp_attn_forward_extend`` in ``flashattention_backend.py``) matches a +single non-CP ``flash_attn_with_kvcache`` over the full sequence. + +Single-process, single-layer, pre-populated paged KV cache. Requires +FA3 ver=3 (Hopper+). +""" + +import math +import sys +from types import SimpleNamespace + +import pytest +import torch + +from sglang.srt.layers.utils.cp_utils import ( + ContextParallelMetadata, + cp_attn_forward_extend, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="extra-a", runner_config="1-gpu-large") + +if not torch.cuda.is_available(): + pytest.skip(reason="CUDA required for FA3", allow_module_level=True) + +_cap = torch.cuda.get_device_capability(0) +if _cap[0] < 9: + pytest.skip( + reason=f"FA3 ver=3 requires Hopper (sm90+); got sm{_cap[0]}{_cap[1]}", + allow_module_level=True, + ) + +try: + from sgl_kernel.flash_attn import flash_attn_with_kvcache +except ImportError as e: + pytest.skip( + reason=f"sgl_kernel.flash_attn unavailable: {e}", + allow_module_level=True, + ) + +DEVICE = torch.device("cuda") +DTYPE = torch.bfloat16 + +# Default shape is DeepSeek V3/R1 TP=8 MLA: 16 heads, v=512, rope=64. +NUM_HEADS = 16 +V_HEAD_DIM = 512 +QK_ROPE_HEAD_DIM = 64 +PAGE_SIZE = 1 + + +def _build_cache_and_q(seq_len): + """Pre-populated paged KV cache + full-sequence q. + + Pre-population mirrors upstream ``rebuild_cp_kv_cache``, which all-gathers + rank-local KV into the global pool before the attention call, so each + rank's FA3 invocation sees the same fully-populated cache. + """ + num_pages = (seq_len + PAGE_SIZE - 1) // PAGE_SIZE + c_kv_cache = torch.randn( + num_pages, PAGE_SIZE, 1, V_HEAD_DIM, dtype=DTYPE, device=DEVICE + ) + k_rope_cache = torch.randn( + num_pages, PAGE_SIZE, 1, QK_ROPE_HEAD_DIM, dtype=DTYPE, device=DEVICE + ) + q_nope = torch.randn(seq_len, NUM_HEADS, V_HEAD_DIM, dtype=DTYPE, device=DEVICE) + q_rope = torch.randn( + seq_len, NUM_HEADS, QK_ROPE_HEAD_DIM, dtype=DTYPE, device=DEVICE + ) + page_table = torch.arange(num_pages, dtype=torch.int32, device=DEVICE).unsqueeze(0) + return c_kv_cache, k_rope_cache, q_nope, q_rope, page_table + + +def _full_seq_attn( + seq_len, q_nope, q_rope, c_kv_cache, k_rope_cache, page_table, softmax_scale +): + """Non-CP reference: single flash_attn_with_kvcache over the full seq.""" + return flash_attn_with_kvcache( + q=q_rope, + qv=q_nope, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + page_table=page_table, + cache_seqlens=torch.tensor([seq_len], dtype=torch.int32, device=DEVICE), + cu_seqlens_q=torch.tensor([0, seq_len], dtype=torch.int32, device=DEVICE), + cu_seqlens_k_new=None, + max_seqlen_q=seq_len, + softmax_scale=softmax_scale, + causal=True, + ver=3, + ) + + +def _cp_attn_for_rank( + rank, + cp_size, + block_size, + q_nope, + q_rope, + c_kv_cache, + k_rope_cache, + page_table, + softmax_scale, +): + """Run the rank-local CP closure from ``flashattention_backend.py``. + + Zigzag layout: rank r gets blocks [r, num_blocks - 1 - r] where + num_blocks = cp_size * 2. kv_len for each half is the cumulative KV + extent through the end of that block. + """ + num_blocks = cp_size * 2 + b_prev, b_next = rank, num_blocks - 1 - rank + prev_slice = slice(b_prev * block_size, (b_prev + 1) * block_size) + next_slice = slice(b_next * block_size, (b_next + 1) * block_size) + + q_nope_local = torch.cat([q_nope[prev_slice], q_nope[next_slice]], dim=0) + q_rope_local = torch.cat([q_rope[prev_slice], q_rope[next_slice]], dim=0) + q_fused = torch.cat([q_nope_local, q_rope_local], dim=-1) + + cp_meta = ContextParallelMetadata( + kv_len_prev_tensor=torch.tensor( + [(b_prev + 1) * block_size], dtype=torch.int32, device=DEVICE + ), + kv_len_next_tensor=torch.tensor( + [(b_next + 1) * block_size], dtype=torch.int32, device=DEVICE + ), + actual_seq_q_prev=block_size, + actual_seq_q_next=block_size, + ) + fb = SimpleNamespace(attn_cp_metadata=cp_meta) + + def _mla_cp_attn(q_chunk, cu_seqlens_q_cp, cache_seqlens_cp, max_seqlen_q_cp): + q_nope_chunk = q_chunk[..., :V_HEAD_DIM] + q_rope_chunk = q_chunk[..., V_HEAD_DIM:] + return flash_attn_with_kvcache( + q=q_rope_chunk, + qv=q_nope_chunk, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + page_table=page_table, + cache_seqlens=cache_seqlens_cp, + cu_seqlens_q=cu_seqlens_q_cp, + cu_seqlens_k_new=None, + max_seqlen_q=max_seqlen_q_cp, + softmax_scale=softmax_scale, + causal=True, + ver=3, + ) + + local_out = cp_attn_forward_extend(fb, q_fused, DEVICE, _mla_cp_attn) + return local_out, prev_slice, next_slice + + +@pytest.mark.parametrize( + "cp_size, block_size", + [ + (2, 64), # DSv3 TP=8 baseline + (2, 128), # longer per-block seq + (4, 32), # multi-rank zigzag: rank r gets blocks [r, 7-r] + ], +) +def test_cp_parity(cp_size, block_size): + torch.manual_seed(0) + seq_len = block_size * cp_size * 2 + softmax_scale = 1.0 / math.sqrt(V_HEAD_DIM + QK_ROPE_HEAD_DIM) + + c_kv_cache, k_rope_cache, q_nope, q_rope, page_table = _build_cache_and_q(seq_len) + ref_out = _full_seq_attn( + seq_len, q_nope, q_rope, c_kv_cache, k_rope_cache, page_table, softmax_scale + ) + + for rank in range(cp_size): + local_out, prev_slice, next_slice = _cp_attn_for_rank( + rank, + cp_size, + block_size, + q_nope, + q_rope, + c_kv_cache, + k_rope_cache, + page_table, + softmax_scale, + ) + torch.testing.assert_close( + local_out[:block_size], + ref_out[prev_slice], + rtol=1e-3, + atol=5e-3, + msg=f"rank={rank} prev-half mismatch", + ) + torch.testing.assert_close( + local_out[block_size:], + ref_out[next_slice], + rtol=1e-3, + atol=5e-3, + msg=f"rank={rank} next-half mismatch", + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/lora/test_lora_overlap_loading.py b/test/registered/lora/test_lora_overlap_loading.py index ea7765cf6528..2d187b20e724 100644 --- a/test/registered/lora/test_lora_overlap_loading.py +++ b/test/registered/lora/test_lora_overlap_loading.py @@ -65,7 +65,10 @@ def setUp(self): self.mock_lora_manager = MagicMock(spec=LoRAManager) self.mock_lora_manager.device = "cuda:0" + self.mock_lora_manager.memory_pool = MagicMock() + self.mock_lora_manager.memory_pool.uid_to_buffer_id = {} self.mock_lora_manager.validate_lora_batch.return_value = True + self.mock_lora_manager.fetch_new_loras.side_effect = self._mark_loras_loaded def tearDown(self): self.torch_patcher.stop() @@ -73,11 +76,85 @@ def tearDown(self): def _create_loader(self) -> LoRAOverlapLoader: return LoRAOverlapLoader(cast(LoRAManager, self.mock_lora_manager)) + def _mark_loras_loaded(self, new_loras, _loras_to_be_loaded): + for lora_id in new_loras: + self.mock_lora_manager.memory_pool.uid_to_buffer_id[lora_id] = len( + self.mock_lora_manager.memory_pool.uid_to_buffer_id + ) + def _create_mock_event(self, query_return: bool = False) -> MagicMock: event = MagicMock(spec=CudaEvent) event.query.return_value = query_return return event + def test_completed_stale_loads_are_reaped_before_capacity_check(self): + loader = self._create_loader() + events = [ + self._create_mock_event(query_return=True), + self._create_mock_event(query_return=False), + ] + self.mock_device_module.Event.side_effect = events + self.mock_lora_manager.validate_lora_batch.side_effect = ( + lambda lora_ids: len(lora_ids) <= 1 + ) + + self.assertTrue( + loader._try_start_overlap_load("stale_lora", running_loras=set()) + ) + self.assertIn("stale_lora", loader.lora_to_overlap_load_event) + + self.mock_lora_manager.fetch_new_loras.reset_mock() + result = loader.try_overlap_load_lora("new_lora", running_loras=set()) + + self.assertFalse(result) + self.assertNotIn("stale_lora", loader.lora_to_overlap_load_event) + self.assertIn("new_lora", loader.lora_to_overlap_load_event) + self.mock_lora_manager.fetch_new_loras.assert_called_once_with( + {"new_lora"}, set() + ) + + def test_loaded_lora_reused_after_stale_event_drain(self): + loader = self._create_loader() + self.mock_lora_manager.memory_pool = MagicMock() + self.mock_lora_manager.memory_pool.uid_to_buffer_id = {} + events = [ + self._create_mock_event(query_return=True), + self._create_mock_event(query_return=False), + ] + self.mock_device_module.Event.side_effect = events + self.mock_lora_manager.validate_lora_batch.side_effect = ( + lambda lora_ids: len(lora_ids) <= 2 + ) + + self.assertTrue(loader._try_start_overlap_load("lora_A", running_loras=set())) + self.assertIn("lora_A", loader.lora_to_overlap_load_event) + + self.mock_lora_manager.fetch_new_loras.reset_mock() + self.assertFalse(loader.try_overlap_load_lora("lora_B", running_loras=set())) + self.assertNotIn("lora_A", loader.lora_to_overlap_load_event) + self.assertIn("lora_B", loader.lora_to_overlap_load_event) + self.mock_lora_manager.fetch_new_loras.assert_called_once_with( + {"lora_B"}, set() + ) + + self.mock_lora_manager.fetch_new_loras.reset_mock() + self.assertTrue(loader.try_overlap_load_lora("lora_A", running_loras=set())) + self.assertIn("lora_B", loader.lora_to_overlap_load_event) + self.mock_lora_manager.fetch_new_loras.assert_not_called() + + def test_pending_lora_load_must_complete_even_if_memory_pool_has_slot(self): + loader = self._create_loader() + self.mock_lora_manager.memory_pool = MagicMock() + self.mock_lora_manager.memory_pool.uid_to_buffer_id = {"lora_A": 0} + + loader.lora_to_overlap_load_event["lora_A"] = self._create_mock_event(False) + + result = loader.try_overlap_load_lora("lora_A", running_loras=set()) + + self.assertFalse(result) + self.mock_lora_manager.fetch_new_loras.assert_not_called() + self.assertIn("lora_A", loader.lora_to_overlap_load_event) + def test_full_lifecycle_single_lora_load(self): loader = self._create_loader() @@ -131,6 +208,7 @@ def test_capacity_constraints_block_new_loads(self): # First lora completes, freeing capacity loader.lora_to_overlap_load_event["lora_0"].query.return_value = True + loader._drain_completed_overlap_loads() self.assertEqual( loader._check_overlap_load_status("lora_0"), LoRAOverlapLoadStatus.LOADED ) diff --git a/test/registered/distributed/test_load_weights_from_remote_instance.py b/test/registered/model_loading/test_load_weights_from_remote_instance.py similarity index 100% rename from test/registered/distributed/test_load_weights_from_remote_instance.py rename to test/registered/model_loading/test_load_weights_from_remote_instance.py diff --git a/test/registered/distributed/test_load_weights_from_remote_instance_npu.py b/test/registered/model_loading/test_load_weights_from_remote_instance_npu.py similarity index 100% rename from test/registered/distributed/test_load_weights_from_remote_instance_npu.py rename to test/registered/model_loading/test_load_weights_from_remote_instance_npu.py diff --git a/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py b/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py index 4fa3fc7b6729..f8536767dd4b 100644 --- a/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py +++ b/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py @@ -20,7 +20,7 @@ try_cached_model, ) -register_cuda_ci(est_time=700, stage="base-c", runner_config="dsv4-4-gpu-b200") +register_cuda_ci(est_time=465, stage="base-c", runner_config="dsv4-4-gpu-b200") MODEL = "deepseek-ai/DeepSeek-V4-Flash" SERVER_LAUNCH_TIMEOUT = 3600 @@ -120,12 +120,10 @@ def tearDownClass(cls): kill_process_tree(cls.process.pid) -class TestDSV4FlashFP4B200Balanced_CP( - BasicDecodeCorrectnessMixin, - GSM8KMixin, - CustomTestCase, +class TestDSV4FlashFP4NonMTPB200( + BasicDecodeCorrectnessMixin, GSM8KMixin, CustomTestCase ): - """Balanced recipe: TP=4, DP=4, DeepEP, EAGLE (1-step spec).""" + """Non-MTP recipe: TP=4, DP=4, DeepEP, no speculative decoding.""" gsm8k_accuracy_thres = 0.93 @@ -141,22 +139,11 @@ def setUpClass(cls): "--trust-remote-code", "--tp", "4", - "--attn-cp-size", + "--dp", "4", "--enable-dp-attention", "--moe-a2a-backend", "deepep", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "1", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "2", - "--enable-dsa-prefill-context-parallel", - "--dsa-prefill-cp-mode", - "round-robin-split", "--deepep-config", DEEPEP_CONFIG, ], diff --git a/test/registered/models_e2e/test_deepseek_v4_flash_fp4_h200.py b/test/registered/models_e2e/test_deepseek_v4_flash_fp4_h200.py index 674a48668766..51d352667627 100644 --- a/test/registered/models_e2e/test_deepseek_v4_flash_fp4_h200.py +++ b/test/registered/models_e2e/test_deepseek_v4_flash_fp4_h200.py @@ -131,5 +131,37 @@ def tearDownClass(cls): kill_process_tree(cls.process.pid) +class TestDSV4FlashFP4NonMTPH200( + BasicDecodeCorrectnessMixin, GSM8KMixin, CustomTestCase +): + """LowLatency recipe without MTP: TP=4, Marlin FP4, no speculative decoding.""" + + gsm8k_accuracy_thres = 0.93 + + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(MODEL) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--moe-runner-backend", + "marlin", + "--watchdog-timeout", + "900", + ], + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + kill_process_tree(cls.process.pid) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py index 06e73443dd04..9690fc6ec29f 100644 --- a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py +++ b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py @@ -143,6 +143,7 @@ def setUpClass(cls): "8", "--moe-dense-tp-size", "1", + "--enable-flashinfer-allreduce-fusion", ], ) diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index 7e812b88181b..a793da567a93 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -1,3 +1,4 @@ +import os import unittest from typing import Dict, List @@ -8,6 +9,8 @@ from sglang.srt.environ import envs from sglang.srt.observability.metrics_collector import ( ROUTING_KEY_REQ_COUNT_BUCKET_BOUNDS, + STAT_LOGGER_ROLE_SCHEDULER, + SchedulerMetricsCollector, compute_routing_key_stats, ) from sglang.srt.utils import kill_process_tree @@ -274,6 +277,66 @@ def _check_metrics_positive(test_case, metrics, metrics_to_check): test_case.assertGreater(value, 0, f"{metric_name} {labels}") +_DI_MARKER_PATH = "/tmp/sglang_di_test_marker" + + +class _MarkingSchedulerCollector(SchedulerMetricsCollector): + """Records its own instantiation to a file so the test can verify the + custom subclass was used in the scheduler subprocess. + + Defined at module level so it is picklable into the scheduler process. + Cross-process signalling uses a filesystem marker because the scheduler + runs in its own subprocess and cannot share in-memory state with the + test runner. + """ + + def __init__(self, *args, **kwargs): + with open(_DI_MARKER_PATH, "w") as f: + f.write("scheduler_collector_initialized\n") + super().__init__(*args, **kwargs) + + +class TestStatLoggersDI(CustomTestCase): + """Verify that a custom MetricsCollector subclass passed through + ``ServerArgs.stat_loggers`` is the one instantiated inside the + scheduler subprocess.""" + + def setUp(self) -> None: + try: + os.unlink(_DI_MARKER_PATH) + except FileNotFoundError: + pass + + def tearDown(self) -> None: + try: + os.unlink(_DI_MARKER_PATH) + except FileNotFoundError: + pass + + def test_engine_custom_scheduler_collector(self): + import sglang as sgl + + engine = sgl.Engine( + model_path=_MODEL_NAME, + enable_metrics=True, + stat_loggers={ + STAT_LOGGER_ROLE_SCHEDULER: _MarkingSchedulerCollector, + }, + ) + try: + # One small generation triggers scheduler init, which is where + # resolve_collector_class() picks the injected subclass. + engine.generate("Hello", {"max_new_tokens": 4}) + finally: + engine.shutdown() + + self.assertTrue( + os.path.exists(_DI_MARKER_PATH), + "Custom SchedulerMetricsCollector was not instantiated; " + "stat_loggers DI did not take effect.", + ) + + class TestComputeRoutingKeyStats(unittest.TestCase): def test_empty(self): num_unique, req_counts = compute_routing_key_stats([]) diff --git a/test/registered/piecewise_cuda_graph/test_pcg_glm5_fp4.py b/test/registered/piecewise_cuda_graph/test_pcg_glm5_fp4.py new file mode 100644 index 000000000000..6cce520debf1 --- /dev/null +++ b/test/registered/piecewise_cuda_graph/test_pcg_glm5_fp4.py @@ -0,0 +1,71 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=900, stage="base-c", runner_config="4-gpu-b200") + +GLM5_FP4_MODEL = "nvidia/GLM-5-NVFP4" + + +class TestPCGGlm5Fp4(CustomTestCase): + """PCG prefill on GLM-5-NVFP4 (DSA model, TP=4, B200). + + GLM-5 uses GlmMoeDsaForCausalLM (DSA attention). This test verifies that + piecewise CUDA graph works correctly after the DSA indexer was updated to + cache k_fp8/k_scale for PCG-compatible prefill. + """ + + @classmethod + def setUpClass(cls): + cls.model = GLM5_FP4_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--trust-remote-code", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", + "--quantization", + "modelopt_fp4", + "--disable-flashinfer-autotune", + "--enforce-piecewise-cuda-graph", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + num_examples=200, + num_threads=200, + max_tokens=4096, + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.92) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/distributed/test_pp_single_node.py b/test/registered/pp/test_pp_single_node.py similarity index 100% rename from test/registered/distributed/test_pp_single_node.py rename to test/registered/pp/test_pp_single_node.py diff --git a/test/registered/quant/test_is_layer_skipped.py b/test/registered/quant/test_is_layer_skipped.py index c311fc5764b2..637edb74fc05 100644 --- a/test/registered/quant/test_is_layer_skipped.py +++ b/test/registered/quant/test_is_layer_skipped.py @@ -47,6 +47,17 @@ def test_mlp_gate_does_not_match_gate_up_proj(self): ) self.assertTrue(is_layer_skipped("model.layers.0.mlp.gate", ignored, {})) + def test_trailing_dot_prefix_matches_child_modules(self): + # Mixed-precision checkpoints may use a trailing-dot layer prefix to keep + # every module under the layer in higher precision. + ignored = ["model.layers.34."] + self.assertTrue( + is_layer_skipped("model.layers.34.mlp.experts.0.down_proj", ignored, {}) + ) + self.assertFalse( + is_layer_skipped("model.layers.340.mlp.experts.0.down_proj", ignored, {}) + ) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py index 9f30b9b33b59..1e0344b7218a 100644 --- a/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py +++ b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py @@ -24,7 +24,7 @@ DSV4_FLASH_MODEL = "sgl-project/DeepSeek-V4-Flash-FP8" DSV4_FLASH_LAUNCH_TIMEOUT = 3600 -register_cuda_ci(est_time=745, stage="base-c", runner_config="8-gpu-h200") +register_cuda_ci(est_time=768, stage="base-c", runner_config="8-gpu-h200") class TestUnifiedMambaHiCache(UnifiedRadixTreeTestMixin, CustomTestCase): diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl_hicache_nightly.py b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache_nightly.py index 799e94a311f9..16efd022020f 100644 --- a/test/registered/radix_cache/test_unified_radix_cache_kl_hicache_nightly.py +++ b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache_nightly.py @@ -28,8 +28,8 @@ register_cuda_ci(est_time=900, suite="nightly-8-gpu-h200", nightly=True) -class GSM8KTwoPassMixin: - """Mixin: run GSM8K twice with flush in between, verify accuracy diff. +class AccuracyTwoPassMixin: + """Mixin: run an eval twice with flush in between, verify accuracy diff. Subclass must provide: - self.base_url @@ -38,9 +38,14 @@ class GSM8KTwoPassMixin: gsm8k_threshold: float = 0.90 num_gsm8k_questions: int = 200 - max_accuracy_diff: float = 0.02 gsm8k_parallel: int = 40 + mmlu_threshold: float = 0.75 + num_mmlu_examples: int = 200 + mmlu_num_threads: int = 32 + + max_accuracy_diff: float = 0.02 + def _run_gsm8k(self): from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_gsm8k @@ -57,6 +62,19 @@ def _run_gsm8k(self): metrics = run_few_shot_gsm8k(args) return metrics["accuracy"] + def _run_mmlu(self): + from sglang.test.run_eval import run_eval as run_simple_eval + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=self.num_mmlu_examples, + num_threads=self.mmlu_num_threads, + ) + metrics = run_simple_eval(args) + return metrics["score"] + def _flush_cache(self): response = requests.post( self.base_url + "/flush_cache", @@ -65,45 +83,52 @@ def _flush_cache(self): ) response.raise_for_status() - def test_gsm8k_two_passes(self): - """Run GSM8K twice with flush in between, verify accuracy diff <= max_accuracy_diff.""" + def _two_pass(self, name: str, run_fn, threshold: float): # First pass - acc1 = self._run_gsm8k() - print(f"[{self.__class__.__name__}] GSM8K pass 1 accuracy: {acc1:.3f}") + acc1 = run_fn() + print(f"[{self.__class__.__name__}] {name} pass 1 accuracy: {acc1:.3f}") self.assertGreaterEqual( acc1, - self.gsm8k_threshold, - f"Pass 1 accuracy {acc1:.3f} < threshold {self.gsm8k_threshold}", + threshold, + f"{name} pass 1 accuracy {acc1:.3f} < threshold {threshold}", ) # Flush cache self._flush_cache() # Second pass - acc2 = self._run_gsm8k() - print(f"[{self.__class__.__name__}] GSM8K pass 2 accuracy: {acc2:.3f}") + acc2 = run_fn() + print(f"[{self.__class__.__name__}] {name} pass 2 accuracy: {acc2:.3f}") self.assertGreaterEqual( acc2, - self.gsm8k_threshold, - f"Pass 2 accuracy {acc2:.3f} < threshold {self.gsm8k_threshold}", + threshold, + f"{name} pass 2 accuracy {acc2:.3f} < threshold {threshold}", ) - # Verify diff + # Verify diff (only fail when 2nd pass regressed) if acc1 > acc2: diff = abs(acc1 - acc2) print( - f"[{self.__class__.__name__}] Accuracy diff: {diff:.3f} " + f"[{self.__class__.__name__}] {name} accuracy diff: {diff:.3f} " f"(max allowed: {self.max_accuracy_diff})" ) self.assertLessEqual( diff, self.max_accuracy_diff, - f"Accuracy diff {diff:.3f} exceeds max {self.max_accuracy_diff} " + f"{name} accuracy diff {diff:.3f} exceeds max {self.max_accuracy_diff} " f"(pass1={acc1:.3f}, pass2={acc2:.3f})", ) + def test_gsm8k_two_passes(self): + """Run GSM8K twice with flush in between, verify accuracy diff <= max_accuracy_diff.""" + self._two_pass("GSM8K", self._run_gsm8k, self.gsm8k_threshold) + + def test_mmlu_two_passes(self): + """Run MMLU twice with flush in between, verify accuracy diff <= max_accuracy_diff.""" + self._two_pass("MMLU", self._run_mmlu, self.mmlu_threshold) + -class TestGLM5HiCacheL3GSM8K(GSM8KTwoPassMixin, CustomTestCase): +class TestGLM5HiCacheL3Accuracy(AccuracyTwoPassMixin, CustomTestCase): """GLM-5.1-FP8 + HiCache L3 (file backend), with HiRadixTree.""" @classmethod diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl_hicache_part2.py b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache_part2.py new file mode 100644 index 000000000000..99331421bd5e --- /dev/null +++ b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache_part2.py @@ -0,0 +1,77 @@ +import os +import shutil +import tempfile +import unittest + +from test_unified_radix_cache_kl_hicache_nightly import AccuracyTwoPassMixin + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct" +MAMBA_TRACK_INTERVAL = 128 + +register_cuda_ci(est_time=768, stage="base-c", runner_config="8-gpu-h200") + + +class TestUnifiedMambaHiCacheL3(AccuracyTwoPassMixin, CustomTestCase): + """Mamba hybrid + HiCache L3 (file backend) + UnifiedRadixCache.""" + + @classmethod + def setUpClass(cls): + cls.model = MAMBA_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.hicache_dir = tempfile.mkdtemp(prefix="hicache_l3_mamba_") + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--chunked-prefill-size", + "2048", + "--mem-fraction-static", + "0.85", + "--mamba-scheduler-strategy", + "extra_buffer", + "--mamba-track-interval", + str(MAMBA_TRACK_INTERVAL), + "--enable-hierarchical-cache", + "--hicache-ratio", + "2", + "--hicache-write-policy", + "write_through", + "--hicache-storage-prefetch-policy", + "wait_complete", + "--hicache-io-backend", + "direct", + "--hicache-mem-layout", + "page_first_direct", + "--hicache-storage-backend", + "file", + "--max-mamba-cache-size", + "500", + "--weight-loader-prefetch-checkpoints", + ], + env={ + "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1", + "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.hicache_dir, + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + if os.path.isdir(cls.hicache_dir): + shutil.rmtree(cls.hicache_dir, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/sessions/test_session_latency.py b/test/registered/sessions/test_session_latency.py index c0f79e9d45da..b47b2598ae0c 100644 --- a/test/registered/sessions/test_session_latency.py +++ b/test/registered/sessions/test_session_latency.py @@ -2,13 +2,9 @@ Benchmark: Streaming Session Inter-Turn Latency Tests: - 1. Latency (bs=8): regular vs streaming, assert speedup >= 2x - 2. Correctness (bs=1): regular vs streaming, assert output equal + speedup + 1. Stability (bs=8): streaming only, assert tail_avg / head_avg <= 1.15 + 2. Correctness (bs=1): regular vs streaming, assert output equal 3. Random lengths (bs=8): streaming only, random input/output lens, no crash - -Usage: - python -m pytest test_session_latency.py -s - python -m unittest test_session_latency.BenchSessionLatency """ import random @@ -16,7 +12,7 @@ import unittest from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field -from typing import Dict, List, Optional +from typing import List, Optional import requests from tabulate import tabulate @@ -37,6 +33,7 @@ INPUT_LEN = 16 GEN_LEN = 8 NUM_CONCURRENT = 8 +HEAD_TURNS = 10 TAIL_TURNS = 10 SAMPLE_TURNS = 8 @@ -65,8 +62,6 @@ class TurnResult: turn: int context_len: int cached_tokens: int - prompt_tokens: int - completion_tokens: int client_latency_ms: float e2e_latency_ms: float @@ -78,11 +73,6 @@ class ModeResult: outputs: List[str] = field(default_factory=list) -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - def _generate_input_chunks( tokenizer, num_turns: int, input_len: int, offset: int = 0 ) -> List[List[int]]: @@ -149,18 +139,11 @@ def _record_turn( turn=turn_idx + 1, context_len=context_len, cached_tokens=meta["cached_tokens"], - prompt_tokens=meta["prompt_tokens"], - completion_tokens=meta["completion_tokens"], client_latency_ms=client_latency_ms, e2e_latency_ms=meta.get("e2e_latency", 0) * 1000, ) -# --------------------------------------------------------------------------- -# Single-session runner (called by worker threads) -# --------------------------------------------------------------------------- - - def _run_one_session( base_url: str, chunks: List[List[int]], @@ -215,19 +198,20 @@ def _run_one_session( return result -# --------------------------------------------------------------------------- -# Stats & reporting -# --------------------------------------------------------------------------- - - def _collect_latencies( - results: List[ModeResult], last_n: Optional[int] = None + results: List[ModeResult], + last_n: Optional[int] = None, + first_n: Optional[int] = None, ) -> List[float]: lats = [] for r in results: - turns = r.turns[1:] # skip turn 1 if last_n is not None: turns = r.turns[-last_n:] + elif first_n is not None: + # Skip turn 1 (includes prefill), then take next `first_n` turns. + turns = r.turns[1 : 1 + first_n] + else: + turns = r.turns[1:] # skip turn 1 lats.extend(t.client_latency_ms for t in turns) return lats @@ -270,49 +254,6 @@ def _print_mode_table(result: ModeResult, label: str = ""): ) -def _print_summary(all_results: Dict[str, List[ModeResult]]): - stats = [ - ( - mode, - _avg(_collect_latencies(rs)), - _avg(_collect_latencies(rs, last_n=TAIL_TURNS)), - ) - for mode, rs in all_results.items() - ] - base_all, base_tail = (stats[0][1] or 1.0), (stats[0][2] or 1.0) - tail_label = f"last {TAIL_TURNS}" - - print(f"\n SUMMARY ({NUM_CONCURRENT} sessions x {NUM_TURNS} turns)") - rows = [ - [ - mode, - f"{a:.1f}ms", - f"{t:.1f}ms", - f"{base_all / a:.2f}x" if a else "inf", - f"{base_tail / t:.2f}x" if t else "inf", - ] - for mode, a, t in stats - ] - print( - tabulate( - rows, - headers=[ - "Mode", - "Avg (all)", - f"Avg ({tail_label})", - "Speedup (all)", - f"Speedup ({tail_label})", - ], - colalign=("left", "right", "right", "right", "right"), - ) - ) - - -# --------------------------------------------------------------------------- -# Test class -# --------------------------------------------------------------------------- - - class TestSessionLatency(CustomTestCase): @classmethod def setUpClass(cls): @@ -346,12 +287,8 @@ def setUpClass(cls): }, ) - cls.all_results: Dict[str, List[ModeResult]] = {} - @classmethod def tearDownClass(cls): - if len(cls.all_results) > 1: - _print_summary(cls.all_results) kill_process_tree(cls.process.pid) def _run_concurrent_session( @@ -391,36 +328,30 @@ def run_one(session_idx): with ThreadPoolExecutor(max_workers=num_concurrent) as pool: return list(pool.map(run_one, range(num_concurrent))) - # ------------------------------------------------------------------ - # Test methods (alphabetical order matters for dependencies) - # ------------------------------------------------------------------ - - def test_regular_session(self): - """Run regular (non-streaming) sessions for latency baseline.""" - results = self._run_concurrent_session(streaming=False) - self.__class__.all_results["regular_session"] = results - _print_mode_table(results[0], label="session 0") - def test_streaming_session(self): - """Latency test: bs=8, assert streaming >= 2x faster than regular.""" + """Stability: streaming reuses KV across turns, so tail/head latency + should stay flat. Skip turn 1 (prefill) when computing head.""" results = self._run_concurrent_session(streaming=True) - self.__class__.all_results["streaming_session"] = results _print_mode_table(results[0], label="session 0") - reg_list = self.__class__.all_results.get("regular_session") - if reg_list: - reg_tail = _avg(_collect_latencies(reg_list, last_n=TAIL_TURNS)) - stm_tail = _avg(_collect_latencies(results, last_n=TAIL_TURNS)) - speedup = reg_tail / stm_tail if stm_tail > 0 else float("inf") - self.assertGreaterEqual( - speedup, - 1.4, - f"streaming should be >=1.4x faster on last {TAIL_TURNS} turns " - f"(regular={reg_tail:.1f}ms, streaming={stm_tail:.1f}ms, speedup={speedup:.2f}x)", - ) + head_avg = _avg(_collect_latencies(results, first_n=HEAD_TURNS)) + tail_avg = _avg(_collect_latencies(results, last_n=TAIL_TURNS)) + ratio = tail_avg / head_avg if head_avg > 0 else float("inf") + print( + f"\n streaming_session " + f"head_avg(first {HEAD_TURNS})={head_avg:.1f}ms " + f"tail_avg(last {TAIL_TURNS})={tail_avg:.1f}ms " + f"ratio={ratio:.2f}" + ) + self.assertLessEqual( + ratio, + 1.15, + f"streaming latency should stay flat across turns " + f"(head={head_avg:.1f}ms, tail={tail_avg:.1f}ms, ratio={ratio:.2f} > 1.15)", + ) def test_streaming_session_correctness(self): - """Correctness test: bs=1, assert output equal + latency speedup.""" + """Correctness test: bs=1, assert regular and streaming outputs match.""" correctness_turns = 30 reg = self._run_concurrent_session( streaming=False, num_concurrent=1, num_turns=correctness_turns diff --git a/test/registered/spec/dflash/test_dflash.py b/test/registered/spec/dflash/test_dflash.py index d139cbf54b42..221957861469 100644 --- a/test/registered/spec/dflash/test_dflash.py +++ b/test/registered/spec/dflash/test_dflash.py @@ -55,8 +55,7 @@ def setUpClass(cls): try: with ( envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1), - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), + envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True), ): cls.process = popen_launch_server( cls.model, diff --git a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py index 99b3a2e88e2a..b6fd25cb03dd 100644 --- a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py +++ b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py @@ -49,10 +49,7 @@ def setUpClass(cls): "--model-loader-extra-config", '{"enable_multithread_load": true,"num_threads": 64}', ] - with ( - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), - ): + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/spec/eagle/test_eagle_constrained_decoding.py b/test/registered/spec/eagle/test_eagle_constrained_decoding.py index 85e7d75e7c4c..267897c5c662 100644 --- a/test/registered/spec/eagle/test_eagle_constrained_decoding.py +++ b/test/registered/spec/eagle/test_eagle_constrained_decoding.py @@ -62,8 +62,7 @@ def setUpClass(cls): launch_args.extend(cls.other_launch_args) with ( envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2), - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), + envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True), ): cls.process = popen_launch_server( cls.model, diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index b31b38ebf915..8fc48cb1bba1 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -60,10 +60,7 @@ def setUpClass(cls): "--cuda-graph-max-bs", "64", ] - with ( - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), - ): + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/spec/eagle/test_eagle_infer_beta.py b/test/registered/spec/eagle/test_eagle_infer_beta.py index 92577ca0ff91..e0b50b0a8b5b 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta.py @@ -65,8 +65,7 @@ def setUpClass(cls): launch_args.extend(cls.other_launch_args) with ( envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1), - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), + envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True), envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True), ): cls.process = popen_launch_server( diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py index 9e4bf5676190..ebd6cc1bf8a5 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py @@ -65,10 +65,7 @@ def setUpClass(cls): "--speculative-num-draft-tokens", "4", ] - with ( - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), - ): + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py index 0c1d63ec9490..8e6c9c1b2971 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py @@ -73,10 +73,7 @@ def setUpClass(cls): "--model-loader-extra-config", '{"enable_multithread_load": true,"num_threads": 64}', ] - with ( - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), - ): + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/spec/test_constrained_decoding_spec_reasoning.py b/test/registered/spec/test_constrained_decoding_spec_reasoning.py index 60e695bc097f..add3849cf2c0 100644 --- a/test/registered/spec/test_constrained_decoding_spec_reasoning.py +++ b/test/registered/spec/test_constrained_decoding_spec_reasoning.py @@ -51,10 +51,7 @@ def setUpClass(cls): "--speculative-num-draft-tokens=8", ] - with ( - envs.SGLANG_SPEC_NAN_DETECTION.override(True), - envs.SGLANG_SPEC_OOB_DETECTION.override(True), - ): + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True): cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/registered/distributed/test_parallel_state.py b/test/registered/unit/distributed/test_parallel_state.py similarity index 100% rename from test/registered/distributed/test_parallel_state.py rename to test/registered/unit/distributed/test_parallel_state.py diff --git a/test/registered/unit/entrypoints/test_server_info.py b/test/registered/unit/entrypoints/test_server_info.py new file mode 100644 index 000000000000..acd3443e3511 --- /dev/null +++ b/test/registered/unit/entrypoints/test_server_info.py @@ -0,0 +1,296 @@ +"""Endpoint-level tests for `/server_info`. + +`/server_info` is the introspection surface that external consumers +(SGLang's own deprecated `/get_server_info` alias, monitoring tools, +KV-aware routers) scrape to learn about the running server's +configuration. New `/server_info` behaviours should add their test +classes to this file as the surface grows. + +Current coverage: + +* `TestServerInfoKvEventsField` — the `kv_events` publisher descriptor + surfaced by `_build_kv_events_block`. Covers the full input matrix + end-to-end (happy path / disabled / malformed JSON / inproc endpoint / + port edge cases / missing-or-non-positive page_size) because the + helper has no separate test target; the handler is its only caller. + +* `TestServerInfoExistingFieldsPreserved` — regression guard that no + field existing consumers depend on is silently dropped: every + `ServerArgs` dataclass field, `internal_states`, `version`, and the + pre-existing flat `kv_events_config` string all remain visible. +""" + +import asyncio +import dataclasses +import unittest +from types import SimpleNamespace + +from sglang.srt.entrypoints import http_server +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _call_server_info_with(server_args: ServerArgs) -> dict: + """Invoke `http_server.server_info()` against a stub global state. + + Bypasses the FastAPI HTTP layer (no TestClient): the handler is an + `async def` that reads module-level `_global_state`, so wiring a + `SimpleNamespace` stub via `set_global_state` and awaiting the + coroutine directly is enough to exercise the handler logic without + booting a model server. + """ + + async def _fake_internal_state(): + return [{"max_req_input_len": 1024}] + + stub_state = SimpleNamespace( + tokenizer_manager=SimpleNamespace( + server_args=server_args, + get_internal_state=_fake_internal_state, + ), + scheduler_info={"max_req_input_len": 1024}, + ) + prior_state = http_server.get_global_state() + http_server.set_global_state(stub_state) + try: + return asyncio.run(http_server.server_info()) + finally: + # Restore so a later test in the same process isn't surprised. + http_server._global_state = prior_state + + +class TestServerInfoKvEventsField(CustomTestCase): + """The new `kv_events` field is wired correctly across the full + `_build_kv_events_block` input matrix. + """ + + # ----- happy path -------------------------------------------------- + + def test_kv_events_key_present_when_publishing_enabled(self): + args = ServerArgs( + model_path="dummy", + kv_events_config=( + '{"publisher": "zmq", "endpoint": "tcp://*:5557", "topic": "kv"}' + ), + page_size=64, + dp_size=2, + ) + + info = _call_server_info_with(args) + + self.assertIn("kv_events", info) + self.assertEqual( + info["kv_events"], + { + "publisher": "zmq", + "endpoint_host": "*", + "endpoint_port_base": 5557, + "topic": "kv", + "block_size": 64, + "dp_size": 2, + }, + ) + + def test_kv_events_descriptor_carries_specific_host_and_topic(self): + args = ServerArgs( + model_path="dummy", + kv_events_config=( + '{"publisher": "zmq", "endpoint": "tcp://0.0.0.0:7777", "topic": "kv"}' + ), + page_size=128, + dp_size=1, + ) + + info = _call_server_info_with(args) + + self.assertIsNotNone(info["kv_events"]) + self.assertEqual(info["kv_events"]["endpoint_host"], "0.0.0.0") + self.assertEqual(info["kv_events"]["endpoint_port_base"], 7777) + self.assertEqual(info["kv_events"]["topic"], "kv") + self.assertEqual(info["kv_events"]["block_size"], 128) + self.assertEqual(info["kv_events"]["dp_size"], 1) + + # ----- disabled / unconfigured ------------------------------------- + + def test_kv_events_is_null_when_no_publisher_configured(self): + args = ServerArgs(model_path="dummy") # no --kv-events-config + + info = _call_server_info_with(args) + + # The key must still be present so consumers can detect + # "publishing disabled" via a single shape check. + self.assertIn("kv_events", info) + self.assertIsNone(info["kv_events"]) + + def test_kv_events_is_null_when_publisher_explicitly_null(self): + args = ServerArgs( + model_path="dummy", + kv_events_config='{"publisher": "null"}', + page_size=64, + ) + + info = _call_server_info_with(args) + + self.assertIsNone(info["kv_events"]) + + # ----- malformed config -------------------------------------------- + + def test_kv_events_is_null_for_malformed_json(self): + # Not JSON — the publisher would have failed at server startup, + # but /server_info must keep working. + args = ServerArgs( + model_path="dummy", + kv_events_config="not-json", + page_size=64, + ) + + info = _call_server_info_with(args) + + self.assertIsNone(info["kv_events"]) + + # ----- unreachable endpoints --------------------------------------- + + def test_kv_events_is_null_for_inproc_endpoint(self): + # `inproc://` is not reachable across process boundaries, so the + # descriptor must hide it from external routers. + args = ServerArgs( + model_path="dummy", + kv_events_config=( + '{"publisher": "zmq", "endpoint": "inproc://cache", "topic": ""}' + ), + page_size=64, + ) + + info = _call_server_info_with(args) + + self.assertIsNone(info["kv_events"]) + + def test_kv_events_is_null_when_endpoint_missing_port(self): + args = ServerArgs( + model_path="dummy", + kv_events_config=( + '{"publisher": "zmq", "endpoint": "tcp://0.0.0.0", "topic": ""}' + ), + page_size=64, + ) + + info = _call_server_info_with(args) + + self.assertIsNone(info["kv_events"]) + + def test_kv_events_is_null_when_port_not_integer(self): + args = ServerArgs( + model_path="dummy", + kv_events_config=( + '{"publisher": "zmq", "endpoint": "tcp://0.0.0.0:abc", "topic": ""}' + ), + page_size=64, + ) + + info = _call_server_info_with(args) + + self.assertIsNone(info["kv_events"]) + + def test_kv_events_is_null_for_port_out_of_range(self): + # TCP ports are 1..65535; values outside the range can't bind, so + # the descriptor refuses to advertise them rather than handing + # subscribers a non-dialable address. + for bad_port in (0, -1, 65536, 1_000_000): + with self.subTest(port=bad_port): + args = ServerArgs( + model_path="dummy", + kv_events_config=( + f'{{"publisher": "zmq", "endpoint": "tcp://0.0.0.0:{bad_port}", "topic": ""}}' + ), + page_size=64, + ) + info = _call_server_info_with(args) + self.assertIsNone(info["kv_events"]) + + # ----- bad scheduler context --------------------------------------- + + def test_kv_events_is_null_when_page_size_missing_or_non_positive(self): + # Without a real positive `page_size` the descriptor's + # `block_size` would be a misleading placeholder; subscribers + # would hash prompts at the wrong granularity and miss every + # cache entry. Refuse to advertise instead. + good_cfg = '{"publisher": "zmq", "endpoint": "tcp://*:5557", "topic": ""}' + for bad_page_size in (None, 0, -1): + with self.subTest(page_size=bad_page_size): + args = ServerArgs( + model_path="dummy", + kv_events_config=good_cfg, + page_size=bad_page_size, + ) + info = _call_server_info_with(args) + self.assertIsNone(info["kv_events"]) + + +class TestServerInfoExistingFieldsPreserved(CustomTestCase): + """Regression guard: the new `kv_events` field is additive — none of + the fields existing consumers depend on may be silently dropped. + + Existing `/server_info` consumers in the wild include: + * SGLang's own deprecated `/get_server_info` (forwards to the + same handler). + * External monitoring tools that scrape the full ServerArgs. + * KV-aware routers reading `kv_events_config`, `page_size`, + `dp_size` directly to derive subscription info (this is the + path the new `kv_events` block enriches but does not replace). + """ + + def test_every_server_args_field_appears_in_response(self): + # `dataclasses.asdict(server_args)` is spread into the response; + # asserting every dataclass field surfaces is the strongest + # backward-compat guarantee that's still implementation-agnostic. + args = ServerArgs(model_path="dummy") + + info = _call_server_info_with(args) + + for field in dataclasses.fields(ServerArgs): + self.assertIn( + field.name, + info, + f"existing ServerArgs field '{field.name}' missing from " + f"/server_info response — kv_events patch must not " + f"shadow or drop ServerArgs fields", + ) + + def test_internal_states_and_version_keys_preserved(self): + # These two top-level keys predate the kv_events patch and are + # named individually (not spread from a dataclass), so a stray + # edit could remove them without breaking syntax. Lock them down. + args = ServerArgs(model_path="dummy") + + info = _call_server_info_with(args) + + self.assertIn("internal_states", info) + self.assertIn("version", info) + + def test_kv_events_config_raw_field_still_surfaced(self): + # The new structured `kv_events` block sits alongside the + # pre-existing flat `kv_events_config` field (the raw CLI string + # already on ServerArgs). Both must remain visible so + # consumers that hand-parse the raw config keep working. + raw_cfg = '{"publisher": "zmq", "endpoint": "tcp://*:5557", "topic": ""}' + args = ServerArgs( + model_path="dummy", + kv_events_config=raw_cfg, + page_size=64, + dp_size=1, + ) + + info = _call_server_info_with(args) + + self.assertIn("kv_events_config", info) + self.assertEqual(info["kv_events_config"], raw_cfg) + # And the new structured block is separately present: + self.assertIn("kv_events", info) + self.assertIsNotNone(info["kv_events"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/function_call/test_minicpm5_detector.py b/test/registered/unit/function_call/test_minicpm5_detector.py new file mode 100644 index 000000000000..8be593668684 --- /dev/null +++ b/test/registered/unit/function_call/test_minicpm5_detector.py @@ -0,0 +1,289 @@ +import json + +import pytest + +from sglang.srt.entrypoints.openai.protocol import Function, Tool +from sglang.srt.function_call.minicpm5_detector import ( + MiniCPM5Detector, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(1.0, "base-a-test-cpu") + + +def make_tools_weather(): + return [ + Tool( + function=Function( + name="get_weather", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "date": {"type": "string"}, + }, + "required": ["city"], + }, + ) + ) + ] + + +def make_tools_sum(): + return [ + Tool( + function=Function( + name="sum_values", + parameters={ + "type": "object", + "properties": { + "nums": {"type": "array"}, + "exact": {"type": "boolean"}, + }, + "required": ["nums"], + }, + ) + ) + ] + + +def make_tools_config(): + return [ + Tool( + function=Function( + name="set_config", + parameters={ + "type": "object", + "properties": { + "config": {"type": "object"}, + }, + "required": ["config"], + }, + ) + ) + ] + + +def make_tools_no_required(): + return [ + Tool( + function=Function( + name="noop", + parameters={ + "type": "object", + "properties": {"note": {"type": "string"}}, + "required": [], + }, + ) + ) + ] + + +def test_detect_and_parse_single_call_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = ( + "Intro before.\n" + '' + '上海' + '2024-06-27' + "\n" + "Outro after.\n" + ) + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 1 + args = json.loads(res.calls[0].parameters) + assert args["city"] == "上海" + assert args["date"] == "2024-06-27" + assert "Intro before." in res.normal_text and "Outro after." in res.normal_text + assert "" not in res.normal_text + + +def test_detect_and_parse_cdata_multiline_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = ( + '' + '' + '2024-06-27' + "\n" + ) + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 1 + args = json.loads(res.calls[0].parameters) + assert args["city"] == "北\n京" + assert args["date"] == "2024-06-27" + + +def test_unknown_tool_block_preserved_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = '' '1' "\n" + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 0 + assert "unknown" in res.normal_text + + +def test_non_string_types_v3(): + detector = MiniCPM5Detector() + tools = make_tools_sum() + text = ( + '' + '[1, 2, 3]' + 'true' + "\n" + ) + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 1 + args = json.loads(res.calls[0].parameters) + assert args["nums"] == [1, 2, 3] + assert args["exact"] is True + + +def test_multiple_calls_interleaved_text_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + make_tools_sum() + text = ( + "Head\n" + '北京\n' + "TXT\n" + '[7,8,9]false\n' + "Tail\n" + ) + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 2 + args0 = json.loads(res.calls[0].parameters) + assert args0["city"] == "北京" + args1 = json.loads(res.calls[1].parameters) + assert args1["nums"] == [7, 8, 9] + assert args1["exact"] is False + assert ( + "Head" in res.normal_text + and "TXT" in res.normal_text + and "Tail" in res.normal_text + ) + assert "" not in res.normal_text + + +def test_incomplete_missing_function_end_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = '' '北京' + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 0 + assert "get_weather" in res.normal_text + + +def test_param_missing_name_invalid_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = ( + '' + "北京" + '2024-06-27' + "\n" + ) + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 0 + assert "北京" in res.normal_text + + +def test_duplicate_param_names_invalid_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = ( + '' + '北京' + '上海' + "\n" + ) + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 0 + + +def test_case_sensitive_param_name_invalid_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = ( + '' + '北京' + "\n" + ) + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 0 + + +def test_no_required_and_zero_param_valid_v3(): + detector = MiniCPM5Detector() + tools = make_tools_no_required() + text = '\n' + res = detector.detect_and_parse(text, tools) + assert len(res.calls) == 1 + args = json.loads(res.calls[0].parameters) + assert args == {} + + +def test_streaming_increment_v3(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + c1 = 'Hello\n\n ' + c2 = '北京\n 2024-06-27\n\n' + + r1 = detector.parse_streaming_increment(c1, tools) + assert r1.normal_text == "Hello\n" + assert len(r1.calls) == 0 + + r2 = detector.parse_streaming_increment(c2, tools) + assert len(r2.calls) == 1 + args = json.loads(r2.calls[0].parameters) + assert args["city"] == "北京" + assert args["date"] == "2024-06-27" + + +def test_streaming_split_bot_token(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = ( + '' '北京' "" + ) + + r1 = detector.parse_streaming_increment("<", tools) + assert r1.normal_text == "" + assert len(r1.calls) == 0 + + r2 = detector.parse_streaming_increment(text[1:], tools) + assert len(r2.calls) == 1 + args = json.loads(r2.calls[0].parameters) + assert args["city"] == "北京" + + +def test_streaming_multiple_complete_blocks_in_one_delta(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + make_tools_sum() + text = ( + '北京' + '[1,2]' + ) + + result = detector.parse_streaming_increment(text, tools) + assert len(result.calls) == 2 + assert json.loads(result.calls[0].parameters)["city"] == "北京" + assert json.loads(result.calls[1].parameters)["nums"] == [1, 2] + + +def test_malformed_xml_with_unescaped_ampersand_falls_back_to_regex(): + detector = MiniCPM5Detector() + tools = make_tools_weather() + text = ( + '' 'A & B' "" + ) + + result = detector.detect_and_parse(text, tools) + assert len(result.calls) == 1 + assert json.loads(result.calls[0].parameters)["city"] == "A & B" + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index 7cfd7d843455..0263170bc839 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -1,6 +1,7 @@ """Regression tests for the SWA chunked-req stash gate (#24252).""" import unittest +from array import array from types import SimpleNamespace from unittest.mock import MagicMock @@ -27,9 +28,9 @@ def _make_req( ) -> Req: req = Req.__new__(Req) req.rid = "test-req" - req.origin_input_ids = list(fill_ids) - req.output_ids = [] - req.fill_ids = list(fill_ids) + req.origin_input_ids = array("q", fill_ids) + req.output_ids = array("q") + req.fill_ids = array("q", fill_ids) req.prefix_indices = prefix_indices req.req_pool_idx = req_pool_idx req.extend_input_len = extend_input_len diff --git a/test/registered/unit/managers/test_template_manager.py b/test/registered/unit/managers/test_template_manager.py index fe7d40c00076..9ab82c97c146 100644 --- a/test/registered/unit/managers/test_template_manager.py +++ b/test/registered/unit/managers/test_template_manager.py @@ -266,6 +266,17 @@ def test_tool_call_parser_rule_values_via_snippets(self): ["<|tool_calls_section_begin|>"], "kimi_k2", ), + ( + "minicpm5", + ( + "{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}" + '\n' + '\n{{ param.value }}' + "\n" + ), + ["", @@ -306,6 +317,25 @@ def test_unrecognized_template_returns_none(self): result = detect_tool_call_parser("Hello {{ user }}", None, config, force) self.assertIsNone(result) + def test_minicpm5_rule_precedes_broad_fallback_rules(self): + rule_names = [rule.name for rule in TOOL_CALL_PARSER_RULES] + minicpm5_idx = rule_names.index("minicpm5") + self.assertLess(minicpm5_idx, rule_names.index("mimo")) + self.assertLess(minicpm5_idx, rule_names.index("qwen")) + + def test_minicpm5_not_misclassified_as_qwen(self): + template = ( + "{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}" + '\n' + '\n{{ param.value }}' + "\n" + ) + force, config = detect_reasoning_pattern(template) + result = detect_tool_call_parser( + template, _DummyTokenizer([" 1 else list(fill_ids) + self.fill_ids = array("q", fill_ids) + self.origin_input_ids = array( + "q", fill_ids[:-1] if len(fill_ids) > 1 else fill_ids ) - self.output_ids = [fill_ids[-1]] if len(fill_ids) > 1 else [] + self.output_ids = array("q", [fill_ids[-1]] if len(fill_ids) > 1 else []) self.req_pool_idx = req_pool_idx self.cache_protected_len = cache_protected_len self.last_node = last_node @@ -99,7 +100,7 @@ def _populate_prefix(self, cache, prefix_ids, prefix_values): """Insert a prefix into the tree so future requests can match it.""" cache.insert( InsertParams( - key=RadixKey(prefix_ids), + key=RadixKey(array("q", prefix_ids)), value=torch.tensor(prefix_values, dtype=torch.int64), ) ) @@ -119,7 +120,7 @@ def test_incremental_transfer_success(self): self._populate_prefix(cache, prefix, prefix_vals) # Match prefix (simulates _match_prefix_and_lock in pop_preallocated) - result = cache.match_prefix(MatchPrefixParams(key=RadixKey(prefix))) + result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", prefix)))) matched_node = result.last_device_node prefix_len = len(result.device_indices) self.assertEqual(prefix_len, 3) @@ -164,7 +165,9 @@ def test_full_transfer_success(self): # No prefix in tree -- match returns root full_ids = [10, 20, 30] - result = cache.match_prefix(MatchPrefixParams(key=RadixKey(full_ids))) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", full_ids))) + ) matched_node = result.last_device_node self.assertEqual(len(result.device_indices), 0) # no match # matched_node is root @@ -212,7 +215,7 @@ def test_incremental_transfer_failure(self): self._populate_prefix(cache, prefix, prefix_vals) # Match and lock - result = cache.match_prefix(MatchPrefixParams(key=RadixKey(prefix))) + result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", prefix)))) matched_node = result.last_device_node prefix_len = len(result.device_indices) @@ -256,7 +259,9 @@ def test_full_transfer_failure(self): # No prefix in tree -- match returns root (simulates _match_prefix_and_lock) full_ids = [10, 20, 30] - result = cache.match_prefix(MatchPrefixParams(key=RadixKey(full_ids))) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", full_ids))) + ) matched_node = result.last_device_node self.assertIs(matched_node, cache.root_node) @@ -356,7 +361,9 @@ def test_repeated_incremental_no_leak(self): self._populate_prefix(cache, prefix, prefix_vals) for iteration in range(5): - result = cache.match_prefix(MatchPrefixParams(key=RadixKey(prefix))) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", prefix))) + ) matched_node = result.last_device_node prefix_len = len(result.device_indices) diff --git a/test/registered/unit/mem_cache/test_mamba_unittest.py b/test/registered/unit/mem_cache/test_mamba_unittest.py index f5ff1d2f1b03..315fb69d3092 100755 --- a/test/registered/unit/mem_cache/test_mamba_unittest.py +++ b/test/registered/unit/mem_cache/test_mamba_unittest.py @@ -1,10 +1,12 @@ import unittest +from array import array import torch from sglang.srt.configs.mamba_utils import Mamba2CacheParams, Mamba2StateShape from sglang.srt.disaggregation.kv_events import BlockRemoved, BlockStored from sglang.srt.environ import envs +from sglang.srt.layers.attention.fla.chunk_delta_h import CHUNK_SIZE as FLA_CHUNK_SIZE from sglang.srt.managers.schedule_batch import Req from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( @@ -116,7 +118,7 @@ def test_mamba_pool(self): req = Req( rid=0, origin_input_text="", - origin_input_ids=[], + origin_input_ids=array("q"), sampling_params=sampling_params, ) @@ -158,7 +160,7 @@ def test_mamba_radix_cache_1(self): print( f"req1: inserting, req1_token_ids: {req1_token_ids}, req1_kv_indices: {req1_kv_indices}" ) - key = RadixKey(req1_token_ids) + key = RadixKey(array("q", req1_token_ids)) result = tree.insert( InsertParams( key=key, @@ -176,7 +178,7 @@ def test_mamba_radix_cache_1(self): print( f"req2: inserting, req2_token_ids: {req2_token_ids}, req2_kv_indices: {req2_kv_indices}" ) - key = RadixKey(req2_token_ids) + key = RadixKey(array("q", req2_token_ids)) result = tree.insert( InsertParams( key=key, @@ -195,7 +197,7 @@ def test_mamba_radix_cache_1(self): print( f"req3: inserting, req3_token_ids: {req3_token_ids}, req3_kv_indices: {req3_kv_indices}" ) - key = RadixKey(req3_token_ids) + key = RadixKey(array("q", req3_token_ids)) result = tree.insert( InsertParams( key=key, @@ -213,7 +215,7 @@ def test_mamba_radix_cache_1(self): print( f"req4: inserting, req4_token_ids: {req4_token_ids}, req4_kv_indices: {req4_kv_indices}" ) - key = RadixKey(req4_token_ids) + key = RadixKey(array("q", req4_token_ids)) result = tree.insert( InsertParams( key=key, @@ -244,7 +246,9 @@ def test_mamba_radix_cache_1(self): tree.pretty_print() req5_token_ids = [1, 2, 3, 4, 5] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req5_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req5_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req5: token_ids: {req5_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -252,7 +256,9 @@ def test_mamba_radix_cache_1(self): assert len(kv_indices) == 0 req6_token_ids = [1, 2, 3, 4, 5, 60, 70] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req6_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req6_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req6: token_ids: {req6_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -261,7 +267,9 @@ def test_mamba_radix_cache_1(self): assert len(last_node.key) == 2 req7_token_ids = [1, 2, 3, 4, 5, 6, 7] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req7_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req7_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req7: token_ids: {req7_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -278,7 +286,9 @@ def test_mamba_radix_cache_1(self): tree.pretty_print() req8_token_ids = [1, 2, 3, 4, 5, 60, 70] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req8_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req8_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req8: token_ids: {req8_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -289,7 +299,9 @@ def test_mamba_radix_cache_1(self): req9_token_ids = [1, 2, 3, 4, 5, 6, 7] req9 = make_dummy_req() result = tree.match_prefix( - MatchPrefixParams(key=RadixKey(req9_token_ids), req=req9, cow_mamba=True) + MatchPrefixParams( + key=RadixKey(array("q", req9_token_ids)), req=req9, cow_mamba=True + ) ) kv_indices, last_node = result.device_indices, result.last_device_node assert req9.mamba_pool_idx is not None @@ -315,7 +327,7 @@ def test_mamba_radix_cache_kv_events(self): stored_hashes = [] req1 = make_dummy_req() - key1 = RadixKey([1, 2, 3]) + key1 = RadixKey(array("q", [1, 2, 3])) tree.insert( InsertParams( key=key1, @@ -330,7 +342,7 @@ def test_mamba_radix_cache_kv_events(self): stored_hashes.extend(e.block_hashes[0] for e in stored_events) req2 = make_dummy_req() - key2 = RadixKey([1, 2, 3, 4, 5]) + key2 = RadixKey(array("q", [1, 2, 3, 4, 5])) tree.insert( InsertParams( key=key2, @@ -367,7 +379,7 @@ def test_mamba_radix_cache_kv_events_split_hash(self): tree.take_events() # Clear the reset event. req1 = make_dummy_req() - key1 = RadixKey([1, 2, 3, 4]) + key1 = RadixKey(array("q", [1, 2, 3, 4])) tree.insert( InsertParams( key=key1, @@ -382,7 +394,7 @@ def test_mamba_radix_cache_kv_events_split_hash(self): split_parent_hash = first_insert_events[1].block_hashes[0] req2 = make_dummy_req() - key2 = RadixKey([1, 2, 5, 6]) + key2 = RadixKey(array("q", [1, 2, 5, 6])) tree.insert( InsertParams( key=key2, @@ -394,14 +406,17 @@ def test_mamba_radix_cache_kv_events_split_hash(self): e for e in tree.take_events() if isinstance(e, BlockStored) ] self.assertEqual(len(second_insert_events), 2) - self.assertEqual(second_insert_events[0].token_ids, [5]) + self.assertEqual(list(second_insert_events[0].token_ids), [5]) self.assertEqual(second_insert_events[0].parent_block_hash, split_parent_hash) def _setup_tree_and_allocator(self, enable_kv_cache_events=False): """Helper to create a MambaRadixCache with allocator for testing.""" - set_global_server_args_for_scheduler( - ServerArgs(model_path="dummy", page_size=1) - ) + server_args = ServerArgs(model_path="dummy", page_size=1) + # MambaRadixCache reads mamba_cache_chunk_size, whose property otherwise + # loads the HF config for self.model_path — impossible for the dummy model. + # Mirror the property's default for a dummy HF config: FLA_CHUNK_SIZE. + server_args._mamba_cache_chunk_size = FLA_CHUNK_SIZE + set_global_server_args_for_scheduler(server_args) size = 128 dtype = torch.bfloat16 head_num = 2 @@ -478,7 +493,7 @@ def make_dummy_req(): req = Req( rid=0, origin_input_text="", - origin_input_ids=[], + origin_input_ids=array("q"), sampling_params=sampling_params, ) req_to_token_pool.alloc([req]) @@ -492,9 +507,9 @@ def test_hi_mamba_tombstone_cleanup_respects_host_ref(self): parent = TreeNode() deleted = TreeNode() - root.key = RadixKey([]) - parent.key = RadixKey([1]) - deleted.key = RadixKey([2]) + root.key = RadixKey(array("q", [])) + parent.key = RadixKey(array("q", [1])) + deleted.key = RadixKey(array("q", [2])) parent.parent = root deleted.parent = parent parent.value = torch.tensor([1], dtype=torch.int64) @@ -668,7 +683,7 @@ def test_insert_prev_prefix_len(self): # Step 1: Insert [1,2,3] to create first node req1 = make_dummy_req() - key1 = RadixKey([1, 2, 3]) + key1 = RadixKey(array("q", [1, 2, 3])) tree.insert( InsertParams( key=key1, @@ -681,7 +696,7 @@ def test_insert_prev_prefix_len(self): # Step 2: Insert [1,2,3,4,5,6,7] with prev_prefix_len=0 (free all matched) # Creates tree: [1,2,3] -> [4,5,6,7] req2 = make_dummy_req() - key2 = RadixKey([1, 2, 3, 4, 5, 6, 7]) + key2 = RadixKey(array("q", [1, 2, 3, 4, 5, 6, 7])) result = tree.insert( InsertParams( key=key2, @@ -699,7 +714,7 @@ def test_insert_prev_prefix_len(self): # Matched prefix = 7 (across two nodes: [1,2,3] len=3, [4,5,6,7] len=4) # Protected [0..1], freed [2..6] = 5 slots, new [7] = 1 slot stored req3 = make_dummy_req() - key3 = RadixKey([1, 2, 3, 4, 5, 6, 7, 8]) + key3 = RadixKey(array("q", [1, 2, 3, 4, 5, 6, 7, 8])) result = tree.insert( InsertParams( key=key3, @@ -716,7 +731,7 @@ def test_insert_prev_prefix_len(self): # Step 4: Insert [1,2,3,4,5,6,7,8,9] with prev_prefix_len=8 (covers all matched) # Matched prefix = 8, prev_prefix_len=8 => nothing freed req4 = make_dummy_req() - key4 = RadixKey([1, 2, 3, 4, 5, 6, 7, 8, 9]) + key4 = RadixKey(array("q", [1, 2, 3, 4, 5, 6, 7, 8, 9])) result = tree.insert( InsertParams( key=key4, diff --git a/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py b/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py index 44d4e0419be1..c6f2c767691e 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py +++ b/test/registered/unit/mem_cache/test_radix_cache_slru_accuracy.py @@ -1,4 +1,5 @@ import unittest +from array import array import torch @@ -63,7 +64,7 @@ def test_eviction_mechanism(self): """Test that SLRU eviction mechanism works correctly""" # Insert one key-value three times (high frequency access) - frequent_key = RadixKey([1, 2]) # High hit rate, should be retained + frequent_key = RadixKey(array("q", [1, 2])) # High hit rate, should be retained frequent_val = torch.tensor([10, 20], dtype=torch.int64) # Insert the frequent key multiple times to increase its hit count @@ -71,7 +72,9 @@ def test_eviction_mechanism(self): self.cache.insert(InsertParams(key=frequent_key, value=frequent_val)) # Insert first low-frequency key-value pair that should be evicted - first_low_freq_key = RadixKey([5, 6]) # Low hit rate, should be evicted + first_low_freq_key = RadixKey( + array("q", [5, 6]) + ) # Low hit rate, should be evicted first_low_freq_val = torch.tensor([50, 60], dtype=torch.int64) self.cache.insert( @@ -81,14 +84,14 @@ def test_eviction_mechanism(self): # Insert other key-values once each (low frequency access) - fill up the cache other_keys = [] for i in range(4): # Reduce the number to fit in our smaller cache - key = RadixKey([i + 10]) # Unique keys for low-frequency items + key = RadixKey(array("q", [i + 10])) # Unique keys for low-frequency items val = torch.tensor([i + 100], dtype=torch.int64) self.cache.insert(InsertParams(key=key, value=val)) other_keys.append(key) # Now insert more items to trigger evictions for i in range(6, 10): # Add more items to definitely exceed capacity - key = RadixKey([i * 2]) # Different pattern to avoid conflicts + key = RadixKey(array("q", [i * 2])) # Different pattern to avoid conflicts val = torch.tensor([i * 200], dtype=torch.int64) self.cache.insert(InsertParams(key=key, value=val)) diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index a445b9b3da40..263670fb74d5 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -28,6 +28,7 @@ import time import unittest import unittest.mock +from array import array import torch @@ -38,6 +39,7 @@ InsertParams, MatchPrefixParams, ) +from sglang.srt.mem_cache.mamba_radix_cache import TreeNode as MambaTreeNode from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode # Test constants @@ -50,30 +52,30 @@ class TestRadixKey(unittest.TestCase): def test_init_basic(self): """Test basic initialization of RadixKey.""" token_ids = [1, 2, 3, 4] - key = RadixKey(token_ids) - self.assertEqual(key.token_ids, token_ids) + key = RadixKey(array("q", token_ids)) + self.assertEqual(list(key.token_ids), token_ids) self.assertIsNone(key.extra_key) def test_init_with_extra_key(self): """Test initialization with extra_key.""" token_ids = [1, 2, 3] extra_key = "test_key" - key = RadixKey(token_ids, extra_key) - self.assertEqual(key.token_ids, token_ids) + key = RadixKey(array("q", token_ids), extra_key) + self.assertEqual(list(key.token_ids), token_ids) self.assertEqual(key.extra_key, extra_key) def test_len(self): """Test __len__ method.""" - key = RadixKey([1, 2, 3]) + key = RadixKey(array("q", [1, 2, 3])) self.assertEqual(len(key), 3) - empty_key = RadixKey([]) + empty_key = RadixKey(array("q", [])) self.assertEqual(len(empty_key), 0) def test_iter(self): """Test __iter__ method.""" token_ids = [1, 2, 3, 4] - key = RadixKey(token_ids) + key = RadixKey(array("q", token_ids)) self.assertEqual(list(key), token_ids) def test_len_and_iter(self): @@ -86,7 +88,7 @@ def test_len_and_iter(self): for tokens, expected in test_cases: with self.subTest(tokens=tokens): - key = RadixKey(tokens) + key = RadixKey(array("q", tokens)) self.assertEqual(len(key), expected) self.assertEqual(list(key), tokens) @@ -100,34 +102,34 @@ def test_getitem_int(self): for tokens, index, expected in test_cases: with self.subTest(tokens=tokens, index=index): - key = RadixKey(tokens) + key = RadixKey(array("q", tokens)) result = key[index] self.assertIsInstance(result, RadixKey) - self.assertEqual(result.token_ids, expected) + self.assertEqual(list(result.token_ids), expected) def test_getitem_slice(self): """Test __getitem__ with slice and edge cases.""" - key = RadixKey([1, 2, 3, 4, 5], "extra") + key = RadixKey(array("q", [1, 2, 3, 4, 5]), "extra") # Basic slice sliced = key[1:4] self.assertIsInstance(sliced, RadixKey) - self.assertEqual(sliced.token_ids, [2, 3, 4]) + self.assertEqual(list(sliced.token_ids), [2, 3, 4]) self.assertEqual(sliced.extra_key, "extra") # Edge cases - self.assertEqual(key[2:2].token_ids, []) # Empty slice - self.assertEqual(key[:].token_ids, [1, 2, 3, 4, 5]) # Full slice + self.assertEqual(list(key[2:2].token_ids), []) # Empty slice + self.assertEqual(list(key[:].token_ids), [1, 2, 3, 4, 5]) # Full slice def test_getitem_invalid_index(self): """Test __getitem__ with invalid indices.""" - key = RadixKey([1, 2, 3]) + key = RadixKey(array("q", [1, 2, 3])) with self.assertRaises(IndexError): _ = key[10] # Out of bounds def test_repr(self): """Test __repr__ method.""" - key = RadixKey([1, 2, 3], "test") + key = RadixKey(array("q", [1, 2, 3]), "test") repr_str = repr(key) self.assertIn("RadixKey", repr_str) self.assertIn("extra_key='test'", repr_str) @@ -136,7 +138,7 @@ def test_repr(self): def test_repr_long_token_ids(self): """Test __repr__ with long token_ids.""" long_tokens = list(range(15)) - key = RadixKey(long_tokens) + key = RadixKey(array("q", long_tokens)) repr_str = repr(key) self.assertIn("...", repr_str) # Should be truncated @@ -225,6 +227,38 @@ def test_get_last_hash_value(self): node.hash_value = ["hash1", "hash2", "hash3"] self.assertEqual(node.get_last_hash_value(), "hash3") + def test_get_prefix_hash_values_not_shared_across_calls(self): + """Regression guard for cached mutable prefix hash lists.""" + for node_cls in (TreeNode, MambaTreeNode): + with self.subTest(node_cls=node_cls.__module__): + root = node_cls() + n1 = node_cls() + n1.parent = root + n1.hash_value = ["h1"] + n2 = node_cls() + n2.parent = n1 + n2.hash_value = ["h2"] + n3 = node_cls() + n3.parent = n2 + n3.hash_value = ["h3"] + + first = n3.get_prefix_hash_values(n2) + self.assertEqual(first, ["h1", "h2"]) + + # Downstream storage code extends prefix_keys in place while + # processing pages. A cached list must not be observable by a + # later call. + first += ["h3"] + + second = n3.get_prefix_hash_values(n2) + self.assertEqual(second, ["h1", "h2"]) + self.assertIsNot(second, first) + + n4 = node_cls() + n4.parent = n3 + n4.hash_value = ["h4"] + self.assertEqual(n4.get_prefix_hash_values(n3), ["h1", "h2", "h3"]) + def test_lt_comparison(self): """Test less than comparison based on last_access_time.""" node1 = TreeNode() @@ -274,7 +308,7 @@ def test_reset(self): # Insert some data cache.insert( InsertParams( - key=RadixKey([1, 2, 3]), + key=RadixKey(array("q", [1, 2, 3])), value=torch.tensor([10, 20, 30], dtype=torch.int64), ) ) @@ -292,7 +326,7 @@ def test_insert_and_match_basic(self): with self.subTest(disable_cache=disable_cache): cache = RadixCache.create_simulated(disable=disable_cache) - key = RadixKey([1, 2, 3]) + key = RadixKey(array("q", [1, 2, 3])) value = torch.tensor([10, 20, 30], dtype=torch.int64) result = cache.insert(InsertParams(key=key, value=value)) prefix_len = result.prefix_len @@ -307,12 +341,16 @@ def test_insert_and_match_basic(self): self.assertEqual(cache.evictable_size(), 3) # Test match_prefix - result = cache.match_prefix(MatchPrefixParams(key=RadixKey([1, 2, 3]))) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]))) + ) self.assertEqual(len(result.device_indices), 3) torch.testing.assert_close(result.device_indices, value) # Test partial match - result = cache.match_prefix(MatchPrefixParams(key=RadixKey([1, 2]))) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", [1, 2]))) + ) self.assertEqual(len(result.device_indices), 2) torch.testing.assert_close( result.device_indices, torch.tensor([10, 20], dtype=torch.int64) @@ -322,7 +360,7 @@ def test_insert_with_none_value(self): """Test insert with None value (should use token_ids as list).""" cache = RadixCache.create_simulated() - key = RadixKey([1, 2, 3]) + key = RadixKey(array("q", [1, 2, 3])) result = cache.insert(InsertParams(key=key, value=None)) prefix_len = result.prefix_len @@ -338,7 +376,7 @@ def test_total_size(self): cache.insert( InsertParams( - key=RadixKey([1, 2, 3]), + key=RadixKey(array("q", [1, 2, 3])), value=torch.tensor([10, 20, 30], dtype=torch.int64), ) ) @@ -346,7 +384,8 @@ def test_total_size(self): cache.insert( InsertParams( - key=RadixKey([4, 5]), value=torch.tensor([40, 50], dtype=torch.int64) + key=RadixKey(array("q", [4, 5])), + value=torch.tensor([40, 50], dtype=torch.int64), ) ) self.assertEqual(cache.total_size(), 5) @@ -366,7 +405,9 @@ def test_kv_cache_events(self): ) # Insert data - cache.insert(InsertParams(key=RadixKey([1, 2, 3, 4, 5]), value=None)) + cache.insert( + InsertParams(key=RadixKey(array("q", [1, 2, 3, 4, 5])), value=None) + ) # Take events events = cache.take_events() @@ -395,7 +436,7 @@ def test_kv_cache_events_with_eviction(self): # Insert and then evict data cache.insert( InsertParams( - key=RadixKey([1, 2, 3]), + key=RadixKey(array("q", [1, 2, 3])), value=torch.tensor([10, 20, 30], dtype=torch.int64), ) ) @@ -427,29 +468,35 @@ def test_extra_key_isolation(self): # Insert same token sequence with different extra keys cache.insert( InsertParams( - key=RadixKey([1, 2, 3], "key1"), + key=RadixKey(array("q", [1, 2, 3]), "key1"), value=torch.tensor([10, 20, 30], dtype=torch.int64), ) ) cache.insert( InsertParams( - key=RadixKey([1, 2, 3], "key2"), + key=RadixKey(array("q", [1, 2, 3]), "key2"), value=torch.tensor([40, 50, 60], dtype=torch.int64), ) ) cache.insert( InsertParams( - key=RadixKey([1, 2, 3], None), + key=RadixKey(array("q", [1, 2, 3]), None), value=torch.tensor([70, 80, 90], dtype=torch.int64), ) ) # Keys with different extra_key should not match each other - result1 = cache.match_prefix(MatchPrefixParams(key=RadixKey([1, 2, 3], "key1"))) - result2 = cache.match_prefix(MatchPrefixParams(key=RadixKey([1, 2, 3], "key2"))) - result3 = cache.match_prefix(MatchPrefixParams(key=RadixKey([1, 2, 3], None))) + result1 = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]), "key1")) + ) + result2 = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]), "key2")) + ) + result3 = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]), None)) + ) result4 = cache.match_prefix( - MatchPrefixParams(key=RadixKey([1, 2, 3], "nonexistent")) + MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]), "nonexistent")) ) # Each should match only its own data @@ -478,13 +525,15 @@ def test_lock_ref_operations(self): # Insert sequence cache.insert( InsertParams( - key=RadixKey([1, 2, 3]), + key=RadixKey(array("q", [1, 2, 3])), value=torch.tensor([10, 20, 30], dtype=torch.int64), ) ) # Get node - result = cache.match_prefix(MatchPrefixParams(key=RadixKey([1, 2, 3]))) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", [1, 2, 3]))) + ) node = result.last_device_node initial_evictable = cache.evictable_size() @@ -510,12 +559,14 @@ def test_evict_functionality(self): # Insert sequences cache.insert( InsertParams( - key=RadixKey([1, 2]), value=torch.tensor([10, 20], dtype=torch.int64) + key=RadixKey(array("q", [1, 2])), + value=torch.tensor([10, 20], dtype=torch.int64), ) ) cache.insert( InsertParams( - key=RadixKey([3, 4]), value=torch.tensor([30, 40], dtype=torch.int64) + key=RadixKey(array("q", [3, 4])), + value=torch.tensor([30, 40], dtype=torch.int64), ) ) @@ -547,7 +598,7 @@ def test_page_alignment_boundary(self): cache = RadixCache.create_simulated(page_size=page_size) tokens = list(range(sequence_length)) - key = RadixKey(tokens) + key = RadixKey(array("q", tokens)) cache.insert( InsertParams( key=key, @@ -555,7 +606,9 @@ def test_page_alignment_boundary(self): ) ) - result = cache.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens))) + ) self.assertGreater(len(result.device_indices), 0) # Match length should be page-aligned @@ -568,7 +621,7 @@ def test_pretty_print_basic(self): cache.insert( InsertParams( - key=RadixKey([1, 2, 3]), + key=RadixKey(array("q", [1, 2, 3])), value=torch.tensor([10, 20, 30], dtype=torch.int64), ) ) @@ -585,12 +638,14 @@ def test_all_values_flatten(self): cache.insert( InsertParams( - key=RadixKey([1, 2]), value=torch.tensor([10, 20], dtype=torch.int64) + key=RadixKey(array("q", [1, 2])), + value=torch.tensor([10, 20], dtype=torch.int64), ) ) cache.insert( InsertParams( - key=RadixKey([3, 4]), value=torch.tensor([30, 40], dtype=torch.int64) + key=RadixKey(array("q", [3, 4])), + value=torch.tensor([30, 40], dtype=torch.int64), ) ) @@ -609,12 +664,12 @@ def test_advanced_prefix_match_with_node_splits(self): # Insert a long sequence that will be split later. seq1 = [1, 2, 3, 4, 5, 6, 7, 8] val1 = torch.tensor([x * 10 for x in seq1], dtype=torch.int64) - cache.insert(InsertParams(key=RadixKey(seq1), value=val1)) + cache.insert(InsertParams(key=RadixKey(array("q", seq1)), value=val1)) # Insert a diverging branch to create an internal node on the path. seq2 = [1, 2, 9, 10] val2 = torch.tensor([x * 10 for x in seq2], dtype=torch.int64) - cache.insert(InsertParams(key=RadixKey(seq2), value=val2)) + cache.insert(InsertParams(key=RadixKey(array("q", seq2)), value=val2)) print(cache.pretty_print()) baseline_total = cache.total_size() @@ -624,24 +679,30 @@ def test_advanced_prefix_match_with_node_splits(self): # Match that causes a split inside an existing node: # take first 4 tokens of seq1, then diverge. query1 = [1, 2, 3, 4, 999, 1000] - result1 = cache.match_prefix(MatchPrefixParams(key=RadixKey(query1))) + result1 = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", query1))) + ) torch.testing.assert_close(result1.device_indices, val1[:4]) # No data change after structural split during matching. self.assertEqual(cache.total_size(), baseline_total) # Full match of the long sequence still returns the full indices. - result_full = cache.match_prefix(MatchPrefixParams(key=RadixKey(seq1))) + result_full = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq1))) + ) torch.testing.assert_close(result_full.device_indices, val1) # Another split deeper on the path (after matching 6 tokens, then diverge). query2 = [1, 2, 3, 4, 5, 6, 777, 888] - result2 = cache.match_prefix(MatchPrefixParams(key=RadixKey(query2))) + result2 = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", query2))) + ) torch.testing.assert_close(result2.device_indices, val1[:6]) self.assertEqual(cache.total_size(), baseline_total) # Matching the short diverging branch should return exactly its indices. result_branch = cache.match_prefix( - MatchPrefixParams(key=RadixKey(seq2)) + MatchPrefixParams(key=RadixKey(array("q", seq2))) ) torch.testing.assert_close(result_branch.device_indices, val2) @@ -653,7 +714,9 @@ def test_hash_value_storage(self): ) # Insert a sequence - cache.insert(InsertParams(key=RadixKey([1, 2, 3, 4, 5, 6, 7, 8]), value=None)) + cache.insert( + InsertParams(key=RadixKey(array("q", [1, 2, 3, 4, 5, 6, 7, 8])), value=None) + ) # Trigger event emission to compute hash_value lazily cache.take_events() @@ -679,7 +742,9 @@ def test_hash_value_repeating_tokens(self): ) # Insert a sequence with repeating token pattern: [1,2,3,4, 1,2,3,4] - cache.insert(InsertParams(key=RadixKey([1, 2, 3, 4, 1, 2, 3, 4]), value=None)) + cache.insert( + InsertParams(key=RadixKey(array("q", [1, 2, 3, 4, 1, 2, 3, 4])), value=None) + ) events = cache.take_events() block_stored_events = [e for e in events if isinstance(e, BlockStored)] @@ -713,11 +778,11 @@ def test_hash_value_split(self): ) # Insert a sequence that will cause a split - cache.insert(InsertParams(key=RadixKey([1, 2, 3, 4]), value=None)) + cache.insert(InsertParams(key=RadixKey(array("q", [1, 2, 3, 4])), value=None)) cache.take_events() # Clear events and compute hash_value for first node # Insert a diverging sequence that will cause a split at page boundary - cache.insert(InsertParams(key=RadixKey([1, 2, 5, 6]), value=None)) + cache.insert(InsertParams(key=RadixKey(array("q", [1, 2, 5, 6])), value=None)) cache.take_events() # Trigger event emission to compute hash_value # Find the split node @@ -754,7 +819,7 @@ def test_memory_allocated(self): cache: RadixCache = RadixCache.create_simulated() for key, value in zip(keys, values): - cache.insert(InsertParams(key=RadixKey(key), value=value)) + cache.insert(InsertParams(key=RadixKey(array("q", key)), value=value)) del values diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index c91481bba352..77d2be9629e9 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -11,6 +11,7 @@ import unittest import unittest.mock +from array import array import torch @@ -27,8 +28,8 @@ class _StubReq: def __init__(self, token_ids): - self.origin_input_ids = list(token_ids) - self.output_ids = [] + self.origin_input_ids = array("q", token_ids) + self.output_ids = array("q") self.extra_key = None self.prefix_indices = None self.last_node = None @@ -42,9 +43,9 @@ def __init__(self, token_ids): class TestZeroMatchResult(unittest.TestCase): def test_zero_replaces_indices_and_nodes(self): tree = RadixCache.create_simulated() - tree.insert(InsertParams(key=RadixKey(token_ids=[1, 2, 3, 4, 5]))) + tree.insert(InsertParams(key=RadixKey(token_ids=array("q", [1, 2, 3, 4, 5])))) match = tree.match_prefix( - MatchPrefixParams(key=RadixKey(token_ids=[1, 2, 3, 9])) + MatchPrefixParams(key=RadixKey(token_ids=array("q", [1, 2, 3, 9]))) ) self.assertGreater(len(match.device_indices), 0) zeroed = zero_match_result(tree, match) @@ -76,7 +77,9 @@ class TestMatchPrefixForReqForceMiss(unittest.TestCase): def test_force_miss_zeros_req_prefix(self): tree = RadixCache.create_simulated() tree.insert( - InsertParams(key=RadixKey(token_ids=[10, 11, 12, 13, 14, 15, 16, 17])) + InsertParams( + key=RadixKey(token_ids=array("q", [10, 11, 12, 13, 14, 15, 16, 17])) + ) ) # Sanity: without the flag, the same lookup hits. diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index 302d61f11cd0..026ee98c47cd 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -178,12 +178,15 @@ def test_cpp_radix_cache_when_env_flag_set(self): # we inject a stand-in module rather than letting patch() trigger # the real import. fake_module = MagicMock() - with patch( - "sglang.srt.mem_cache.registry.envs.SGLANG_EXPERIMENTAL_CPP_RADIX_TREE.get", - return_value=True, - ), patch.dict( - "sys.modules", - {"sglang.srt.mem_cache.radix_cache_cpp": fake_module}, + with ( + patch( + "sglang.srt.mem_cache.registry.envs.SGLANG_EXPERIMENTAL_CPP_RADIX_TREE.get", + return_value=True, + ), + patch.dict( + "sys.modules", + {"sglang.srt.mem_cache.radix_cache_cpp": fake_module}, + ), ): result = default_radix_cache_factory(ctx) fake_module.RadixCacheCpp.assert_called_once_with( @@ -196,15 +199,18 @@ def test_unified_radix_cache_when_env_flag_set(self): # Shim both factory imports — each transitively loads sgl_kernel. fake_components = MagicMock() fake_radix = MagicMock() - with patch( - "sglang.srt.mem_cache.registry.envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.get", - return_value=True, - ), patch.dict( - "sys.modules", - { - "sglang.srt.mem_cache.unified_cache_components": fake_components, - "sglang.srt.mem_cache.unified_radix_cache": fake_radix, - }, + with ( + patch( + "sglang.srt.mem_cache.registry.envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.get", + return_value=True, + ), + patch.dict( + "sys.modules", + { + "sglang.srt.mem_cache.unified_cache_components": fake_components, + "sglang.srt.mem_cache.unified_radix_cache": fake_radix, + }, + ), ): result = default_radix_cache_factory(ctx) fake_radix.UnifiedRadixCache.assert_called_once_with(ctx.params) diff --git a/test/registered/unit/mem_cache/test_swa_lock_release_lifecycle.py b/test/registered/unit/mem_cache/test_swa_lock_release_lifecycle.py index 173f338dc62c..180e54c5f2f5 100644 --- a/test/registered/unit/mem_cache/test_swa_lock_release_lifecycle.py +++ b/test/registered/unit/mem_cache/test_swa_lock_release_lifecycle.py @@ -12,6 +12,7 @@ """ import unittest +from array import array import torch @@ -110,6 +111,7 @@ def _swa_alloc(allocator, need_size): def _insert_chain(tree, allocator, token_ids): + token_ids = array("q", token_ids) indices = _swa_alloc(allocator, len(token_ids)) assert indices is not None tree.insert(InsertParams(key=RadixKey(token_ids), value=indices)) diff --git a/test/registered/unit/mem_cache/test_swa_unittest.py b/test/registered/unit/mem_cache/test_swa_unittest.py index 96e0bd1db41f..aac705945b71 100644 --- a/test/registered/unit/mem_cache/test_swa_unittest.py +++ b/test/registered/unit/mem_cache/test_swa_unittest.py @@ -1,4 +1,5 @@ import unittest +from array import array import torch @@ -113,12 +114,12 @@ def _swa_alloc(allocator, need_size): def _insert(tree, allocator, token_ids): indices = _swa_alloc(allocator, len(token_ids)) assert indices is not None - tree.insert(InsertParams(key=RadixKey(token_ids), value=indices)) + tree.insert(InsertParams(key=RadixKey(array("q", token_ids)), value=indices)) def _insert_chain(tree, allocator, token_ids): _insert(tree, allocator, token_ids) - match = tree.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) + match = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", token_ids)))) return match.last_device_node @@ -193,7 +194,7 @@ def test_swa_radix_cache_kv_events_split_hash(self): e for e in tree.take_events() if isinstance(e, BlockStored) ] self.assertEqual(len(second_insert_events), 2) - self.assertEqual(second_insert_events[0].token_ids, [5]) + self.assertEqual(list(second_insert_events[0].token_ids), [5]) self.assertEqual(second_insert_events[0].parent_block_hash, split_parent_hash) def test_swa_memory_pool(self): @@ -313,7 +314,7 @@ def test_swa_radix_cache_1(self): print( f"req1: inserting, req1_token_ids: {req1_token_ids}, req1_kv_indices: {req1_kv_indices}" ) - key = RadixKey(req1_token_ids) + key = RadixKey(array("q", req1_token_ids)) result = tree.insert(InsertParams(key=key, value=req1_kv_indices[: len(key)])) prefix_len = result.prefix_len print( @@ -324,7 +325,7 @@ def test_swa_radix_cache_1(self): print( f"req2: inserting, req2_token_ids: {req2_token_ids}, req2_kv_indices: {req2_kv_indices}" ) - key = RadixKey(req2_token_ids) + key = RadixKey(array("q", req2_token_ids)) result = tree.insert(InsertParams(key=key, value=req2_kv_indices[: len(key)])) prefix_len = result.prefix_len print( @@ -335,7 +336,7 @@ def test_swa_radix_cache_1(self): print( f"req3: inserting, req3_token_ids: {req3_token_ids}, req3_kv_indices: {req3_kv_indices}" ) - key = RadixKey(req3_token_ids) + key = RadixKey(array("q", req3_token_ids)) result = tree.insert(InsertParams(key=key, value=req3_kv_indices[: len(key)])) prefix_len = result.prefix_len print( @@ -346,7 +347,7 @@ def test_swa_radix_cache_1(self): print( f"req4: inserting, req4_token_ids: {req4_token_ids}, req4_kv_indices: {req4_kv_indices}" ) - key = RadixKey(req4_token_ids) + key = RadixKey(array("q", req4_token_ids)) result = tree.insert(InsertParams(key=key, value=req4_kv_indices[: len(key)])) prefix_len = result.prefix_len print( @@ -376,7 +377,9 @@ def test_swa_radix_cache_1(self): tree.pretty_print() req5_token_ids = [1, 2, 3, 4, 5] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req5_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req5_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req5: token_ids: {req5_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -384,7 +387,9 @@ def test_swa_radix_cache_1(self): self.assertEqual(len(kv_indices), 0) req6_token_ids = [1, 2, 3, 4, 5, 60, 70] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req6_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req6_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req6: token_ids: {req6_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -468,7 +473,7 @@ def test_swa_radix_cache_eagle(self): print( f"req1: inserting, req1_token_ids: {req1_token_ids}, req1_kv_indices: {req1_kv_indices}" ) - key = RadixKey(req1_token_ids) + key = RadixKey(array("q", req1_token_ids)) result = tree.insert(InsertParams(key=key, value=req1_kv_indices[: len(key)])) prefix_len = result.prefix_len self.assertEqual(prefix_len, 0) @@ -480,7 +485,7 @@ def test_swa_radix_cache_eagle(self): print( f"req2: inserting, req2_token_ids: {req2_token_ids}, req2_kv_indices: {req2_kv_indices}" ) - key = RadixKey(req2_token_ids) + key = RadixKey(array("q", req2_token_ids)) result = tree.insert(InsertParams(key=key, value=req2_kv_indices[: len(key)])) prefix_len = result.prefix_len self.assertEqual(prefix_len, 2) @@ -492,7 +497,7 @@ def test_swa_radix_cache_eagle(self): print( f"req3: inserting, req3_token_ids: {req3_token_ids}, req3_kv_indices: {req3_kv_indices}" ) - key = RadixKey(req3_token_ids) + key = RadixKey(array("q", req3_token_ids)) result = tree.insert(InsertParams(key=key, value=req3_kv_indices[: len(key)])) prefix_len = result.prefix_len self.assertEqual(prefix_len, 0) @@ -504,7 +509,7 @@ def test_swa_radix_cache_eagle(self): print( f"req4: inserting, req4_token_ids: {req4_token_ids}, req4_kv_indices: {req4_kv_indices}" ) - key = RadixKey(req4_token_ids) + key = RadixKey(array("q", req4_token_ids)) result = tree.insert(InsertParams(key=key, value=req4_kv_indices[: len(key)])) prefix_len = result.prefix_len self.assertEqual(prefix_len, 4) @@ -553,7 +558,9 @@ def test_swa_radix_cache_eagle(self): tree.pretty_print() req5_token_ids = [1, 2, 3, 4, 5] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req5_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req5_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req5: token_ids: {req5_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -561,7 +568,9 @@ def test_swa_radix_cache_eagle(self): self.assertEqual(len(kv_indices), 0) # no swa prefix matched req6_token_ids = [1, 2, 3, 4, 5, 60, 70] - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(req6_token_ids))) + result = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", req6_token_ids))) + ) kv_indices, last_node = result.device_indices, result.last_device_node print( f"req6: token_ids: {req6_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" @@ -578,8 +587,8 @@ def test_swa_cache_finished_req_eagle_uses_cache_protected_len_and_bigram_key(se # Case 1: is_insert=True should pass bigram key and use cache_protected_len. req = _DummyReq() req.req_pool_idx = 0 - req.origin_input_ids = [1, 2, 3, 4, 5, 6] - req.output_ids = [] + req.origin_input_ids = array("q", [1, 2, 3, 4, 5, 6]) + req.output_ids = array("q") req._kv_committed_len = len(req.origin_input_ids) kv_indices = allocator.alloc(req._kv_committed_len) req_to_token_pool.write( @@ -613,8 +622,8 @@ def wrapped_insert(params): # even when len(prefix_indices) is intentionally larger. req2 = _DummyReq() req2.req_pool_idx = 1 - req2.origin_input_ids = [11, 12, 13, 14, 15, 16] - req2.output_ids = [] + req2.origin_input_ids = array("q", [11, 12, 13, 14, 15, 16]) + req2.output_ids = array("q") req2._kv_committed_len = len(req2.origin_input_ids) kv_indices2 = allocator.alloc(req2._kv_committed_len) req_to_token_pool.write( @@ -730,7 +739,9 @@ def test_match_prefix_returns_full_chain_after_split(self): with envs.SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT.override(True): inserted_leaf = _insert_chain(tree, allocator, token_ids) self.assertEqual(len(inserted_leaf.value), 4) - match = tree.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) + match = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", token_ids))) + ) self.assertEqual(match.device_indices.shape[0], 12) self.assertIs(match.last_device_node, inserted_leaf) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py index bef9eec69601..7516bfa4857a 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py @@ -12,6 +12,7 @@ import statistics import time import unittest +from array import array from contextlib import contextmanager from dataclasses import dataclass from typing import Callable @@ -20,6 +21,7 @@ from sglang.srt.configs.mamba_utils import Mamba2CacheParams, Mamba2StateShape from sglang.srt.environ import envs +from sglang.srt.layers.attention.fla.chunk_delta_h import CHUNK_SIZE as FLA_CHUNK_SIZE from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, @@ -335,7 +337,7 @@ def _insert_seq(env, seq): if env.has_mamba: req = env.make_req() mamba_val = req.mamba_pool_idx.unsqueeze(0) - key = RadixKey(seq) + key = RadixKey(array("q", seq)) env.tree.insert(InsertParams(key=key, value=v[: len(key)], mamba_value=mamba_val)) return True @@ -357,7 +359,7 @@ def _fill_no_evict(env): if env.has_mamba: req = env.make_req() mamba_val = req.mamba_pool_idx.unsqueeze(0) - key = RadixKey(seq) + key = RadixKey(array("q", seq)) env.tree.insert( InsertParams(key=key, value=v[: len(key)], mamba_value=mamba_val) ) @@ -505,7 +507,7 @@ def bench_match_prefix( queries.append([rng.randint(1, 32000)] * rng.randint(50, 300)) def verify_fn(q): - k = RadixKey(q) + k = RadixKey(array("q", q)) r1 = env.tree.match_prefix(MatchPrefixParams(key=k)) r2 = env.tree.match_prefix(MatchPrefixParams(key=k)) assert len(r1.device_indices) == len(r2.device_indices), "match not idempotent" @@ -514,7 +516,7 @@ def verify_fn(q): return bench_api( "match_prefix", lambda: queries, - lambda q: env.tree.match_prefix(MatchPrefixParams(key=RadixKey(q))), + lambda q: env.tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", q)))), min(len(queries) - warmup, num_seqs), env.avg_tokens, warmup, @@ -566,7 +568,7 @@ def bench_lock_unlock( nodes = [] for seq in env.seqs[: num_seqs // 2]: - r = env.tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + r = env.tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) if r.last_device_node != env.tree.root_node: nodes.append(r.last_device_node) if not nodes: @@ -613,7 +615,7 @@ def bench_cache_finished( # Pre-build Req objects with token IDs filled into req_to_token req_items: list = [] for seq in env.seqs: - key = RadixKey(seq) + key = RadixKey(array("q", seq)) mr = env.tree.match_prefix(MatchPrefixParams(key=key)) matched_len = len(mr.device_indices) node = mr.last_device_node @@ -635,9 +637,9 @@ def bench_cache_finished( kv_indices = mr.device_indices req = env.make_req() - req.origin_input_ids = list(seq) - req.output_ids = [] - req.fill_ids = list(seq) + req.origin_input_ids = array("q", seq) + req.output_ids = array("q") + req.fill_ids = array("q", seq) req.last_node = node req.cache_protected_len = matched_len req.kv_committed_len = len(seq) @@ -690,9 +692,11 @@ def run_all_benchmarks( if benchmarks is None or "all" in benchmarks: benchmarks = list(ALL_BENCHMARKS.keys()) - set_global_server_args_for_scheduler( - ServerArgs(model_path="dummy", page_size=page_size) - ) + server_args = ServerArgs(model_path="dummy", page_size=page_size) + # MambaRadixCache reads mamba_cache_chunk_size, whose property otherwise + # loads the HF config for self.model_path — impossible for the dummy model. + server_args._mamba_cache_chunk_size = max(FLA_CHUNK_SIZE, page_size) + set_global_server_args_for_scheduler(server_args) impl_name = (tree_cls or UnifiedRadixCache).__name__ results = [] @@ -779,9 +783,11 @@ class _BenchSuite: @classmethod def setUpClass(cls): - set_global_server_args_for_scheduler( - ServerArgs(model_path="dummy", page_size=cls.bench_cfg["page_size"]) - ) + page_size = cls.bench_cfg["page_size"] + server_args = ServerArgs(model_path="dummy", page_size=page_size) + # See run_all_benchmarks for why _mamba_cache_chunk_size is preset. + server_args._mamba_cache_chunk_size = max(FLA_CHUNK_SIZE, page_size) + set_global_server_args_for_scheduler(server_args) def _run(self, bench_fn): cfg = self.bench_cfg diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index be313de31c96..a5e262a91590 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -1,6 +1,7 @@ """Unit tests for UnifiedRadixCache""" import unittest +from array import array from dataclasses import dataclass from typing import Optional from unittest import mock @@ -9,6 +10,7 @@ from sglang.srt.configs.mamba_utils import Mamba2CacheParams, Mamba2StateShape from sglang.srt.environ import envs +from sglang.srt.layers.attention.fla.chunk_delta_h import CHUNK_SIZE as FLA_CHUNK_SIZE from sglang.srt.managers.schedule_batch import Req from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( @@ -113,9 +115,12 @@ def label(self) -> str: def build_fixture(cfg: CacheConfig): """Create (tree, allocator, req_to_token_pool) from a CacheConfig.""" - set_global_server_args_for_scheduler( - ServerArgs(model_path="dummy", page_size=cfg.page_size) - ) + server_args = ServerArgs(model_path="dummy", page_size=cfg.page_size) + # MambaRadixCache reads mamba_cache_chunk_size, whose property otherwise + # loads the HF config for self.model_path — impossible for the dummy model. + # Mirror the property's default for a dummy HF config: FLA_CHUNK_SIZE. + server_args._mamba_cache_chunk_size = max(FLA_CHUNK_SIZE, cfg.page_size) + set_global_server_args_for_scheduler(server_args) device = get_device() mamba2_cache_params = None @@ -272,7 +277,7 @@ def _alloc(self, allocator, need_size): def _insert(self, tree, allocator, req_to_token_pool, tokens): """Insert tokens, attaching mamba data when the config has mamba.""" - key = RadixKey(tokens) + key = RadixKey(array("q", tokens)) value = self._alloc(allocator, len(tokens)) params = InsertParams(key=key, value=value[: len(key)]) if self.cfg.has_mamba: @@ -290,15 +295,17 @@ def test_insert_and_match_basic(self): result = self._insert(tree, allocator, req_to_token_pool, seq_b) self.assertEqual(result.prefix_len, len(seq_a)) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_b))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_b)))) self.assertEqual(len(m.device_indices), len(seq_b)) m = tree.match_prefix( - MatchPrefixParams(key=RadixKey(seq_a + self._make_seq(9000, 1))) + MatchPrefixParams(key=RadixKey(array("q", seq_a + self._make_seq(9000, 1)))) ) self.assertEqual(len(m.device_indices), len(seq_a)) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(self._make_seq(5000, 2)))) + m = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", self._make_seq(5000, 2)))) + ) self.assertEqual(len(m.device_indices), 0) tree.sanity_check() @@ -317,11 +324,11 @@ def test_shared_prefix_split(self): self.assertEqual(result_b.prefix_len, len(base)) for seq in (branch_a, branch_b): - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) self.assertEqual(len(m.device_indices), len(seq)) m = tree.match_prefix( - MatchPrefixParams(key=RadixKey(base + self._make_seq(999, 1))) + MatchPrefixParams(key=RadixKey(array("q", base + self._make_seq(999, 1)))) ) self.assertEqual(len(m.device_indices), len(base)) tree.sanity_check() @@ -350,13 +357,13 @@ def test_evict_respects_lock_ref(self): self._insert(tree, allocator, req_to_token_pool, seq_a) self._insert(tree, allocator, req_to_token_pool, seq_b) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_a))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) lock_result = tree.inc_lock_ref(m.last_device_node) result = tree.evict(EvictParams(num_tokens=len(seq_a) + len(seq_b))) self.assertGreaterEqual(result.num_tokens_evicted, len(seq_b)) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_a))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) self.assertEqual(len(m.device_indices), len(seq_a)) # Unlock -> should now be evictable @@ -395,7 +402,7 @@ def test_evict_until_empty(self): if self.cfg.has_mamba: self.assertEqual(tree.mamba_evictable_size(), 0) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seqs[0]))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seqs[0])))) self.assertEqual(len(m.device_indices), 0) tree.sanity_check() @@ -413,7 +420,7 @@ def test_prev_prefix_len(self): self.assertEqual(allocator.available_size(), initial_avail - len(seq_1p)) # Step 2: insert 2 pages with prev_prefix_len=0 → frees overlap of 1 page - key_2p = RadixKey(seq_2p) + key_2p = RadixKey(array("q", seq_2p)) value_2p = self._alloc(allocator, len(seq_2p)) params = InsertParams( key=key_2p, @@ -432,7 +439,7 @@ def test_prev_prefix_len(self): # Step 3: insert 3 pages with prev_prefix_len=len(seq_2p) → nothing freed avail_before = allocator.available_size() - key_3p = RadixKey(seq_3p) + key_3p = RadixKey(array("q", seq_3p)) value_3p = self._alloc(allocator, len(seq_3p)) params = InsertParams( key=key_3p, @@ -461,11 +468,11 @@ def test_node_split_at_boundary(self): self.assertEqual(result.prefix_len, len(base)) for seq in (fork_a, fork_b): - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) self.assertEqual(len(m.device_indices), len(seq)) m = tree.match_prefix( - MatchPrefixParams(key=RadixKey(base + self._make_seq(999, 1))) + MatchPrefixParams(key=RadixKey(array("q", base + self._make_seq(999, 1)))) ) self.assertEqual(len(m.device_indices), len(base)) tree.sanity_check() @@ -477,8 +484,8 @@ def test_cache_finished_req_insert(self): req = self._make_req(req_to_token_pool) input_ids = self._make_seq(1, 3) output_ids = self._make_seq(2000, 1) - req.origin_input_ids = input_ids - req.output_ids = output_ids + req.origin_input_ids = array("q", input_ids) + req.output_ids = array("q", output_ids) kv_len = len(input_ids) + len(output_ids) kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -487,7 +494,7 @@ def test_cache_finished_req_insert(self): req.cache_protected_len = 0 req.swa_uuid_for_lock = None req.extra_key = None - req.fill_ids = input_ids + output_ids + req.fill_ids = array("q", input_ids + output_ids) if self.cfg.has_mamba: req.mamba_last_track_seqlen = kv_len @@ -495,7 +502,9 @@ def test_cache_finished_req_insert(self): all_ids = input_ids + output_ids aligned_len = (len(all_ids) // ps) * ps - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(all_ids[:aligned_len]))) + m = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", all_ids[:aligned_len]))) + ) self.assertEqual(len(m.device_indices), aligned_len) tree.sanity_check() @@ -506,9 +515,9 @@ def test_cache_finished_req_strips_thinking(self): req = self._make_req(req_to_token_pool) prompt_ids = self._make_seq(1, 3) output_ids = self._make_seq(2000, 7) - req.origin_input_ids = prompt_ids - req.output_ids = output_ids - req.fill_ids = prompt_ids + output_ids + req.origin_input_ids = array("q", prompt_ids) + req.output_ids = array("q", output_ids) + req.fill_ids = array("q", prompt_ids + output_ids) kv_len = len(req.fill_ids) kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -538,7 +547,9 @@ def test_cache_finished_req_strips_thinking(self): prompt_aligned = (len(prompt_ids) // ps) * ps # Thinking+answer must not be reachable past the prompt. - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(prompt_ids + output_ids))) + m = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", prompt_ids + output_ids))) + ) self.assertEqual(len(m.device_indices), prompt_aligned) # Only prompt-aligned pages remain owned by the tree. self.assertEqual( @@ -550,8 +561,8 @@ def test_cache_finished_req_no_insert(self): tree, allocator, req_to_token_pool = build_fixture(self.cfg) req = self._make_req(req_to_token_pool) tokens = self._make_seq(1, 2) - req.origin_input_ids = tokens - req.output_ids = [] + req.origin_input_ids = array("q", tokens) + req.output_ids = array("q") kv_len = len(tokens) kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -560,13 +571,13 @@ def test_cache_finished_req_no_insert(self): req.cache_protected_len = 0 req.swa_uuid_for_lock = None req.extra_key = None - req.fill_ids = tokens + req.fill_ids = array("q", tokens) avail_before = allocator.available_size() tree.cache_finished_req(req, is_insert=False) self.assertEqual(allocator.available_size(), avail_before + kv_len) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) self.assertEqual(len(m.device_indices), 0) tree.sanity_check() @@ -575,9 +586,9 @@ def test_cache_unfinished_req(self): req = self._make_req(req_to_token_pool) tokens = self._make_seq(1, 3) - req.origin_input_ids = tokens - req.output_ids = [] - req.fill_ids = tokens[:] + req.origin_input_ids = array("q", tokens) + req.output_ids = array("q") + req.fill_ids = array("q", tokens) kv_len = len(tokens) kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -628,11 +639,11 @@ def test_multi_branch_tree(self): for suffix_start in [100, 200, 300]: seq = base + self._make_seq(suffix_start, 2) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) self.assertEqual(len(m.device_indices), len(seq)) m = tree.match_prefix( - MatchPrefixParams(key=RadixKey(base + self._make_seq(999, 1))) + MatchPrefixParams(key=RadixKey(array("q", base + self._make_seq(999, 1)))) ) self.assertEqual(len(m.device_indices), len(base)) tree.sanity_check() @@ -641,7 +652,7 @@ def test_paged_child_key_is_tuple(self): if self.cfg.page_size == 1: self.skipTest("page_size > 1 only") tree, _, _ = build_fixture(self.cfg) - key = RadixKey(self._make_seq(1, 1)) + key = RadixKey(array("q", self._make_seq(1, 1))) child_key = key.child_key(tree.page_size) self.assertIsInstance(child_key, tuple) @@ -656,11 +667,13 @@ def test_paged_match_truncates_unaligned_key(self): # Tree truncates unaligned tail internally, so it matches the seq prefix. unaligned = seq + list(range(9000, 9000 + ps - 1)) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(unaligned))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", unaligned)))) self.assertEqual(len(m.device_indices), len(seq)) # Below-page-size key aligns to 0 -> no match. - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq[: ps - 1]))) + m = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq[: ps - 1]))) + ) self.assertEqual(len(m.device_indices), 0) tree.sanity_check() @@ -679,12 +692,12 @@ def test_paged_page_boundary_mismatch(self): # Mismatch in second page → only first page matches bad_page2 = seq[:ps] + [9999] * ps - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(bad_page2))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", bad_page2)))) self.assertEqual(len(m.device_indices), ps) # Mismatch in first page → 0 match bad_page1 = [9999] + seq[1:] - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(bad_page1))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", bad_page1)))) self.assertEqual(len(m.device_indices), 0) tree.sanity_check() @@ -699,8 +712,8 @@ def test_paged_cache_finished_unaligned_tail_freed(self): tail_extra = ps // 2 input_ids = self._make_seq(1, 1) + list(range(8000, 8000 + tail_extra)) req = self._make_req(req_to_token_pool) - req.origin_input_ids = input_ids - req.output_ids = [] + req.origin_input_ids = array("q", input_ids) + req.output_ids = array("q") kv_len = len(input_ids) kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -709,7 +722,7 @@ def test_paged_cache_finished_unaligned_tail_freed(self): req.cache_protected_len = 0 req.swa_uuid_for_lock = None req.extra_key = None - req.fill_ids = input_ids + req.fill_ids = array("q", input_ids) if self.cfg.has_mamba: req.mamba_last_track_seqlen = kv_len @@ -718,7 +731,7 @@ def test_paged_cache_finished_unaligned_tail_freed(self): self.assertEqual(allocator.available_size(), avail_before + tail_extra) aligned = input_ids[: (len(input_ids) // ps) * ps] - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(aligned))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", aligned)))) self.assertEqual(len(m.device_indices), len(aligned)) tree.sanity_check() @@ -749,7 +762,7 @@ def test_mamba_evict_breaks_match(self): tree.evict(EvictParams(num_tokens=0, mamba_num=10)) self.assertEqual(tree.mamba_evictable_size(), 0) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_long))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_long)))) self.assertEqual(len(m.device_indices), 0) tree.sanity_check() @@ -788,7 +801,7 @@ def test_mamba_cow_on_match(self): req2 = self._make_req(req_to_token_pool) m = tree.match_prefix( - MatchPrefixParams(key=RadixKey(seq), cow_mamba=True, req=req2) + MatchPrefixParams(key=RadixKey(array("q", seq)), cow_mamba=True, req=req2) ) self.assertEqual(len(m.device_indices), len(seq)) self.assertIsNotNone(req2.mamba_pool_idx) @@ -809,7 +822,7 @@ def test_swa_insert_and_match(self): seq = self._make_seq(1, 3) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) self.assertEqual(len(m.device_indices), len(seq)) tree.sanity_check() @@ -866,13 +879,13 @@ def test_swa_lock_protects_from_eviction(self): self._insert(tree, allocator, req_to_token_pool, seq_a) self._insert(tree, allocator, req_to_token_pool, seq_b) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_a))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) lock_result = tree.inc_lock_ref(m.last_device_node) result = tree.evict(EvictParams(num_tokens=len(seq_a) + len(seq_b))) self.assertGreaterEqual(result.num_tokens_evicted, len(seq_b)) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_a))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) self.assertEqual(len(m.device_indices), len(seq_a)) tree.dec_lock_ref( @@ -886,8 +899,8 @@ def test_tombstone_cleanup_respects_locked_parent(self): parent = UnifiedTreeNode(self.cfg.components) deleted = UnifiedTreeNode(self.cfg.components) - parent.key = RadixKey(self._make_seq(1, 1)) - deleted.key = RadixKey(self._make_seq(1000, 1)) + parent.key = RadixKey(array("q", self._make_seq(1, 1))) + deleted.key = RadixKey(array("q", self._make_seq(1000, 1))) parent.parent = tree.root_node deleted.parent = parent parent.component_data[ComponentType.FULL].value = torch.arange( @@ -924,15 +937,15 @@ def count_nodes(node): node_count_before = count_nodes(tree.root_node) self.assertEqual(node_count_before, 2) - tree._match_prefix_helper(RadixKey([1, 2])) + tree._match_prefix_helper(RadixKey(array("q", [1, 2]))) ( value, best_match_node, best_match_device_node, best_value_len, - ) = tree._match_prefix_helper(RadixKey([1, 2, 3, 4])) + ) = tree._match_prefix_helper(RadixKey(array("q", [1, 2, 3, 4]))) self.assertEqual(best_value_len, 2) - self.assertEqual(best_match_node.key.token_ids, [3, 4]) + self.assertEqual(list(best_match_node.key.token_ids), [3, 4]) self.assertIs(best_match_device_node, best_match_node) node_count_after_regular = count_nodes(tree.root_node) self.assertEqual(node_count_after_regular, node_count_before + 2) @@ -942,9 +955,9 @@ def count_nodes(node): best_match_node, best_match_device_node, best_value_len, - ) = tree._match_prefix_helper_readonly(RadixKey([1, 2, 3])) + ) = tree._match_prefix_helper_readonly(RadixKey(array("q", [1, 2, 3]))) self.assertEqual(best_value_len, 1) - self.assertEqual(best_match_node.key.token_ids, [1, 2]) + self.assertEqual(list(best_match_node.key.token_ids), [1, 2]) self.assertIs(best_match_device_node, best_match_node) node_count_after_readonly = count_nodes(tree.root_node) self.assertEqual(node_count_after_readonly, node_count_after_regular) @@ -971,7 +984,7 @@ def test_aux_evict_full_locked_leaf_tombstones_aux_only(self): seq = self._make_seq(1, 2) self._insert(tree, allocator, req_to_token_pool, seq) - match = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + match = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = match.last_device_node full_cd = node.component_data[ComponentType.FULL] aux_cd = node.component_data[aux] @@ -1040,7 +1053,7 @@ def test_evict_cascade_parent_becomes_d_leaf(self): self._insert(tree, allocator, req_to_token_pool, leaf) # Lock the base node to prevent it from being evicted - m_base = tree.match_prefix(MatchPrefixParams(key=RadixKey(base))) + m_base = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", base)))) lock_result = tree.inc_lock_ref(m_base.last_device_node) # Evict the leaf — parent (base) should become D-leaf after unlock @@ -1089,14 +1102,14 @@ def test_evict_respects_lru_order(self): self._insert(tree, allocator, req_to_token_pool, seq_new) # Touch seq_new to make it MRU - tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_new))) + tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_new)))) # Evict just enough for one sequence tree.evict(EvictParams(num_tokens=len(seq_old))) # seq_old should be gone (LRU), seq_new should remain - m_old = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_old))) - m_new = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_new))) + m_old = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_old)))) + m_new = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_new)))) self.assertEqual(len(m_old.device_indices), 0) self.assertEqual(len(m_new.device_indices), len(seq_new)) tree.sanity_check() @@ -1136,13 +1149,13 @@ def test_evict_shared_prefix_keeps_common_path(self): self._insert(tree, allocator, req_to_token_pool, branch_b) # Lock branch_b - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(branch_b))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", branch_b)))) lr = tree.inc_lock_ref(m.last_device_node) # Evict — branch_a should go, base + branch_b stay tree.evict(EvictParams(num_tokens=len(branch_a))) - m_b = tree.match_prefix(MatchPrefixParams(key=RadixKey(branch_b))) + m_b = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", branch_b)))) self.assertEqual(len(m_b.device_indices), len(branch_b)) tree.dec_lock_ref( @@ -1173,7 +1186,7 @@ def test_evict_locked_subtree_skipped(self): self._insert(tree, allocator, req_to_token_pool, seq_b) # Lock seq_a - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_a))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) lr = tree.inc_lock_ref(m.last_device_node) # Try to evict everything @@ -1181,7 +1194,7 @@ def test_evict_locked_subtree_skipped(self): result = tree.evict(EvictParams(num_tokens=total)) # seq_a should still be matchable (protected) - m2 = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_a))) + m2 = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_a)))) self.assertEqual(len(m2.device_indices), len(seq_a)) tree.dec_lock_ref( @@ -1220,7 +1233,7 @@ def test_evict_reinsert_after_full_eviction(self): # Re-insert seq_b = self._make_seq(500, 2) self._insert(tree, allocator, req_to_token_pool, seq_b) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq_b))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq_b)))) self.assertEqual(len(m.device_indices), len(seq_b)) tree.sanity_check() @@ -1247,7 +1260,7 @@ def test_evict_d_leaf_set_consistency(self): self._insert(tree, allocator, req_to_token_pool, s) # Lock some, evict some, unlock - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seqs[0]))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seqs[0])))) lr = tree.inc_lock_ref(m.last_device_node) tree.evict(EvictParams(num_tokens=len(seqs[1]))) @@ -1327,6 +1340,8 @@ def mamba_host_pool_wrapper(*args, **kwargs): hicache_io_backend="direct", hicache_write_policy=write_policy, ) + # See build_fixture for why _mamba_cache_chunk_size is preset. + server_args._mamba_cache_chunk_size = max(FLA_CHUNK_SIZE, self.cfg.page_size) set_global_server_args_for_scheduler(server_args) tree.init_hicache(server_args, tree.cache_init_params) tree.write_through_threshold = 1 << 30 @@ -1409,7 +1424,7 @@ def test_hicache_node_states(self): self._insert(tree, allocator, req_to_token_pool, seq) # Find the leaf node - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = m.last_device_node self.assertIsNot(node, tree.root_node) @@ -1435,7 +1450,7 @@ def test_hicache_evict_to_host(self): seq = self._make_seq(1, 2) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = m.last_device_node self._backup_node(tree, node) @@ -1469,7 +1484,7 @@ def test_hicache_match_through_evicted_node(self): self._backup_tree(tree) # Lock leaf so only base can be evicted - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(leaf))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", leaf)))) lr = tree.inc_lock_ref(m.last_device_node) # Evict base (inner node won't be evicted while child is locked) @@ -1479,7 +1494,7 @@ def test_hicache_match_through_evicted_node(self): m.last_device_node, DecLockRefParams(swa_uuid_for_lock=getattr(lr, "swa_uuid_for_lock", None)), ) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(leaf))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", leaf)))) self.assertGreaterEqual(len(m.device_indices), len(base)) tree.sanity_check() @@ -1495,7 +1510,7 @@ def test_hicache_partial_match_splits_evicted_backed_up_node(self): query = expected_prefix + self._make_seq(9000, 1) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = m.last_device_node self._backup_node(tree, node) @@ -1503,7 +1518,7 @@ def test_hicache_partial_match_splits_evicted_backed_up_node(self): self.assertTrue(node.evicted) self.assertTrue(node.backuped) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(query))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", query)))) self.assertEqual(len(m.device_indices), 0) self.assertIs(m.last_device_node, tree.root_node) @@ -1512,8 +1527,8 @@ def test_hicache_partial_match_splits_evicted_backed_up_node(self): self.assertIsNot(split_parent, tree.root_node) self.assertTrue(split_parent.evicted) self.assertTrue(split_parent.backuped) - self.assertEqual(split_parent.key.token_ids, expected_prefix) - self.assertEqual(node.key.token_ids, expected_suffix) + self.assertEqual(list(split_parent.key.token_ids), expected_prefix) + self.assertEqual(list(node.key.token_ids), expected_suffix) if self.cfg.has_mamba: self.assertEqual(m.host_hit_length, 0) @@ -1536,7 +1551,7 @@ def test_hicache_d_leaf_h_leaf_mutual_exclusion(self): self._insert(tree, allocator, req_to_token_pool, s) for i in range(2): - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seqs[i]))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seqs[i])))) self._backup_node(tree, m.last_device_node) # Evict one backed-up node @@ -1555,7 +1570,7 @@ def test_hicache_host_leaf_eviction(self): seq = self._make_seq(1, 2) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = m.last_device_node self._backup_node(tree, node) @@ -1580,7 +1595,7 @@ def test_hicache_load_back_restores_data(self): base = self._make_seq(1, 2) self._insert(tree, allocator, req_to_token_pool, base) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(base))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", base)))) node = m.last_device_node original_device_indices = m.device_indices.clone() self._fill_full_kv(allocator, original_device_indices, marker=3) @@ -1648,6 +1663,41 @@ def test_hicache_backup_continuity(self): ) tree.sanity_check() + def test_hicache_write_through_offloads_swa_split_leaf(self): + """A SWA boundary-split leaf should offload normally under write-through.""" + if not self.cfg.has_swa: + self.skipTest("requires SWA") + if self.cfg.has_mamba: + self.skipTest("SWA-only path keeps the split setup simple") + + ps = self.cfg.page_size + tree, allocator, _ = build_fixture(self.cfg) + self._init_hicache(tree) + tree.write_through_threshold = 1 + + seq = self._make_seq(1, 2) + value = self._alloc(allocator, len(seq)) + result = tree.insert( + InsertParams( + key=RadixKey(seq), + value=value, + swa_evicted_seqlen=ps, + ) + ) + self.assertEqual(result.prefix_len, 0) + + self.assertEqual(len(tree.root_node.children), 1) + split_parent = next(iter(tree.root_node.children.values())) + self.assertEqual(len(split_parent.children), 1) + split_leaf = next(iter(split_parent.children.values())) + + tree.writing_check(write_back=True) + tree.evict(EvictParams(num_tokens=len(seq))) + self.assertTrue(split_leaf.evicted) + self.assertTrue(split_leaf.backuped) + self.assertIn(split_leaf, tree.evictable_host_leaves) + tree.sanity_check() + def test_hicache_evict_to_host_updates_aux_lru(self): """Aux components (MAMBA / SWA) move from device LRU to host LRU on D->H eviction.""" aux_types = [ @@ -1662,7 +1712,7 @@ def test_hicache_evict_to_host_updates_aux_lru(self): seq = self._make_seq(1, 2) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = m.last_device_node for aux in aux_types: @@ -1688,7 +1738,7 @@ def _build_chain_pages(self, tree, allocator, req_to_token_pool, num_pages): for i in range(num_pages): seq = seq + self._make_seq(1000 * (i + 1), 1) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) chain: list = [] cur = m.last_device_node while cur is not tree.root_node: @@ -1738,7 +1788,7 @@ def test_match_prefix_best_and_device_node_without_hicache(self): seq = self._make_seq(1, (min_tokens + ps - 1) // ps) self._insert(tree, allocator, req_to_token_pool, seq) - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + result = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) self.assertEqual(len(result.device_indices), len(seq)) self.assertIs(result.best_match_node, result.last_device_node) @@ -1760,7 +1810,7 @@ def test_hicache_mamba_host_best_match_keeps_device_anchor(self): tree.evict(EvictParams(num_tokens=len(leaf.key))) self.assertTrue(leaf.evicted) - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) + result = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) self.assertIs(result.best_match_node, leaf) self.assertIs(result.last_device_node, parent) @@ -1782,7 +1832,7 @@ def test_hicache_swa_host_best_match_keeps_device_anchor(self): tree.evict(EvictParams(num_tokens=len(leaf.key))) self.assertTrue(leaf.evicted) - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) + result = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) self.assertIs(result.best_match_node, leaf) self.assertIs(result.last_device_node, parent) @@ -1797,12 +1847,14 @@ def test_mamba_branching_seqlen_disabled_under_hicache(self): tokens = self._make_seq(1, chunk_size + 1) self._insert(tree, allocator, req_to_token_pool, tokens) leaf = tree.match_prefix( - MatchPrefixParams(key=RadixKey(tokens)) + MatchPrefixParams(key=RadixKey(array("q", tokens))) ).last_device_node mamba_cd = leaf.component_data[ComponentType.MAMBA] mamba_cd.value = None - no_hicache = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) + no_hicache = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens))) + ) self.assertIs(no_hicache.best_match_node, tree.root_node) self.assertIs(no_hicache.last_device_node, tree.root_node) self.assertEqual(no_hicache.mamba_branching_seqlen, chunk_size) @@ -1810,11 +1862,13 @@ def test_mamba_branching_seqlen_disabled_under_hicache(self): tree_h, allocator_h, req_to_token_pool_h = self._build_hicache_fixture() self._insert(tree_h, allocator_h, req_to_token_pool_h, tokens) leaf_h = tree_h.match_prefix( - MatchPrefixParams(key=RadixKey(tokens)) + MatchPrefixParams(key=RadixKey(array("q", tokens))) ).last_device_node self._backup_node(tree_h, leaf_h) tree_h.evict(EvictParams(num_tokens=len(tokens))) - with_hicache = tree_h.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) + with_hicache = tree_h.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens))) + ) self.assertIs(with_hicache.best_match_node, leaf_h) self.assertIs(with_hicache.last_device_node, tree_h.root_node) self.assertIsNone(with_hicache.mamba_branching_seqlen) @@ -1834,7 +1888,9 @@ def test_scheduler_hicache_full_mamba_init_load_back_appends_new_indices(self): self.assertTrue(leaf.evicted) req = self._make_req(req_to_token_pool) - match = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens), req=req)) + match = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) + ) req.prefix_indices = match.device_indices req.last_node = match.last_device_node req.best_match_node = match.best_match_node @@ -1876,7 +1932,9 @@ def test_scheduler_hicache_aux_only_load_back_appends_full_device_indices(self): self._set_aux_host_tombstone(tree, leaf, aux) req = self._make_req(req_to_token_pool) - match = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens), req=req)) + match = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) + ) req.prefix_indices = match.device_indices req.last_node = match.last_device_node req.best_match_node = match.best_match_node @@ -1915,7 +1973,9 @@ def test_scheduler_hicache_load_back_fallback_keeps_old_anchor(self): tree.evict(EvictParams(num_tokens=len(leaf.key))) req = self._make_req(req_to_token_pool) - match = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens), req=req)) + match = tree.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", tokens)), req=req) + ) req.prefix_indices = match.device_indices req.last_node = match.last_device_node req.best_match_node = match.best_match_node @@ -1986,7 +2046,7 @@ def test_hicache_swa_host_independent_of_full(self): tree, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 2) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = m.last_device_node self._simulate_backup(tree, node) @@ -2077,7 +2137,9 @@ def test_hicache_swa_finalize_match_result(self): ) result = swa_comp.finalize_match_result( result=result, - params=MatchPrefixParams(key=RadixKey(self._make_seq(1, 1))), + params=MatchPrefixParams( + key=RadixKey(array("q", self._make_seq(1, 1))) + ), value_chunks=[], best_value_len=0, ) @@ -2212,13 +2274,6 @@ def _swa_anchor_setup(self): tokens = self._swa_anchor_chain_tokens(len(chain)) return tree, chain, n, y, x, tokens - def test_hicache_swa_match_prefix_picks_best_match_node_above_last_host(self): - tree, _, n, y, x, tokens = self._swa_anchor_setup() - result = tree.match_prefix(MatchPrefixParams(key=RadixKey(tokens))) - self.assertIs(result.best_match_node, x) - self.assertIs(result.last_device_node, n.parent) - self.assertIs(result.last_host_node, y) - def test_hicache_swa_load_back_anchored_on_best_match_node(self): tree, _, _, y, x, _ = self._swa_anchor_setup() ps = self.cfg.page_size @@ -2247,7 +2302,7 @@ def test_hicache_swa_finalize_anchored_on_best_match_node(self): ) result = swa_comp.finalize_match_result( result=base, - params=MatchPrefixParams(key=RadixKey(self._make_seq(1, 1))), + params=MatchPrefixParams(key=RadixKey(array("q", self._make_seq(1, 1)))), value_chunks=[], best_value_len=0, ) @@ -2365,7 +2420,7 @@ def test_hicache_mamba_temp_lock_does_not_release_restored_tombstone(self): tree, allocator, req_to_token_pool = build_fixture(self.cfg) seq = self._make_seq(1, 2) self._insert(tree, allocator, req_to_token_pool, seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq)))) node = m.last_device_node cd = node.component_data[ComponentType.MAMBA] old_mamba = cd.value @@ -2414,7 +2469,7 @@ def test_hicache_mixed_backup_evict_insert(self): tree.sanity_check() for i in range(3): - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(seqs[i]))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seqs[i])))) self._backup_node(tree, m.last_device_node) # Evict to free some tokens @@ -2443,7 +2498,7 @@ def test_hicache_write_back_leaf_backup(self): self._insert(tree, allocator, req_to_token_pool, base) self._insert(tree, allocator, req_to_token_pool, leaf_seq) - m = tree.match_prefix(MatchPrefixParams(key=RadixKey(leaf_seq))) + m = tree.match_prefix(MatchPrefixParams(key=RadixKey(array("q", leaf_seq)))) leaf = m.last_device_node parent = leaf.parent self.assertIsNot(parent, tree.root_node) diff --git a/test/registered/unit/mem_cache/test_unified_radix_hicache_dispatch.py b/test/registered/unit/mem_cache/test_unified_radix_hicache_dispatch.py new file mode 100644 index 000000000000..3ce1ed789553 --- /dev/null +++ b/test/registered/unit/mem_cache/test_unified_radix_hicache_dispatch.py @@ -0,0 +1,183 @@ +import unittest +from unittest.mock import MagicMock + +from sglang.srt.mem_cache.hicache_storage import PoolName, SidecarPoolSpec +from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler +from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( + _STRATEGIES, + StackBuildResult, + StackStrategy, + _apply_stack_result, + _DeepSeekV4Strategy, + _DsaStrategy, + _MambaStrategy, + _PlainKvStrategy, + _select_strategy, + _SwaStrategy, + register_stack_strategy, +) +from sglang.srt.mem_cache.unified_cache_components import ComponentType +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +def _mock_kvcache(cls): + return MagicMock(spec=cls) + + +FULL = ComponentType.FULL +SWA = ComponentType.SWA +MAMBA = ComponentType.MAMBA + + +class TestUnifiedRadixHiCacheDispatch(unittest.TestCase): + def test_strategy_registry_ordering(self): + order = [type(s) for s in _STRATEGIES] + # DeepSeekV4 inherits from SWAKVPool, so it must resolve before _SwaStrategy. + self.assertLess(order.index(_DeepSeekV4Strategy), order.index(_SwaStrategy)) + self.assertEqual(order[-1], _PlainKvStrategy) + + def test_deepseek_v4_full_swa(self): + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DeepSeekV4TokenToKVPool, + ) + + kvcache = _mock_kvcache(DeepSeekV4TokenToKVPool) + strategy = _select_strategy(kvcache, {FULL, SWA}) + self.assertIsInstance(strategy, _DeepSeekV4Strategy) + + def test_mamba(self): + from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool + + kvcache = _mock_kvcache(HybridLinearKVPool) + strategy = _select_strategy(kvcache, {FULL, MAMBA}) + self.assertIsInstance(strategy, _MambaStrategy) + + def test_swa(self): + from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool + + kvcache = _mock_kvcache(SWAKVPool) + strategy = _select_strategy(kvcache, {FULL, SWA}) + self.assertIsInstance(strategy, _SwaStrategy) + + def test_dsa(self): + from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool + + kvcache = _mock_kvcache(DSATokenToKVPool) + strategy = _select_strategy(kvcache, {FULL}) + self.assertIsInstance(strategy, _DsaStrategy) + + def test_plain_kv_fallback(self): + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + + kvcache = _mock_kvcache(MHATokenToKVPool) + strategy = _select_strategy(kvcache, {FULL}) + self.assertIsInstance(strategy, _PlainKvStrategy) + + def test_mla_routes_to_plain(self): + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + + kvcache = _mock_kvcache(MLATokenToKVPool) + strategy = _select_strategy(kvcache, {FULL}) + self.assertIsInstance(strategy, _PlainKvStrategy) + + def test_unknown_combo_raises(self): + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DeepSeekV4TokenToKVPool, + ) + from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool + + for cls in (SWAKVPool, DeepSeekV4TokenToKVPool): + kvcache = _mock_kvcache(cls) + with self.assertRaises(AssertionError) as cm: + _select_strategy(kvcache, {FULL}) + self.assertIn("No matching HiCache strategy", str(cm.exception)) + + def test_register_custom_strategy_takes_precedence(self): + class _CustomStrategy(StackStrategy): + def matches(self, kvcache, components): + return components == {FULL} + + def build(self, **_): + raise NotImplementedError + + custom = _CustomStrategy() + original = list(hybrid_pool_assembler._STRATEGIES) + try: + register_stack_strategy(custom) + from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool + + kvcache = _mock_kvcache(MHATokenToKVPool) + self.assertIs(_select_strategy(kvcache, {FULL}), custom) + finally: + hybrid_pool_assembler._STRATEGIES[:] = original + + +class TestApplyStackResult(unittest.TestCase): + @staticmethod + def _fake_cache(component_types): + cache = MagicMock() + cache.components = {ct: MagicMock() for ct in component_types} + return cache + + def test_wires_components_sidecars_and_counters(self): + full_host, swa_host, mamba_host = MagicMock(), MagicMock(), MagicMock() + cache = self._fake_cache([FULL, SWA, MAMBA]) + kvcache = MagicMock() + params = MagicMock() + controller = MagicMock() + sidecar = SidecarPoolSpec( + pool_name=PoolName.INDEXER, indices_from_pool=PoolName.KV + ) + result = StackBuildResult( + host_pool_group=MagicMock(), + cache_controller=controller, + component_host_pools={FULL: full_host, SWA: swa_host, MAMBA: mamba_host}, + sidecars=[sidecar], + register_req_to_token_counter=True, + transfer_layer_num=8, + pools_desc="KV + SWA + MAMBA", + ) + + _apply_stack_result(cache, kvcache, params, result) + + self.assertIs(cache.host_pool_group, result.host_pool_group) + self.assertIs(cache.cache_controller, controller) + self.assertIs(cache.full_kv_pool_host, full_host) + self.assertIs(cache.swa_kv_pool_host, swa_host) + self.assertIs(cache.mamba_pool_host, mamba_host) + self.assertIs(cache.components[FULL]._full_kv_pool_host, full_host) + self.assertIs(cache.components[SWA]._swa_kv_pool_host, swa_host) + self.assertIs(cache.components[MAMBA]._mamba_pool_host, mamba_host) + cache.register_sidecar_pool.assert_called_once_with(sidecar) + kvcache.register_layer_transfer_counter.assert_called_once_with( + controller.layer_done_counter + ) + params.req_to_token_pool.register_layer_transfer_counter.assert_called_once_with( + controller.layer_done_counter + ) + + def test_skips_req_to_token_counter_when_flag_false(self): + cache = self._fake_cache([FULL]) + kvcache = MagicMock() + params = MagicMock() + result = StackBuildResult( + host_pool_group=MagicMock(), + cache_controller=MagicMock(), + component_host_pools={FULL: MagicMock()}, + sidecars=[], + register_req_to_token_counter=False, + transfer_layer_num=1, + pools_desc="KV", + ) + + _apply_stack_result(cache, kvcache, params, result) + + kvcache.register_layer_transfer_counter.assert_called_once() + params.req_to_token_pool.register_layer_transfer_counter.assert_not_called() + cache.register_sidecar_pool.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/observability/test_stat_loggers_di.py b/test/registered/unit/observability/test_stat_loggers_di.py new file mode 100644 index 000000000000..271a4cbed09c --- /dev/null +++ b/test/registered/unit/observability/test_stat_loggers_di.py @@ -0,0 +1,201 @@ +"""Unit tests for class-level DI on the five *MetricsCollector classes via +ServerArgs.stat_loggers — no server, no model loading.""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +import unittest + +import prometheus_client + +from sglang.srt.observability.metrics_collector import ( + STAT_LOGGER_ROLE_EXPERT_DISPATCH, + STAT_LOGGER_ROLE_RADIX_CACHE, + STAT_LOGGER_ROLE_SCHEDULER, + STAT_LOGGER_ROLE_STORAGE, + STAT_LOGGER_ROLE_TOKENIZER, + ExpertDispatchCollector, + RadixCacheMetricsCollector, + SchedulerMetricsCollector, + StorageMetricsCollector, + TokenizerMetricsCollector, + resolve_collector_class, +) + + +class _StubArgs: + """Minimal ServerArgs stand-in. Avoids triggering heavy ServerArgs import chain.""" + + def __init__(self, stat_loggers=None): + self.stat_loggers = stat_loggers + + +# ── _gauge_cls / _counter_cls / _histogram_cls / _summary_cls override surface ── + + +class TestCollectorClassAttrs(unittest.TestCase): + """All five collectors expose four DI hook class attrs, all defaulting to None + so the existing prometheus_client backend is used unchanged.""" + + def test_scheduler_collector_attrs_default_none(self): + self.assertIsNone(SchedulerMetricsCollector._counter_cls) + self.assertIsNone(SchedulerMetricsCollector._gauge_cls) + self.assertIsNone(SchedulerMetricsCollector._histogram_cls) + self.assertIsNone(SchedulerMetricsCollector._summary_cls) + + def test_tokenizer_collector_attrs_default_none(self): + self.assertIsNone(TokenizerMetricsCollector._counter_cls) + self.assertIsNone(TokenizerMetricsCollector._histogram_cls) + + def test_storage_collector_attrs_default_none(self): + self.assertIsNone(StorageMetricsCollector._counter_cls) + self.assertIsNone(StorageMetricsCollector._histogram_cls) + + def test_expert_dispatch_collector_attrs_default_none(self): + self.assertIsNone(ExpertDispatchCollector._histogram_cls) + + def test_radix_cache_collector_attrs_default_none(self): + self.assertIsNone(RadixCacheMetricsCollector._counter_cls) + self.assertIsNone(RadixCacheMetricsCollector._histogram_cls) + + +# ── resolve_collector_class helper ── + + +class TestResolveCollectorClass(unittest.TestCase): + def test_returns_default_when_server_args_none(self): + cls = resolve_collector_class(None, "scheduler", SchedulerMetricsCollector) + self.assertIs(cls, SchedulerMetricsCollector) + + def test_returns_default_when_stat_loggers_none(self): + cls = resolve_collector_class( + _StubArgs(stat_loggers=None), "scheduler", SchedulerMetricsCollector + ) + self.assertIs(cls, SchedulerMetricsCollector) + + def test_returns_default_when_stat_loggers_empty(self): + cls = resolve_collector_class( + _StubArgs(stat_loggers={}), "scheduler", SchedulerMetricsCollector + ) + self.assertIs(cls, SchedulerMetricsCollector) + + def test_returns_default_when_role_missing(self): + # Different role registered. Default still wins for "scheduler". + class MyTokenizer(TokenizerMetricsCollector): + pass + + cls = resolve_collector_class( + _StubArgs(stat_loggers={"tokenizer": MyTokenizer}), + "scheduler", + SchedulerMetricsCollector, + ) + self.assertIs(cls, SchedulerMetricsCollector) + + def test_returns_subclass_when_role_registered(self): + class MyScheduler(SchedulerMetricsCollector): + pass + + cls = resolve_collector_class( + _StubArgs(stat_loggers={"scheduler": MyScheduler}), + "scheduler", + SchedulerMetricsCollector, + ) + self.assertIs(cls, MyScheduler) + + def test_role_constants_match_collector_keys(self): + """The exported role constants must be the exact strings the + instantiation sites use to look up subclasses.""" + self.assertEqual(STAT_LOGGER_ROLE_SCHEDULER, "scheduler") + self.assertEqual(STAT_LOGGER_ROLE_TOKENIZER, "tokenizer") + self.assertEqual(STAT_LOGGER_ROLE_STORAGE, "storage") + self.assertEqual(STAT_LOGGER_ROLE_RADIX_CACHE, "radix_cache") + self.assertEqual(STAT_LOGGER_ROLE_EXPERT_DISPATCH, "expert_dispatch") + + +# ── DI swap behavior — actually instantiate with a custom backend ── + + +class _RecordingGauge: + """Test double that mirrors prometheus_client.Gauge constructor signature. + Records every instantiation so the test can assert the override took effect.""" + + instances = [] + + def __init__(self, *args, **kwargs): + type(self).instances.append((args, kwargs)) + + def labels(self, **kwargs): + return self + + def set(self, value): + pass + + def inc(self, amount=1): + pass + + +class _RecordingCounter(_RecordingGauge): + pass + + +class _RecordingHistogram(_RecordingGauge): + def observe(self, value): + pass + + +class _RecordingSummary(_RecordingGauge): + def observe(self, value): + pass + + +class TestDISwap(unittest.TestCase): + """Subclasses that set the DI hooks at class level cause the collector to + instantiate the test doubles instead of prometheus_client classes.""" + + def setUp(self): + _RecordingGauge.instances = [] + _RecordingCounter.instances = [] + _RecordingHistogram.instances = [] + _RecordingSummary.instances = [] + + def test_radix_cache_di_swap(self): + """Smallest collector (4 metrics, Counter + Histogram) — verifies the + DI shim flows through both class types.""" + + class RaySwapRadixCache(RadixCacheMetricsCollector): + _counter_cls = _RecordingCounter + _histogram_cls = _RecordingHistogram + + labels = {"cache_type": "test"} + RaySwapRadixCache(labels=labels) + + # 4 instruments total in RadixCacheMetricsCollector: + # eviction_duration_seconds (H), eviction_num_tokens (C), + # load_back_duration_seconds (H), load_back_num_tokens (C). + self.assertEqual(len(_RecordingCounter.instances), 2) + self.assertEqual(len(_RecordingHistogram.instances), 2) + + def test_expert_dispatch_di_swap(self): + """Smallest collector (1 Histogram metric).""" + + class RaySwapExpert(ExpertDispatchCollector): + _histogram_cls = _RecordingHistogram + + RaySwapExpert(ep_size=4) + self.assertEqual(len(_RecordingHistogram.instances), 1) + + def test_default_path_uses_prometheus_client(self): + """Without any subclass override, the collector instantiates the real + prometheus_client classes — the existing behavior is unchanged.""" + labels = {"cache_type": "test_default"} + collector = RadixCacheMetricsCollector(labels=labels) + # The instruments must be real prometheus_client objects, not test doubles. + self.assertIsInstance(collector.eviction_num_tokens, prometheus_client.Counter) + self.assertIsInstance( + collector.eviction_duration_seconds, prometheus_client.Histogram + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/utils/test_common.py b/test/registered/unit/utils/test_common.py new file mode 100644 index 000000000000..a15f6b003857 --- /dev/null +++ b/test/registered/unit/utils/test_common.py @@ -0,0 +1,50 @@ +import unittest +from array import array + +import torch + +from sglang.srt.utils.common import flatten_arrays_to_int64_tensor +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=5, stage="base-b", runner_config="1-gpu-small") + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestFlattenArraysToInt64Tensor(CustomTestCase): + """`flatten_arrays_to_int64_tensor` is invoked by `prepare_for_extend` + to build the per-batch input_ids tensor (pinned, async H2D) from a + list of array.array('q') per-req fill_ids slices. Tests the full + matrix of (device, pin) the production code paths through. + """ + + DEVICES = ("cpu", "cuda") + PIN_OPTIONS = (False, True) + + def _check(self, parts: list, expected: list[int]) -> None: + for device in self.DEVICES: + for pin in self.PIN_OPTIONS: + with self.subTest(device=device, pin=pin): + out = flatten_arrays_to_int64_tensor(parts, device, pin) + if device == "cuda": + torch.cuda.synchronize() + self.assertEqual(out.dtype, torch.int64) + self.assertEqual(out.device.type, device) + self.assertEqual(out.shape, (len(expected),)) + self.assertEqual(out.cpu().tolist(), expected) + + def test_single_part(self): + parts = [array("q", [1, 2, 3, 4, 5])] + self._check(parts, [1, 2, 3, 4, 5]) + + def test_multiple_parts(self): + parts = [ + array("q", [10, 20, 30]), + array("q", [100, 200]), + array("q", [1000]), + ] + self._check(parts, [10, 20, 30, 100, 200, 1000]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 61b41e4227f1..e45d361c398f 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -37,7 +37,7 @@ # TestFile("test_wave_attention_backend.py", 150), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127 # The time estimation for `test_int4fp8_moe.py` assumes `mistralai/Mixtral-8x7B-Instruct-v0.1` is already cached (running on 1xMI300X). ], - # per-commit-4-gpu-amd migrated to test/registered/distributed/ using the CI registry system + # per-commit-4-gpu-amd migrated to test/registered/ using the CI registry system "per-commit-4-gpu-amd": [], # NOTE: AMD nightly suites (nightly-amd, nightly-amd-vlm, nightly-amd-8-gpu) # have been migrated to test/registered/amd/nightly/ and are now managed